diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260818_125952.csv b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052553.csv similarity index 60% rename from benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260818_125952.csv rename to benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052553.csv index ddd0a0873..e3388de98 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260818_125952.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052553.csv @@ -1,21 +1,21 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.232032,1332.737,325.375,4.0672,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,12.190784,1462.761,47.028,0.5879,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.266464,1059.635,1092.806,13.6601,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.450592,1302.882,657.117,8.214,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.774688,1515.625,394.39,4.9299,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.43648,1634.741,225.832,2.8229,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.069632,4.117,4116.772,51.4596,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.059616,0.541,270.535,3.3817,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.060992,0.811,270.189,3.3774,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.059136,1.083,270.667,3.3833,20,True,,NVIDIA GB200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.351456,879.876,214.813,2.6852,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,13.334592,1337.289,42.994,0.5374,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.122272,251.592,259.468,3.2433,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.121664,523.391,263.975,3.2997,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,1.15248,1018.791,265.106,3.3138,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.950816,1203.739,166.291,2.0786,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.095648,0.262,261.634,3.2704,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.12048,0.512,255.84,3.198,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.121024,0.767,255.72,3.1965,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.118944,1.025,256.199,3.2025,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.23184,1333.841,325.645,4.0706,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,11.76816,1515.292,48.717,0.609,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.249216,1132.972,1168.438,14.6055,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.416448,1409.704,710.993,8.8874,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.748832,1567.957,408.007,5.1001,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.429984,1642.168,226.858,2.8357,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.069152,4.145,4145.348,51.8168,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.067456,0.537,268.548,3.3569,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.069088,0.804,268.143,3.3518,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.066208,1.075,268.871,3.3609,20,True,,NVIDIA GB200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.35376,874.145,213.414,2.6677,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,12.388768,1439.384,46.277,0.5785,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.131776,249.479,257.289,3.2161,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.132864,518.216,261.366,3.2671,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,1.162432,1010.069,262.836,3.2854,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.91392,1226.944,169.497,2.1187,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.103072,0.26,259.873,3.2484,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.129952,0.507,253.695,3.1712,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.126752,0.763,254.42,3.1803,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.13104,1.014,253.46,3.1682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png old mode 100644 new mode 100755 index 2e1298d1e..f90fd2ac7 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png old mode 100644 new mode 100755 index 349024f31..6efa7402b Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb200/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260818_130601.csv b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052426.csv similarity index 60% rename from benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260818_130601.csv rename to benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052426.csv index ba1c167b8..1e4984e15 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260818_130601.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052426.csv @@ -1,21 +1,21 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.179424,1723.502,420.777,5.2597,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,9.415008,1894.019,60.893,0.7612,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.205088,1376.749,1419.846,17.7481,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.362016,1621.664,817.897,10.2237,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.602688,1948.167,506.944,6.3368,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.155936,2031.491,280.641,3.508,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.066848,4.288,4288.222,53.6028,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.055136,0.543,271.684,3.3961,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.056224,0.814,271.409,3.3926,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.055936,1.086,271.487,3.3936,20,True,,NVIDIA GB300,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.307744,1004.854,245.326,3.0666,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,10.21984,1744.861,56.098,0.7012,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.965952,292.307,301.457,3.7682,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.969824,605.335,305.305,3.8163,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,1.006496,1166.559,303.557,3.7945,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.616224,1452.938,200.717,2.509,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.943008,0.304,303.984,3.7998,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,0.98736,0.581,290.333,3.6292,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.989728,0.869,289.644,3.6205,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.989248,1.159,289.789,3.6224,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.173344,1783.954,435.536,5.4442,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,9.20752,1936.7,62.265,0.7783,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.192832,1464.252,1510.088,18.8761,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.33296,1763.18,889.271,11.1159,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.592128,1982.91,515.984,6.4498,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.143488,2053.606,283.696,3.5462,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.070976,4.039,4038.817,50.4852,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.054912,0.543,271.742,3.3968,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.055968,0.814,271.474,3.3934,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.05536,1.086,271.635,3.3954,20,True,,NVIDIA GB300,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.300768,1028.16,251.016,3.1377,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,10.121312,1761.847,56.644,0.708,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.972672,290.288,299.375,3.7422,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,0.965376,608.124,306.711,3.8339,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,0.997888,1176.622,306.175,3.8272,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,1.609376,1459.12,201.571,2.5196,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.938336,0.305,305.497,3.8187,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,0.987808,0.58,290.202,3.6275,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.987968,0.87,290.159,3.627,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.991264,1.157,289.199,3.615,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png old mode 100644 new mode 100755 index 08f20b040..ca70ecfd3 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png old mode 100644 new mode 100755 index 2812d75ff..0859d7a2c Binary files a/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/gb300/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260818_133441.csv b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260819_062044.csv similarity index 62% rename from benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260818_133441.csv rename to benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260819_062044.csv index f99aad5d3..aa305fd3d 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260818_133441.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_20260819_062044.csv @@ -1,11 +1,11 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.527392,586.353,143.152,2.9823,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,28.938208,616.216,19.811,0.4127,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.743296,379.868,391.76,8.1617,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.487552,394.654,199.046,4.1468,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,2.224064,527.924,137.374,2.862,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,3.962112,592.682,81.876,1.7058,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.099072,2.893,2893.442,60.28,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,0.74272,0.772,385.965,8.0409,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.74416,1.156,385.224,8.0255,20,True,,NVIDIA H200,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.741632,1.546,386.543,8.053,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,0.532576,580.645,141.759,2.9533,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,33.954334,525.182,16.885,0.3518,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,0.741184,380.951,392.876,8.1849,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,1.491712,393.553,198.491,4.1352,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,2.270752,517.07,134.55,2.8031,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,3.827872,613.467,84.748,1.7656,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.098304,2.916,2916.047,60.751,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,0.742944,0.772,385.848,8.0385,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,0.743744,1.156,385.439,8.03,20,True,,NVIDIA H200,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,0.741696,1.546,386.51,8.0523,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png old mode 100644 new mode 100755 index 2037774e0..42936b11c Binary files a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png old mode 100644 new mode 100755 index a00bbe167..f982e25e8 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/h200/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260818_134936.csv b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260819_062411.csv similarity index 62% rename from benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260818_134936.csv rename to benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260819_062411.csv index bd2ea60a3..771785fbf 100644 --- a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260818_134936.csv +++ b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260819_062411.csv @@ -1,21 +1,21 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.142784,270.6,66.065,3.6866,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,48.970783,364.14,11.707,0.6533,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.983968,142.318,146.773,8.1905,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.014176,291.468,147.004,8.2033,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,4.009664,292.827,76.198,4.2521,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.96848,294.695,40.711,2.2718,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.204864,1.399,1399.265,78.084,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.143264,0.501,250.741,13.9923,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.143744,0.752,250.64,13.9866,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.174624,0.976,244.055,13.6191,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.13792,271.757,66.347,3.7024,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,45.043297,395.89,12.728,0.7103,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.416576,199.322,205.561,11.4711,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.450688,239.552,120.82,6.7422,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,3.974304,295.432,76.876,4.29,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.507008,312.811,43.213,2.4115,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.386624,0.207,206.732,11.5364,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.432032,0.4,200.18,11.1707,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.398112,0.615,205.04,11.4419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.438976,0.797,199.22,11.1172,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.076768,287.191,70.115,3.9127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,47.698273,373.854,12.019,0.6707,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,2.007552,140.646,145.049,8.0943,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.01568,291.251,146.894,8.1972,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,4.00656,293.054,76.257,4.2554,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.968416,294.698,40.711,2.2718,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,0.205184,1.397,1397.083,77.9622,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.157888,0.495,247.575,13.8155,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.157312,0.743,247.702,13.8226,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,generation,cudnn,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.157408,0.991,247.685,13.8217,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,9,9,128,128,gqa,,64,1.074048,287.918,70.292,3.9226,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,62208,62208,9,9,128,128,gqa,,64,44.031937,404.983,13.02,0.7266,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,985,62208,9,9,128,128,gqa,,64,1.438464,196.289,202.434,11.2965,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,2048,62208,9,9,128,128,gqa,,64,2.431712,241.422,121.763,6.7948,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,4096,62208,9,9,128,128,gqa,,64,3.973152,295.518,76.898,4.2912,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,context,cudnn_oss,bfloat16,bfloat16,1,8192,62208,9,9,128,128,gqa,,64,7.503264,312.967,43.235,2.4127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,1,62208,9,9,128,128,gqa,,64,1.40624,0.204,203.848,11.3754,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,2,62208,9,9,128,128,gqa,,64,1.418016,0.404,202.158,11.2812,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,3,62208,9,9,128,128,gqa,,64,1.41664,0.607,202.358,11.2923,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +auto_regressive_dit,ar_dit,generation,cudnn_oss,bfloat16,bfloat16,1,4,62208,9,9,128,128,gqa,,64,1.413984,0.811,202.741,11.3137,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png old mode 100644 new mode 100755 index 673394e33..877bec120 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png and b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_context.png differ diff --git a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png old mode 100644 new mode 100755 index dd1972ea0..4e32da253 Binary files a/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png and b/benchmark/attention_inference/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260818_130603.csv b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260818_130603.csv deleted file mode 100644 index 060996b4c..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260818_130603.csv +++ /dev/null @@ -1,225 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.58544,469.524,462.101,5.7763,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.91232,493.479,121.42,1.5177,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.745376,502.9,15.347,0.1918,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.902529,491.332,11.246,0.1406,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.805887,523.394,11.98,0.1497,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,34.038719,516.829,7.886,0.0986,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.387328,44.355,346.861,4.3358,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.478306,57.15,446.918,5.5865,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.527904,3.264,12.774,0.1597,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1264.319702,3.479,13.615,0.1702,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.507776,4.905,12.811,0.1601,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1262.240112,5.226,13.65,0.1706,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.501792,6.544,12.83,0.1604,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1261.393921,6.973,13.673,0.1709,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.593536,463.119,455.798,5.6975,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.943968,1115.132,274.376,3.4297,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.824832,1556.923,47.514,0.5939,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,6.154944,1429.11,32.71,0.4089,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,6.083488,1445.896,33.094,0.4137,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,12.182304,1444.077,22.035,0.2754,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.6568,6.466,50.568,0.6321,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,321.818939,6.833,53.436,0.6679,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.695232,12.748,49.895,0.6237,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,322.061829,13.656,53.448,0.6681,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.699328,19.093,49.868,0.6234,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,322.499268,20.456,53.427,0.6678,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.702016,25.433,49.867,0.6233,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,322.130676,27.306,53.54,0.6693,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.314944,436.392,432.823,5.4103,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,4.273632,514.556,127.587,1.5948,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,4.265408,515.548,23.6,0.295,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.769984,501.489,19.13,0.2391,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.762432,501.921,15.317,0.1915,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,17.629601,498.939,11.42,0.1427,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.228256,37.633,588.301,7.3538,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.333696,49.231,769.611,9.6201,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.584448,1.623,12.693,0.1587,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,616.182922,3.569,27.908,0.3489,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.567232,2.439,12.72,0.159,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,615.440735,5.36,27.956,0.3494,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.560192,3.254,12.735,0.1592,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,615.218201,7.149,27.979,0.3497,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.330432,415.937,412.535,5.1567,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.991968,1103.945,273.729,3.4216,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.468064,1497.907,68.569,0.8571,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.848704,1543.876,58.894,0.7362,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.944544,1493.626,45.582,0.5698,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.98928,1468.639,33.614,0.4202,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.355808,6.336,99.043,1.238,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,160.919128,6.833,106.813,1.3352,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.397344,12.295,96.146,1.2018,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,161.492767,13.617,106.486,1.3311,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.40576,18.332,95.617,1.1952,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,161.01236,20.486,106.855,1.3357,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.397344,24.589,96.24,1.203,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,161.447586,27.241,106.619,1.3327,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.17392,395.121,397.919,4.974,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.92256,571.9,143.987,1.7998,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.951104,563.533,42.994,0.5374,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,4.098656,536.523,36.84,0.4605,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.102016,536.084,24.54,0.3067,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.888576,494.798,18.875,0.2359,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.13472,31.881,996.515,12.4564,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.741728,46.821,1463.504,18.2938,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.521024,0.816,12.763,0.1595,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,311.232666,3.533,55.226,0.6903,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.506464,1.226,12.784,0.1598,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,311.148712,5.301,55.255,0.6907,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.493984,1.637,12.802,0.16,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,310.647064,7.079,55.358,0.692,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.197536,347.883,350.346,4.3793,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.05824,1039.0,261.589,3.2699,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.791776,1388.665,105.947,1.3243,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.441568,1525.438,104.744,1.3093,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.495648,1470.281,67.304,0.8413,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.806496,1567.095,59.78,0.7472,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.351072,3.179,99.366,1.2421,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,81.012131,6.786,212.117,2.6515,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.393696,6.163,96.35,1.2044,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,81.427681,13.503,211.086,2.6386,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.391328,9.261,96.538,1.2067,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,81.959709,20.123,209.767,2.6221,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.395968,12.307,96.241,1.203,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,82.159615,26.765,209.308,2.6163,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.1032,332.943,345.461,4.3183,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.949088,579.246,150.256,1.8782,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.791808,306.816,42.135,0.5267,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.645728,301.589,39.116,0.489,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.870624,587.778,44.844,0.5605,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,4.349088,505.629,34.719,0.434,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.133056,16.14,1008.854,12.6107,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.743136,23.408,1463.15,18.2894,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.517472,0.408,12.765,0.1596,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,158.423294,3.47,108.469,1.3559,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.501216,0.613,12.786,0.1598,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,158.292633,5.21,108.572,1.3572,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.490112,0.819,12.801,0.16,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,158.119553,6.954,108.704,1.3588,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.151744,226.432,234.946,2.9368,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.559168,983.168,255.033,3.1879,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.758336,724.95,99.557,1.2445,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.398688,786.102,101.957,1.2745,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.785184,1400.323,106.836,1.3355,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.440128,1526.964,104.848,1.3106,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.348704,1.592,99.528,1.2441,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,41.669086,6.597,412.343,5.1543,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.39232,3.085,96.422,1.2053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,41.977825,13.096,409.36,5.117,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.395232,4.617,96.233,1.2029,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,42.002335,19.633,409.172,5.1146,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.392256,6.17,96.45,1.2056,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,42.002911,26.177,409.216,5.1152,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.122336,489.832,480.22,6.0027,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,17.89024,491.67,120.505,1.5063,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,16.913536,520.062,11.903,0.1488,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,34.155521,515.061,7.859,0.0982,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,34.215744,514.155,9.807,0.1226,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,68.596642,512.917,5.87,0.0734,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.71184,48.269,188.919,2.3615,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,72.559875,60.613,237.231,2.9654,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.605728,2.911,5.708,0.0714,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2667.791504,3.297,6.465,0.0808,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.590401,4.37,5.723,0.0715,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2663.615967,4.953,6.488,0.0811,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.57056,5.831,5.739,0.0717,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2661.652344,6.609,6.505,0.0813,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.10896,495.74,486.012,6.0752,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.995264,1100.163,269.644,3.3705,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,5.971392,1473.039,33.715,0.4214,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,12.37232,1421.899,21.696,0.2712,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,11.800512,1490.798,28.435,0.3554,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,23.63184,1488.855,17.039,0.213,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.241568,6.555,25.656,0.3207,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,643.842957,6.831,26.735,0.3342,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.323968,12.908,25.309,0.3164,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,647.710876,13.58,26.628,0.3328,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.293088,19.474,25.506,0.3188,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,646.527893,20.408,26.728,0.3341,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.302592,25.919,25.509,0.3189,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,649.327881,27.093,26.665,0.3333,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.5768,476.557,469.023,5.8628,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.901824,494.061,121.563,1.5195,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.83936,497.553,15.184,0.1898,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.553568,501.1,11.469,0.1434,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.840193,522.327,11.955,0.1494,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,33.932255,518.45,7.911,0.0989,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.38864,44.205,345.69,4.3211,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.517025,57.092,446.469,5.5809,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.523008,3.265,12.78,0.1597,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1263.834351,3.48,13.62,0.1702,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.51456,4.902,12.802,0.16,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1262.46582,5.226,13.648,0.1706,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.504768,6.542,12.827,0.1603,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1261.344238,6.974,13.673,0.1709,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.592352,464.045,456.709,5.7089,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.92672,1120.031,275.581,3.4448,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.831168,1553.439,47.407,0.5926,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.993024,1467.722,33.593,0.4199,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.819264,1511.547,34.597,0.4325,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,11.92,1475.855,22.52,0.2815,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.660128,6.458,50.505,0.6313,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,321.155518,6.847,53.546,0.6693,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.697952,12.735,49.845,0.6231,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,321.720825,13.67,53.504,0.6688,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.699744,19.091,49.861,0.6233,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,321.58429,20.514,53.579,0.6697,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.700064,25.451,49.903,0.6238,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,322.900818,27.241,53.413,0.6677,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.314112,437.548,433.969,5.4246,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.82032,575.612,142.726,1.7841,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,4.16128,528.449,24.19,0.3024,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.825856,498.314,19.009,0.2376,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.568064,513.307,15.665,0.1958,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,17.556032,501.03,11.468,0.1433,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.229184,37.481,585.919,7.324,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.329248,49.241,769.764,9.6221,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.584544,1.623,12.693,0.1587,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,616.499634,3.567,27.894,0.3487,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.567136,2.439,12.72,0.159,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,615.499146,5.359,27.953,0.3494,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.559104,3.254,12.736,0.1592,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,614.701233,7.155,28.003,0.35,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.33008,416.381,412.975,5.1622,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.99456,1102.51,273.373,3.4172,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.482528,1483.293,67.9,0.8487,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.888032,1522.852,58.092,0.7262,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.86448,1535.373,46.856,0.5857,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,6.094432,1443.3,33.035,0.4129,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.356992,6.33,98.957,1.237,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,160.382553,6.856,107.17,1.3396,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.40016,12.27,95.952,1.1994,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,160.724121,13.682,106.995,1.3374,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.397376,18.442,96.191,1.2024,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,160.791962,20.514,107.002,1.3375,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.398272,24.573,96.176,1.2022,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,161.117691,27.297,106.838,1.3355,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.177824,386.447,389.183,4.8648,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.916032,573.848,144.478,1.806,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.908384,576.148,43.957,0.5495,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.914144,561.815,38.577,0.4822,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.056224,542.136,24.817,0.3102,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.664352,507.602,19.363,0.242,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.132928,32.31,1009.949,12.6244,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.738208,46.835,1463.943,18.2993,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.517152,0.817,12.768,0.1596,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,311.520935,3.529,55.175,0.6897,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.505056,1.227,12.786,0.1598,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,311.573639,5.293,55.179,0.6897,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.497792,1.637,12.798,0.16,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,310.68396,7.078,55.351,0.6919,20,True,,NVIDIA GB200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.198912,345.477,347.923,4.349,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.0512,1045.959,263.341,3.2918,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.789024,1393.508,106.316,1.329,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.441376,1525.642,104.758,1.3095,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.468192,1497.776,68.563,0.857,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.836992,1550.25,59.137,0.7392,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.349824,3.182,99.458,1.2432,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,81.346786,6.758,211.245,2.6406,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.396544,6.151,96.154,1.2019,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,81.109856,13.556,211.913,2.6489,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.397088,9.223,96.14,1.2017,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,81.605217,20.21,210.678,2.6335,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.397728,12.291,96.119,1.2015,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,81.62307,26.941,210.684,2.6335,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260819_055704.csv b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260819_055704.csv new file mode 100644 index 000000000..f1d879a24 --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_20260819_055704.csv @@ -0,0 +1,225 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.558912,491.809,484.034,6.0504,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.597472,511.551,125.866,1.5733,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.625952,509.862,15.56,0.1945,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.451103,504.042,11.537,0.1442,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.525024,532.289,12.183,0.1523,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,33.608894,523.438,7.987,0.0998,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.39824,43.139,337.356,4.217,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.641216,56.909,445.034,5.5629,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.616,3.237,12.668,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1276.078979,3.447,13.489,0.1686,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.620256,4.853,12.675,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1274.32666,5.177,13.521,0.169,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.604288,6.48,12.706,0.1588,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1273.384644,6.908,13.544,0.1693,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.599232,458.717,451.466,5.6433,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.922912,1121.118,275.849,3.4481,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.770816,1587.275,48.44,0.6055,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.774976,1523.139,34.862,0.4358,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.986528,1469.315,33.63,0.4204,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,11.871968,1481.826,22.611,0.2826,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.644672,6.496,50.8,0.635,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,315.228149,6.976,54.553,0.6819,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.688896,12.778,50.013,0.6252,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,317.050995,13.872,54.292,0.6787,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.707712,19.034,49.714,0.6214,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,317.656586,20.768,54.242,0.678,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.684384,25.6,50.195,0.6274,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,316.928223,27.754,54.419,0.6802,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.30528,450.206,446.524,5.5816,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,4.062528,541.294,134.217,1.6777,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,4.063168,541.209,24.775,0.3097,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.559328,513.831,19.601,0.245,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.464448,519.59,15.857,0.1982,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,17.563393,500.82,11.463,0.1433,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.22912,37.491,586.083,7.326,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.42016,49.041,766.643,9.583,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.687584,1.607,12.571,0.1571,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,621.229736,3.54,27.682,0.346,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.66912,2.415,12.598,0.1575,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,620.331604,5.317,27.735,0.3467,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.662048,3.223,12.613,0.1577,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,619.824951,7.096,27.771,0.3471,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.329152,417.555,414.14,5.1767,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.987712,1106.309,274.315,3.4289,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.442144,1524.829,69.801,0.8725,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.768832,1588.412,60.593,0.7574,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.775936,1584.347,48.35,0.6044,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.79872,1516.903,34.719,0.434,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.356448,6.333,98.996,1.2375,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,158.135681,6.953,108.693,1.3587,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.39408,12.323,96.371,1.2046,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,157.529984,13.959,109.164,1.3646,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.394784,18.476,96.369,1.2046,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,158.188477,20.852,108.763,1.3595,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.398912,24.562,96.132,1.2016,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,157.492157,27.925,109.297,1.3662,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.173408,396.288,399.094,4.9887,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.816,605.458,152.436,1.9055,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,2.07504,529.875,40.426,0.5053,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,4.26064,516.125,35.439,0.443,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.023232,546.581,25.021,0.3128,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.59312,511.81,19.524,0.2441,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.139264,30.84,964.0,12.05,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.858624,46.359,1449.077,18.1135,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.618624,0.809,12.646,0.1581,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,312.820984,3.515,54.946,0.6868,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.611968,1.214,12.657,0.1582,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,312.224365,5.282,55.064,0.6883,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.60352,1.62,12.67,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,312.292053,7.042,55.066,0.6883,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.199456,344.535,346.974,4.3372,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.040704,1056.508,265.997,3.325,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.77504,1418.652,108.235,1.3529,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.411744,1557.664,106.956,1.337,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.43664,1530.671,70.069,0.8759,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.849216,1543.599,58.884,0.736,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.351328,3.178,99.347,1.2418,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,79.039261,6.955,217.412,2.7176,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.392512,6.169,96.432,1.2054,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,79.475037,13.835,216.272,2.7034,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.399232,9.209,95.993,1.1999,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,79.385986,20.775,216.568,2.7071,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.39408,12.323,96.371,1.2046,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,79.229568,27.755,217.048,2.7131,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.09952,345.255,358.235,4.4779,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.9568,574.578,149.045,1.8631,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.831904,300.101,41.213,0.5152,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.721216,295.471,38.323,0.479,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.911328,575.261,43.889,0.5486,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,4.176992,526.461,36.149,0.4519,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.139776,15.364,960.352,12.0044,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.858816,23.179,1448.877,18.111,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.611328,0.405,12.652,0.1581,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,158.748383,3.463,108.247,1.3531,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.603808,0.608,12.662,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,158.647293,5.198,108.329,1.3541,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.599072,0.81,12.669,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,158.535614,6.935,108.419,1.3552,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.157216,218.551,226.768,2.8346,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.55712,986.782,255.971,3.1996,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.75792,725.348,99.611,1.2451,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.398272,786.336,101.988,1.2748,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.776512,1415.962,108.029,1.3504,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.419552,1549.097,106.368,1.3296,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.351008,1.59,99.358,1.242,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,39.710274,6.922,432.683,5.4085,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.392256,3.085,96.427,1.2053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,40.053185,13.726,429.031,5.3629,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.39376,4.622,96.334,1.2042,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,39.690529,20.777,433.004,5.4126,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.394464,6.16,96.297,1.2037,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,39.829056,27.606,431.551,5.3944,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.077536,510.197,500.186,6.2523,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,17.346464,507.083,124.283,1.5535,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,16.592833,530.114,12.133,0.1517,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,33.557472,524.241,7.999,0.1,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,33.469727,525.615,10.025,0.1253,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,67.924416,517.993,5.928,0.0741,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.715904,47.995,187.846,2.3481,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,73.038017,60.216,235.678,2.946,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.764032,2.892,5.67,0.0709,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2690.69043,3.269,6.41,0.0801,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.748192,4.341,5.685,0.0711,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2688.665527,4.907,6.427,0.0803,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.740641,5.789,5.698,0.0712,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2684.7771,6.553,6.449,0.0806,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.12256,489.734,480.124,6.0016,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.8728,1117.276,273.838,3.423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,5.916128,1486.799,34.03,0.4254,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,11.582976,1518.797,23.175,0.2897,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,11.523968,1526.574,29.117,0.364,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,22.773504,1544.97,17.681,0.221,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.222304,6.579,25.751,0.3219,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,634.460999,6.932,27.131,0.3391,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.277984,13.02,25.529,0.3191,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,637.260986,13.803,27.064,0.3383,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.26512,19.578,25.641,0.3205,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,638.134094,20.676,27.08,0.3385,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.267648,26.091,25.679,0.321,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,639.128906,27.525,27.09,0.3386,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.552288,497.708,489.84,6.123,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,8.7088,505.012,124.257,1.5532,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,8.561312,513.712,15.677,0.196,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,17.627808,498.99,11.421,0.1428,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,16.567583,530.922,12.152,0.1519,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,33.467999,525.642,8.021,0.1003,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.394112,43.591,340.89,4.2611,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,38.611904,56.952,445.372,5.5671,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.619808,3.235,12.663,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1275.861328,3.447,13.492,0.1686,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.612736,4.856,12.684,0.1585,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1274.179932,5.178,13.523,0.169,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.602176,6.482,12.709,0.1589,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1272.534058,6.912,13.553,0.1694,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.597952,459.699,452.432,5.6554,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.884576,1132.182,278.571,3.4821,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.898432,1517.388,46.307,0.5788,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.978336,1471.328,33.676,0.421,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.848192,1504.07,34.425,0.4303,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,11.8256,1487.636,22.7,0.2837,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.64464,6.496,50.8,0.635,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,315.169983,6.977,54.563,0.682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.686272,12.791,50.062,0.6258,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,316.288727,13.905,54.423,0.6803,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.6832,19.208,50.168,0.6271,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,316.526489,20.842,54.435,0.6804,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.689248,25.553,50.104,0.6263,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,315.72699,27.86,54.626,0.6828,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.304608,451.199,447.509,5.5939,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,4.179936,526.09,130.447,1.6306,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,4.056512,542.097,24.815,0.3102,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,8.563264,513.595,19.592,0.2449,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,8.5776,512.736,15.647,0.1956,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,17.283232,508.938,11.649,0.1456,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.229696,37.397,584.613,7.3077,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,22.410273,49.063,766.981,9.5873,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.686592,1.608,12.572,0.1571,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,620.949585,3.541,27.694,0.3462,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.664992,2.416,12.603,0.1575,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,620.364685,5.317,27.734,0.3467,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.65712,3.224,12.619,0.1577,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,619.917969,7.095,27.767,0.3471,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.330112,416.34,412.935,5.1617,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.982528,1109.202,275.032,3.4379,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.455232,1511.115,69.173,0.8647,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.816096,1561.753,59.576,0.7447,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.906784,1513.028,46.174,0.5772,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.902912,1490.128,34.106,0.4263,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.355232,6.338,99.085,1.2386,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,156.882751,7.008,109.561,1.3695,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.40176,12.256,95.843,1.198,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,157.391129,13.972,109.261,1.3658,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.399552,18.413,96.041,1.2005,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,157.935043,20.885,108.937,1.3617,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.397568,24.585,96.224,1.2028,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,157.446686,27.934,109.329,1.3666,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.172224,399.012,401.837,5.023,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.823584,602.94,151.802,1.8975,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.954944,562.426,42.91,0.5364,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,4.111264,534.878,36.727,0.4591,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,4.007872,548.676,25.116,0.314,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,8.630624,509.586,19.439,0.243,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.139232,30.848,964.222,12.0528,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.854432,46.376,1449.59,18.1199,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.616064,0.809,12.649,0.1581,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,312.44635,3.519,55.012,0.6876,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.60496,1.215,12.665,0.1583,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,312.206543,5.283,55.068,0.6883,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.599648,1.621,12.675,0.1584,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,312.025665,7.048,55.113,0.6889,20,True,,NVIDIA GB200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.19984,343.872,346.307,4.3288,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.039616,1057.613,266.275,3.3284,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.775648,1417.539,108.15,1.3519,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.41984,1548.782,106.346,1.3293,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.453248,1513.178,69.268,0.8658,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.861408,1537.022,58.633,0.7329,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.350816,3.18,99.385,1.2423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,79.123489,6.948,217.18,2.7148,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.397664,6.146,96.077,1.201,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,79.271965,13.87,216.826,2.7103,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.395712,9.232,96.235,1.2029,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,79.486145,20.749,216.295,2.7037,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.395808,12.308,96.252,1.2031,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,79.293953,27.733,216.872,2.7109,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png old mode 100644 new mode 100755 index ca2c380eb..64caef7b5 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png old mode 100644 new mode 100755 index 7ca2b9368..91897e5f3 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/gb200/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260818_131234.csv b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260818_131234.csv deleted file mode 100644 index 03b0df8df..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260818_131234.csv +++ /dev/null @@ -1,225 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.499232,550.602,541.898,6.7737,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,7.333728,599.701,147.555,1.8444,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.557984,581.907,17.758,0.222,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.237344,577.272,13.213,0.1652,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.971104,629.592,14.41,0.1801,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,27.949535,629.427,9.604,0.1201,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.380032,45.206,353.52,4.419,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.917919,57.994,453.523,5.669,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.456576,3.286,12.861,0.1608,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1244.747559,3.533,13.829,0.1729,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.436128,4.939,12.899,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1244.266235,5.302,13.848,0.1731,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.426336,6.591,12.923,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1243.691528,7.073,13.868,0.1733,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.58096,473.144,465.665,5.8208,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.824768,1149.886,282.927,3.5366,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.738848,1605.802,49.005,0.6126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.363392,1640.024,37.537,0.4692,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.488032,1602.777,36.685,0.4586,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.8536,1620.862,24.732,0.3092,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.642816,6.501,50.835,0.6354,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,300.115112,7.327,57.3,0.7163,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.681248,12.815,50.156,0.6269,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,300.765839,14.623,57.232,0.7154,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.688192,19.173,50.075,0.6259,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,300.883087,21.926,57.265,0.7158,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.686976,25.575,50.146,0.6268,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,301.090546,29.214,57.282,0.716,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.2672,514.367,510.16,6.377,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.513568,625.866,155.187,1.9398,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.636128,604.771,27.684,0.3461,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.524416,584.503,22.297,0.2787,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.509312,585.679,17.874,0.2234,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,15.117248,581.858,13.318,0.1665,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.221568,38.769,606.059,7.5757,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.907455,50.189,784.585,9.8073,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.485792,1.638,12.812,0.1602,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,606.523804,3.626,28.353,0.3544,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.46416,2.463,12.845,0.1606,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,606.406677,5.439,28.372,0.3547,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.464448,3.283,12.851,0.1606,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,605.588928,7.262,28.424,0.3553,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.319808,429.755,426.24,5.328,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.951328,1126.937,279.43,3.4929,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.426688,1541.348,70.557,0.882,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.718432,1617.862,61.717,0.7715,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.726432,1613.114,49.228,0.6154,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.403168,1627.951,37.261,0.4658,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.352544,6.351,99.282,1.241,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,150.154526,7.323,114.47,1.4309,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.39024,12.357,96.637,1.208,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,150.422562,14.619,114.322,1.429,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.392576,18.505,96.522,1.2065,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,150.446976,21.925,114.359,1.4295,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.399296,24.555,96.105,1.2013,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,150.498398,29.223,114.376,1.4297,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.149696,459.06,462.31,5.7789,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.659232,662.663,166.839,2.0855,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.797536,611.677,46.667,0.5833,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.676224,598.174,41.073,0.5134,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.625408,606.559,27.766,0.3471,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.461216,589.454,22.486,0.2811,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.131008,32.784,1024.75,12.8094,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.413248,48.168,1505.624,18.8203,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.419104,0.824,12.888,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.551941,3.575,55.887,0.6986,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.39792,1.239,12.918,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.616852,5.361,55.889,0.6986,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.39744,1.652,12.921,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,306.854156,7.166,56.042,0.7005,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.192576,356.843,359.37,4.4921,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.016736,1081.413,272.267,3.4033,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.764096,1438.97,109.785,1.3723,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.400512,1570.157,107.814,1.3477,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.423712,1544.57,70.705,0.8838,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.728736,1611.752,61.483,0.7685,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.349344,3.183,99.493,1.2437,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,75.172546,7.313,228.595,2.8574,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.394208,6.161,96.315,1.2039,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,75.363167,14.59,228.072,2.8509,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.392736,9.252,96.44,1.2055,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,75.419037,21.868,227.959,2.8495,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.396768,12.3,96.185,1.2023,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,75.442047,29.149,227.945,2.8493,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.088864,386.655,401.193,5.0149,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.847744,648.493,168.219,2.1027,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.70768,321.931,44.211,0.5526,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.469856,316.875,41.099,0.5137,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.768992,621.547,47.42,0.5928,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,3.562048,617.348,42.39,0.5299,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.13152,16.328,1020.636,12.758,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.40352,24.105,1506.725,18.8341,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.409824,0.413,12.897,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,156.845093,3.505,109.561,1.3695,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.390624,0.62,12.922,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,156.631714,5.265,109.723,1.3715,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.382656,0.827,12.933,0.1617,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,156.417892,7.029,109.887,1.3736,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.15616,220.029,228.302,2.8538,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.538848,1020.243,264.65,3.3081,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.750816,732.211,100.554,1.2569,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.394592,788.411,102.257,1.2782,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.761088,1444.658,110.219,1.3777,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.40592,1564.117,107.399,1.3425,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.349568,1.591,99.465,1.2433,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,37.830078,7.266,454.188,5.6773,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.39248,3.084,96.411,1.2051,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,37.883297,14.512,453.605,5.6701,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.385248,4.651,96.926,1.2116,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,37.870399,21.775,453.815,5.6727,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.380128,6.224,97.298,1.2162,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,37.872543,29.032,453.845,5.6731,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,0.965696,569.285,558.114,6.9764,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,14.645376,600.605,147.205,1.8401,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,14.01936,627.425,14.361,0.1795,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,27.989344,628.532,9.591,0.1199,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,28.068417,626.761,11.955,0.1494,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,56.594913,621.688,7.115,0.0889,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.701888,48.953,191.597,2.395,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,70.960449,61.979,242.578,3.0322,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.306528,2.949,5.781,0.0723,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2618.563477,3.359,6.586,0.0823,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.274944,4.429,5.8,0.0725,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2615.818359,5.044,6.606,0.0826,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.2752,5.905,5.812,0.0726,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2614.961182,6.728,6.621,0.0828,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.09248,503.218,493.344,6.1668,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.587616,1159.27,284.13,3.5516,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,5.422624,1622.11,37.127,0.4641,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,10.87952,1617.0,24.673,0.3084,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,10.526016,1671.305,31.878,0.3985,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,20.704224,1699.381,19.448,0.2431,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.193888,6.615,25.892,0.3236,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,602.45636,7.3,28.572,0.3572,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.258816,13.067,25.622,0.3203,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,605.099121,14.537,28.503,0.3563,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.254656,19.617,25.692,0.3212,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,605.268738,21.799,28.55,0.3569,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.249728,26.18,25.766,0.3221,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,605.571411,29.051,28.591,0.3574,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.499488,550.319,541.62,6.7702,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,7.291264,603.194,148.415,1.8552,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.591968,579.303,17.679,0.221,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.257536,576.508,13.195,0.1649,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.969472,629.665,14.412,0.1801,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,28.010559,628.056,9.583,0.1198,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.380832,45.111,352.777,4.4097,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.91581,57.998,453.548,5.6694,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.454016,3.287,12.864,0.1608,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1245.676025,3.531,13.819,0.1727,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.432032,4.941,12.904,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1244.362915,5.302,13.847,0.1731,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.421056,6.594,12.93,0.1616,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1243.388184,7.074,13.871,0.1734,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.576352,476.927,469.388,5.8673,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.822688,1150.511,283.081,3.5385,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.738336,1606.102,49.014,0.6127,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.39888,1629.244,37.29,0.4661,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.448736,1614.336,36.949,0.4619,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.82832,1624.646,24.79,0.3099,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.6392,6.509,50.905,0.6363,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,299.948822,7.331,57.332,0.7166,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.690688,12.77,49.98,0.6247,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,299.889984,14.666,57.399,0.7175,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.684064,19.202,50.152,0.6269,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,299.520233,22.025,57.526,0.7191,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.685344,25.591,50.177,0.6272,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,299.761688,29.344,57.536,0.7192,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.267136,514.491,510.283,6.3785,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.535104,622.053,154.241,1.928,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.700224,594.295,27.205,0.3401,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.506912,585.866,22.349,0.2794,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.565152,581.356,17.742,0.2218,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,15.115488,581.926,13.319,0.1665,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.221952,38.702,605.01,7.5626,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.888832,50.232,785.252,9.8157,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.483968,1.639,12.815,0.1602,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,606.831238,3.624,28.338,0.3542,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.464448,2.463,12.845,0.1606,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,606.071167,5.442,28.388,0.3548,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.46144,3.284,12.855,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,605.916443,7.259,28.409,0.3551,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.320768,428.468,424.964,5.3121,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.951392,1126.9,279.421,3.4928,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.421376,1547.109,70.821,0.8853,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.719424,1617.271,61.694,0.7712,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.733568,1608.903,49.1,0.6137,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.421824,1622.349,37.133,0.4642,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.350432,6.361,99.437,1.243,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,149.930084,7.333,114.642,1.433,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.391488,12.346,96.55,1.2069,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,150.377441,14.623,114.357,1.4295,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.39984,18.409,96.021,1.2003,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,150.438873,21.926,114.366,1.4296,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.396384,24.606,96.306,1.2038,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,150.506912,29.222,114.37,1.4296,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.150432,456.814,460.048,5.7506,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.669184,658.712,165.844,2.073,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.80016,610.785,46.599,0.5825,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.60176,610.541,41.923,0.524,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.592224,612.162,28.023,0.3503,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.539296,583.35,22.253,0.2782,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.13168,32.617,1019.521,12.744,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.40112,48.219,1507.226,18.8403,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.417792,0.825,12.89,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.552246,3.575,55.887,0.6986,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.399488,1.239,12.916,0.1614,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.677856,5.36,55.878,0.6985,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.390176,1.653,12.93,0.1616,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,307.19812,7.158,55.979,0.6997,20,True,,NVIDIA GB300,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.195968,350.667,353.15,4.4144,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.01376,1084.588,273.067,3.4133,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.768224,1431.238,109.195,1.3649,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.404768,1565.4,107.487,1.3436,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.421568,1546.9,70.811,0.8851,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.722688,1615.333,61.62,0.7703,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.349152,3.183,99.507,1.2438,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,75.156769,7.315,228.643,2.858,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.397376,6.147,96.097,1.2012,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,75.259453,14.61,228.387,2.8548,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.391712,9.258,96.511,1.2064,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,75.35405,21.887,228.156,2.8519,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.39072,12.353,96.604,1.2075,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,75.34816,29.185,228.229,2.8529,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260819_055559.csv b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260819_055559.csv new file mode 100644 index 000000000..9dd802d55 --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_20260819_055559.csv @@ -0,0 +1,225 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.488576,562.61,553.717,6.9215,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,7.103904,619.103,152.329,1.9041,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.417568,592.923,18.095,0.2262,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.072256,583.595,13.357,0.167,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.5552,648.909,14.852,0.1857,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,27.301121,644.376,9.832,0.1229,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.385888,44.52,348.155,4.3519,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.888962,58.039,453.87,5.6734,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.459424,3.285,12.857,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1244.87439,3.533,13.827,0.1728,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.438176,4.938,12.896,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1243.452271,5.305,13.857,0.1732,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.428064,6.59,12.921,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1242.27124,7.081,13.883,0.1735,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.577312,476.134,468.607,5.8576,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.819808,1151.379,283.294,3.5412,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.727776,1612.319,49.204,0.6151,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.385152,1633.397,37.385,0.4673,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.405664,1627.199,37.244,0.4655,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.721632,1640.812,25.037,0.313,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.632832,6.525,51.028,0.6379,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,299.032074,7.354,57.508,0.7188,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.67616,12.839,50.251,0.6281,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,299.847076,14.668,57.407,0.7176,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.681056,19.224,50.208,0.6276,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,299.682098,22.014,57.495,0.7187,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.680128,25.64,50.274,0.6284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,299.921387,29.328,57.505,0.7188,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.26576,517.154,512.925,6.4116,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.364608,653.575,162.057,2.0257,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.615424,608.234,27.843,0.348,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.449792,590.358,22.52,0.2815,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.50688,585.869,17.879,0.2235,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,14.996512,586.543,13.425,0.1678,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.223776,38.386,600.079,7.501,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.877056,50.259,785.675,9.8209,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.509888,1.635,12.783,0.1598,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,606.695129,3.625,28.345,0.3543,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.486528,2.457,12.818,0.1602,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,605.82782,5.445,28.399,0.355,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.48592,3.277,12.825,0.1603,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,605.638367,7.262,28.422,0.3553,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.320864,428.34,424.837,5.3105,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.945088,1130.552,280.326,3.5041,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.419168,1549.516,70.931,0.8866,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.710752,1622.445,61.891,0.7736,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.728128,1612.112,49.198,0.615,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.364128,1639.799,37.532,0.4692,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.347712,6.374,99.638,1.2455,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,149.034363,7.378,115.331,1.4416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.39168,12.345,96.537,1.2067,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,149.31897,14.727,115.167,1.4396,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.392512,18.506,96.527,1.2066,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,149.243332,22.102,115.282,1.441,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.389632,24.726,96.774,1.2097,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,149.055969,29.506,115.483,1.4435,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.148064,464.12,467.406,5.8426,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.652512,665.358,167.517,2.094,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.783264,616.573,47.041,0.588,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.569184,616.114,42.305,0.5288,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.564608,616.905,28.24,0.353,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.464608,589.187,22.476,0.2809,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.13456,31.919,997.7,12.4712,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.398464,48.231,1507.577,18.8447,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.448288,0.822,12.852,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.523682,3.575,55.892,0.6987,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.4248,1.236,12.884,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.361572,5.366,55.936,0.6992,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.418816,1.649,12.895,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,306.934021,7.164,56.027,0.7003,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.19344,355.25,357.765,4.4721,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.006144,1092.797,275.134,3.4392,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.755648,1455.058,111.012,1.3877,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.40288,1567.506,107.632,1.3454,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.41456,1554.563,71.162,0.8895,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.696448,1631.052,62.22,0.7777,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.346496,3.19,99.704,1.2463,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,74.538849,7.375,230.538,2.8817,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.388384,6.187,96.719,1.209,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,74.661728,14.727,230.215,2.8777,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.38896,9.277,96.703,1.2088,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,74.657249,22.091,230.285,2.8786,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.391392,12.347,96.557,1.207,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,74.770912,29.41,229.991,2.8749,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.090048,381.571,395.918,4.949,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.837376,656.522,170.301,2.1288,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,1.701344,323.13,44.375,0.5547,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,3.467872,317.057,41.122,0.514,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,1.759584,624.87,47.674,0.5959,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,3.535328,622.014,42.71,0.5339,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.134784,15.933,995.92,12.449,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,11.395232,24.122,1507.821,18.8478,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.440544,0.411,12.859,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,156.61264,3.51,109.723,1.3715,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.416384,0.618,12.89,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,156.401886,5.273,109.885,1.3736,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.407744,0.825,12.902,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,156.339111,7.033,109.942,1.3743,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.154464,222.445,230.808,2.8851,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,0.534336,1028.858,266.885,3.3361,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,0.752928,730.157,100.272,1.2534,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,1.387008,792.722,102.816,1.2852,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,0.758848,1448.922,110.544,1.3818,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,1.396896,1574.221,108.093,1.3512,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,1.344416,1.597,99.846,1.2481,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,37.341698,7.361,460.128,5.7516,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,1.38512,3.101,96.923,1.2115,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,37.496288,14.662,458.287,5.7286,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,1.382432,4.66,97.124,1.214,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,37.477409,22.003,458.574,5.7322,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,1.38272,6.212,97.115,1.2139,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_flash-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,37.555614,29.277,457.675,5.7209,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,0.953632,576.486,565.174,7.0647,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,14.507648,606.307,148.602,1.8575,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,13.640192,644.866,14.76,0.1845,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,27.584448,637.757,9.731,0.1216,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,28.125216,625.495,11.93,0.1491,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,57.374016,613.246,7.018,0.0877,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.699392,49.128,192.281,2.4035,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,70.82045,62.101,243.057,3.0382,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,23.310305,2.948,5.78,0.0723,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2616.91333,3.361,6.591,0.0824,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,23.283232,4.427,5.798,0.0725,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2615.435791,5.045,6.607,0.0826,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,23.283712,5.903,5.809,0.0726,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2614.49585,6.729,6.622,0.0828,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,1.08848,505.067,495.157,6.1895,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,7.569824,1161.994,284.798,3.56,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,5.454464,1612.641,36.91,0.4614,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,10.732416,1639.164,25.012,0.3126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,10.346976,1700.225,32.429,0.4054,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,20.5056,1715.842,19.636,0.2455,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,5.19376,6.616,25.893,0.3237,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,597.522461,7.36,28.808,0.3601,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,5.245856,13.1,25.685,0.3211,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,601.865784,14.615,28.656,0.3582,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,5.24736,19.644,25.728,0.3216,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,601.77832,21.925,28.716,0.3589,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,5.23104,26.274,25.858,0.3232,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,601.027893,29.27,28.807,0.3601,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.497376,552.656,543.92,6.799,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,6.937504,633.952,155.983,1.9498,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,7.44976,590.361,18.016,0.2252,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,15.040736,584.818,13.385,0.1673,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,13.556192,648.862,14.851,0.1856,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,28.117376,625.67,9.547,0.1193,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.382528,44.911,351.213,4.3902,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,37.918465,57.993,453.516,5.669,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.455232,3.286,12.862,0.1608,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1244.385986,3.534,13.833,0.1729,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.433664,4.94,12.902,0.1613,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1243.915771,5.303,13.852,0.1731,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.425824,6.591,12.924,0.1615,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1242.8479,7.077,13.877,0.1735,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,0.579328,474.477,466.977,5.8372,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,3.813472,1153.292,283.765,3.5471,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,2.728192,1612.074,49.197,0.615,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,5.341344,1646.794,37.692,0.4712,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,5.377536,1635.711,37.438,0.468,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,10.72144,1640.842,25.037,0.313,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,2.636224,6.517,50.963,0.637,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,298.720856,7.361,57.568,0.7196,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,2.676192,12.839,50.25,0.6281,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,299.616241,14.679,57.452,0.7181,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,2.68112,19.223,50.207,0.6276,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,299.665924,22.015,57.498,0.7187,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,2.674528,25.694,50.38,0.6297,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,299.657471,29.354,57.556,0.7194,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.265376,517.903,513.667,6.4208,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,3.279744,670.486,166.251,2.0781,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,3.607808,609.518,27.902,0.3488,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,7.384224,595.6,22.72,0.284,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,7.352224,598.193,18.255,0.2282,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,15.003104,586.285,13.419,0.1677,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.224672,38.233,597.686,7.4711,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,21.868769,50.278,785.973,9.8247,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.506848,1.635,12.787,0.1598,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,606.50531,3.626,28.354,0.3544,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.484512,2.458,12.82,0.1603,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,605.378906,5.449,28.42,0.3553,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.481568,3.278,12.83,0.1604,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,605.303101,7.266,28.438,0.3555,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,0.319904,429.626,426.112,5.3264,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,1.942048,1132.322,280.765,3.5096,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,1.42,1548.608,70.89,0.8861,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,2.702752,1627.247,62.075,0.7759,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,2.719296,1617.347,49.358,0.617,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,5.367328,1638.822,37.51,0.4689,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,1.348064,6.372,99.612,1.2451,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,148.894684,7.384,115.439,1.443,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,1.39056,12.355,96.615,1.2077,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,149.10672,14.748,115.331,1.4416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,1.394048,18.486,96.42,1.2053,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,149.073883,22.127,115.413,1.4427,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,1.390816,24.705,96.691,1.2086,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,149.110535,29.495,115.441,1.443,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.146688,468.474,471.791,5.8974,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.649472,666.584,167.826,2.0978,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,1.783456,616.506,47.036,0.5879,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,3.577632,614.659,42.205,0.5276,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,3.554816,618.604,28.317,0.354,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,7.444416,590.785,22.537,0.2817,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.135072,31.798,993.918,12.424,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,11.408544,48.188,1506.245,18.8281,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.441952,0.823,12.86,0.1607,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,307.884766,3.571,55.827,0.6978,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.421856,1.236,12.888,0.1611,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,307.282318,5.367,55.95,0.6994,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.415104,1.65,12.899,0.1612,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,306.473572,7.175,56.111,0.7014,20,True,,NVIDIA GB300,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.194656,353.03,355.53,4.4441,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,1.007424,1091.409,274.784,3.4348,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,0.753856,1458.517,111.276,1.3909,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,1.400096,1570.623,107.846,1.3481,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,1.41888,1549.83,70.946,0.8868,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,2.699616,1629.138,62.147,0.7768,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,1.348288,3.185,99.571,1.2446,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,74.42128,7.387,230.903,2.8863,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,1.38672,6.194,96.835,1.2104,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,74.65123,14.729,230.247,2.8781,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,1.392992,9.25,96.423,1.2053,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,74.656067,22.092,230.289,2.8786,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,1.387904,12.378,96.8,1.21,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +deepseek_v4,dsv4_pro-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,74.573441,29.488,230.6,2.8825,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d512[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png old mode 100644 new mode 100755 index 5b00e1cbf..8e05f9af2 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png old mode 100644 new mode 100755 index 7d88ce722..bc5e374d4 Binary files a/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/gb300/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260818_135743.csv b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260818_135743.csv deleted file mode 100644 index 09ade9aa4..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260818_135743.csv +++ /dev/null @@ -1,113 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.940544,93.479,92.001,1.9167,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,43.415329,101.302,24.925,0.5193,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.576,88.713,2.707,0.0564,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,100.397728,87.612,2.005,0.0418,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,96.960098,90.719,2.076,0.0433,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,198.092987,88.808,1.355,0.0282,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.440128,39.034,305.249,6.3594,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,44.833088,49.049,383.57,7.9911,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.892416,3.154,12.346,0.2572,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1522.686768,2.888,11.305,0.2355,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.89344,4.731,12.357,0.2574,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1523.757446,4.329,11.308,0.2356,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.920768,6.293,12.338,0.257,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1523.969482,5.772,11.317,0.2358,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.530304,89.812,89.077,1.8558,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,21.85984,100.596,24.943,0.5197,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.751553,88.844,4.067,0.0847,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.655998,88.57,3.379,0.0704,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.867329,90.0,2.747,0.0572,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,99.551903,88.357,2.022,0.0421,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.244736,35.099,548.686,11.431,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,23.144705,47.506,742.643,15.4717,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.408448,1.651,12.908,0.2689,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,740.466187,2.97,23.224,0.4838,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.401728,2.477,12.922,0.2692,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,739.674255,4.459,23.26,0.4846,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.397984,3.304,12.933,0.2694,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,737.853149,5.961,23.329,0.486,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.829536,82.841,83.427,1.7381,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.039584,99.597,25.076,0.5224,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,12.238688,89.839,6.854,0.1428,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,24.441055,89.973,6.178,0.1287,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.483232,89.818,4.112,0.0857,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.496769,88.855,3.39,0.0706,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.142016,30.243,945.32,19.6942,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.349184,44.518,1391.514,28.9899,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.30464,0.834,13.031,0.2715,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,364.844757,3.014,47.111,0.9815,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.325536,1.248,13.008,0.271,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,365.180939,4.516,47.079,0.9808,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.316672,1.665,13.022,0.2713,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,364.567505,6.032,47.17,0.9827,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.478752,71.769,74.468,1.5514,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,5.718144,96.142,24.939,0.5196,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,5.550624,99.044,13.602,0.2834,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,11.01232,99.844,12.95,0.2698,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,12.206048,90.079,6.873,0.1432,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,24.410879,90.084,6.186,0.1289,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.141632,15.162,947.767,19.7451,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,12.319584,22.312,1394.687,29.056,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.26704,0.418,13.076,0.2724,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,183.655777,2.993,93.567,1.9493,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.289248,0.626,13.049,0.2719,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,183.560486,4.492,93.627,1.9506,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.364448,0.829,12.956,0.2699,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,184.38327,5.963,93.22,1.9421,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,5.749856,95.612,93.736,1.9528,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,87.396355,100.646,24.668,0.5139,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,99.764702,88.168,2.018,0.042,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,200.149277,87.895,1.341,0.0279,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,193.401505,90.962,1.735,0.0361,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,395.77713,88.899,1.017,0.0212,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.769536,44.65,174.754,3.6407,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,87.832161,50.073,195.981,4.0829,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,24.629057,2.79,5.471,0.114,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,3184.618652,2.762,5.416,0.1128,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,24.614208,4.188,5.485,0.1143,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,3170.215088,4.162,5.451,0.1136,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,24.47056,5.617,5.528,0.1152,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,3170.186768,5.549,5.462,0.1138,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.936416,93.61,92.13,1.9194,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,43.463074,101.19,24.898,0.5187,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.597729,88.674,2.706,0.0564,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,100.353218,87.651,2.006,0.0418,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,96.970016,90.709,2.076,0.0433,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,198.196701,88.761,1.354,0.0282,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.438464,39.182,306.408,6.3835,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,45.13232,48.724,381.027,7.9381,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,10.871008,3.161,12.371,0.2577,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1522.545654,2.889,11.306,0.2355,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,10.923616,4.718,12.323,0.2567,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1516.224487,4.351,11.364,0.2367,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,10.948992,6.276,12.306,0.2564,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1521.641724,5.781,11.334,0.2361,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.53456,89.562,88.83,1.8506,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,21.858433,100.603,24.945,0.5197,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.756161,88.827,4.066,0.0847,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.666401,88.552,3.378,0.0704,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.627232,90.444,2.76,0.0575,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,98.992386,88.856,2.034,0.0424,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.243488,35.279,551.498,11.4896,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,23.134144,47.528,742.982,15.4788,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.412736,1.65,12.902,0.2688,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,739.211487,2.975,23.263,0.4847,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.422432,2.473,12.897,0.2687,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,737.283264,4.474,23.336,0.4862,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.389664,3.307,12.944,0.2697,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,736.510986,5.971,23.372,0.4869,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.826624,83.133,83.721,1.7442,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.041984,99.576,25.07,0.5223,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,12.2488,89.765,6.849,0.1427,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,24.456127,89.917,6.174,0.1286,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.501375,89.751,4.108,0.0856,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.49181,88.864,3.39,0.0706,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.142688,30.1,940.867,19.6014,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.360096,44.478,1390.286,28.9643,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.303616,0.834,13.033,0.2715,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,365.529114,3.008,47.023,0.9796,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.3048,1.25,13.034,0.2715,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,364.886108,4.52,47.117,0.9816,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.246848,1.677,13.111,0.2732,20,True,,NVIDIA H200,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,365.036407,6.024,47.109,0.9814,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260819_065009.csv b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260819_065009.csv new file mode 100644 index 000000000..1f54cc5fd --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_20260819_065009.csv @@ -0,0 +1,113 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.918208,94.194,92.705,1.9314,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,53.580734,82.083,20.196,0.4208,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,65.627678,67.015,2.045,0.0426,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,139.605087,63.007,1.442,0.03,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,146.335358,60.109,1.376,0.0287,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,337.950226,52.056,0.794,0.0165,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.438336,39.193,306.497,6.3854,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,61.946495,35.499,277.605,5.7834,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,11.226976,3.06,11.978,0.2495,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,2264.325439,1.942,7.602,0.1584,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.433184,4.145,10.827,0.2256,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,2306.325195,2.86,7.471,0.1556,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,13.70688,5.014,9.83,0.2048,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,2342.602539,3.755,7.362,0.1534,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.522336,90.282,89.543,1.8655,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,26.767649,82.152,20.37,0.4244,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,32.554527,67.549,3.092,0.0644,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,72.903458,60.327,2.301,0.0479,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,95.052254,46.27,1.412,0.0294,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,198.222046,44.375,1.016,0.0212,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.246784,34.808,544.133,11.3361,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,28.82048,38.15,596.39,12.4248,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.275424,1.672,13.075,0.2724,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,1174.363892,1.873,14.643,0.3051,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.278144,2.507,13.078,0.2725,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,1221.535767,2.7,14.085,0.2934,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.256544,3.35,13.112,0.2732,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,1230.955566,3.573,13.984,0.2913,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.822848,83.514,84.105,1.7522,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.591968,94.851,23.881,0.4975,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,14.022624,78.41,5.982,0.1246,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.193665,68.306,4.69,0.0977,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,33.446625,65.747,3.01,0.0627,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,85.891838,51.204,1.953,0.0407,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.14752,29.114,910.049,18.9594,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.275008,44.787,1399.923,29.1651,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.061856,0.854,13.346,0.278,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,581.921387,1.889,29.537,0.6154,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.07104,1.279,13.337,0.2779,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,587.96521,2.805,29.241,0.6092,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.075264,1.705,13.335,0.2778,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,618.383484,3.556,27.809,0.5794,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.466368,73.675,76.445,1.5926,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,5.860448,93.808,24.334,0.507,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,5.69968,96.454,13.246,0.276,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,11.488512,95.705,12.413,0.2586,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,12.784288,86.005,6.562,0.1367,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,32.403198,67.864,4.66,0.0971,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.146976,14.611,913.306,19.0272,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,12.241408,22.455,1403.594,29.2415,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,10.057664,0.427,13.348,0.2781,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,273.267212,2.012,62.884,1.3101,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,10.077504,0.639,13.323,0.2776,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,287.957367,2.864,59.683,1.2434,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,10.114208,0.849,13.277,0.2766,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,297.733704,3.693,57.73,1.2027,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,5.720448,96.104,94.218,1.9629,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,157.976578,55.68,13.647,0.2843,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,189.377762,46.447,1.063,0.0221,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,398.368225,44.161,0.674,0.014,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,415.10199,42.38,0.808,0.0168,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,859.012024,40.959,0.469,0.0098,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,0.7664,44.833,175.47,3.6556,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,212.774918,20.67,80.9,1.6854,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,30.117697,2.282,4.474,0.0932,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,4687.981934,1.876,3.679,0.0766,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,30.640127,3.364,4.406,0.0918,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,4704.096191,2.805,3.674,0.0765,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,31.734465,4.331,4.262,0.0888,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,4710.078125,3.735,3.676,0.0766,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,2.932864,93.723,92.242,1.9217,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,64.50528,68.181,16.776,0.3495,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,91.635681,47.995,1.465,0.0305,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,189.351257,46.454,1.063,0.0222,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,219.415619,40.089,0.918,0.0191,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,409.534271,42.957,0.655,0.0137,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.438336,39.193,306.497,6.3854,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,66.258461,33.189,259.539,5.4071,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,13.411904,2.562,10.027,0.2089,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,2312.934326,1.902,7.442,0.155,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,14.656896,3.516,9.184,0.1913,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,2338.385986,2.821,7.368,0.1535,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.743008,5.393,10.574,0.2203,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,2354.349609,3.736,7.326,0.1526,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.520512,90.39,89.651,1.8677,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,28.55616,77.007,19.094,0.3978,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,32.121056,68.46,3.134,0.0653,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,75.671715,58.12,2.217,0.0462,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,72.009628,61.076,1.864,0.0388,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,196.291611,44.811,1.026,0.0214,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.246304,34.875,545.193,11.3582,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,28.16416,39.039,610.288,12.7143,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,10.280512,1.671,13.068,0.2723,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,1174.248047,1.873,14.645,0.3051,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,10.273984,2.508,13.083,0.2726,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,1190.35498,2.771,14.454,0.3011,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,10.265152,3.347,13.101,0.2729,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,1190.325806,3.695,14.461,0.3013,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.823328,83.465,84.056,1.7512,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.612576,94.683,23.838,0.4966,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,12.429056,88.463,6.749,0.1406,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.959934,66.718,4.581,0.0954,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,33.451328,65.738,3.009,0.0627,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,72.365341,60.776,2.318,0.0483,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.147424,29.133,910.642,18.9717,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,12.38336,44.395,1387.674,28.9099,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,10.074336,0.853,13.329,0.2777,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,563.364197,1.952,30.51,0.6356,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,10.075424,1.279,13.331,0.2777,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,589.40979,2.798,29.169,0.6077,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,10.077184,1.705,13.332,0.2777,20,True,,NVIDIA H200,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,618.20282,3.557,27.817,0.5795,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png old mode 100644 new mode 100755 index 01c341581..dc45938e9 Binary files a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png old mode 100644 new mode 100755 index 832436803..aa2114abe Binary files a/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/h200/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260818_140843.csv b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260818_140843.csv deleted file mode 100644 index 28341933e..000000000 --- a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260818_140843.csv +++ /dev/null @@ -1,113 +0,0 @@ -config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.152512,87.193,85.815,4.7888,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,46.747265,94.081,23.149,1.2918,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.045025,89.674,2.737,0.1527,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,98.164192,89.606,2.051,0.1144,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,92.054047,95.554,2.187,0.122,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,184.666809,95.264,1.454,0.0811,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.625248,27.477,214.873,11.9907,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.170334,32.258,252.26,14.077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.38096,2.775,10.862,0.6061,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1061.547363,4.143,16.215,0.9049,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.396352,4.158,10.859,0.606,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1074.984497,6.137,16.028,0.8944,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.391456,5.546,10.874,0.6068,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1082.197998,8.128,15.937,0.8893,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.63136,84.248,83.559,4.6629,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.570816,93.294,23.133,1.2909,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.575071,89.482,4.096,0.2286,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.269215,89.266,3.405,0.19,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.844994,90.041,2.748,0.1533,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,97.96582,89.787,2.055,0.1147,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.329088,26.102,408.047,22.7705,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.53392,30.096,470.474,26.2541,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.796896,1.754,13.713,0.7653,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,533.176575,4.124,32.253,1.7998,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.808832,2.627,13.703,0.7647,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,539.776306,6.111,31.874,1.7787,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.810176,3.502,13.708,0.765,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,541.378662,8.124,31.796,1.7743,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.877536,78.31,78.864,4.4009,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,12.001248,91.616,23.066,1.2872,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,16.131008,68.161,5.2,0.2902,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.483711,67.696,4.648,0.2594,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.42416,90.035,4.121,0.23,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.004383,89.748,3.424,0.191,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.22112,19.424,607.139,33.8805,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.438911,29.815,931.946,52.0059,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.768,0.879,13.747,0.7671,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,267.556946,4.109,64.241,3.5849,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.745184,1.322,13.783,0.7691,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,269.226776,6.126,63.859,3.5635,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.757504,1.761,13.769,0.7683,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,270.881073,8.118,63.484,3.5426,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.50656,67.83,70.38,3.9274,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,6.2136,88.476,22.951,1.2807,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,7.35856,74.71,10.26,0.5725,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,14.691872,74.838,9.706,0.5417,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,16.161856,68.031,5.19,0.2896,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,32.410816,67.848,4.659,0.26,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.215904,9.946,621.731,34.6948,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,18.169312,15.129,945.659,52.7711,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,9.803136,0.438,13.695,0.7642,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,135.232315,4.065,127.071,7.091,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,9.802144,0.657,13.698,0.7644,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,135.997986,6.064,126.371,7.0519,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,9.79232,0.877,13.713,0.7652,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,136.518051,8.054,125.905,7.0259,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,6.157472,89.283,87.531,4.8845,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,93.065056,94.516,23.165,1.2927,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,92.355011,95.242,2.18,0.1216,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,184.985352,95.1,1.451,0.081,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,183.194748,96.03,1.832,0.1022,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,370.236786,95.032,1.088,0.0607,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,1.200512,28.621,112.019,6.251,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,135.988129,32.341,126.58,7.0636,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,21.219135,3.239,6.35,0.3544,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2143.854248,4.103,8.045,0.4489,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,21.241632,4.853,6.356,0.3547,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2280.630371,5.785,7.577,0.4228,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,21.251936,6.467,6.365,0.3552,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2295.632324,7.663,7.542,0.4209,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.14368,87.438,86.056,4.8022,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,46.745983,94.084,23.149,1.2918,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,49.011646,89.735,2.738,0.1528,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,98.141922,89.626,2.051,0.1145,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,92.045853,95.562,2.187,0.1221,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,185.612961,94.779,1.446,0.0807,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.624416,27.513,215.159,12.0066,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.137184,32.273,252.383,14.0839,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.38368,2.775,10.859,0.606,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1068.296875,4.117,16.113,0.8992,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.39184,4.159,10.863,0.6062,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1079.147461,6.113,15.966,0.891,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.387424,5.548,10.877,0.607,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1107.02063,7.946,15.58,0.8694,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.633952,84.114,83.426,4.6555,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.560863,93.334,23.143,1.2914,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.69376,89.052,4.076,0.2275,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.302177,89.206,3.403,0.1899,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.820606,90.086,2.749,0.1534,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,97.902367,89.846,2.056,0.1148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.329312,26.084,407.769,22.755,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.571522,30.065,469.99,26.2271,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.801888,1.753,13.706,0.7649,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,533.418335,4.123,32.239,1.799,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.79392,2.631,13.724,0.7659,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,538.994385,6.12,31.921,1.7813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.79984,3.506,13.723,0.7658,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,542.323608,8.11,31.74,1.7712,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.877184,78.341,78.896,4.4027,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.992672,91.682,23.083,1.2881,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,16.149696,68.082,5.194,0.2899,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.478432,67.707,4.649,0.2594,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.427649,90.022,4.121,0.23,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,48.89613,89.947,3.431,0.1915,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.2232,19.243,601.481,33.5648,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.414944,29.854,933.159,52.0736,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.75472,0.881,13.766,0.7682,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,267.116638,4.116,64.347,3.5908,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.745216,1.322,13.783,0.7691,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,268.723694,6.137,63.978,3.5702,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.747968,1.762,13.782,0.7691,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,270.309692,8.135,63.618,3.5501,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260819_064211.csv b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260819_064211.csv new file mode 100644 index 000000000..301bdd0bc --- /dev/null +++ b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_20260819_064211.csv @@ -0,0 +1,113 @@ +config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.1392,87.563,86.179,4.8091,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,46.735554,94.105,23.154,1.2921,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,48.999104,89.758,2.739,0.1529,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,98.109535,89.656,2.052,0.1145,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,92.079903,95.527,2.186,0.122,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,184.73082,95.231,1.453,0.0811,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.625376,27.471,214.829,11.9882,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.203743,32.242,252.136,14.0701,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.394048,2.772,10.85,0.6055,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1046.032593,4.205,16.456,0.9183,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.37648,4.164,10.876,0.6069,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1047.977905,6.295,16.441,0.9175,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.367616,5.556,10.895,0.608,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1052.745483,8.355,16.383,0.9142,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.633216,84.152,83.464,4.6576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.582081,93.25,23.122,1.2903,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.641953,89.239,4.085,0.228,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.428513,88.978,3.394,0.1894,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.81485,90.096,2.75,0.1534,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,97.840446,89.902,2.058,0.1148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.330304,26.006,406.544,22.6866,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.537502,30.093,470.428,26.2516,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.797792,1.753,13.712,0.7652,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,525.897644,4.181,32.7,1.8248,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.81216,2.626,13.699,0.7644,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,526.581543,6.264,32.673,1.8233,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.810432,3.502,13.708,0.7649,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,529.113403,8.312,32.533,1.8154,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.876064,78.441,78.997,4.4083,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,11.990592,91.698,23.087,1.2883,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,16.140736,68.12,5.197,0.29,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.490974,67.681,4.647,0.2593,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.505953,89.734,4.108,0.2292,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.018303,89.723,3.423,0.191,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.220352,19.491,609.255,33.9986,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.398144,29.881,934.011,52.1211,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.753184,0.881,13.768,0.7683,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,265.517212,4.141,64.735,3.6124,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.768832,1.319,13.749,0.7673,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,265.533966,6.211,64.747,3.6131,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.77456,1.758,13.745,0.767,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,265.816681,8.273,64.694,3.6101,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,512,512,mla_absorbed,,64,0.504864,68.057,70.616,3.9406,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,512,512,mla_absorbed,,64,6.211744,88.503,22.958,1.2811,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,512,512,mla_absorbed,,64,7.352384,74.772,10.268,0.573,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,512,512,mla_absorbed,,64,14.705536,74.769,9.697,0.5412,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,512,512,mla_absorbed,,64,16.155392,68.058,5.192,0.2898,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,512,512,mla_absorbed,,64,32.403648,67.863,4.66,0.26,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,512,512,mla_absorbed,,64,0.217152,9.889,618.157,34.4954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,512,512,mla_absorbed,,64,18.191681,15.11,944.496,52.7062,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,512,512,mla_absorbed,,64,9.819616,0.437,13.672,0.7629,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,512,512,mla_absorbed,,64,134.237061,4.095,128.013,7.1436,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,512,512,mla_absorbed,,64,9.81008,0.657,13.687,0.7638,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,512,512,mla_absorbed,,64,134.34758,6.138,127.923,7.1386,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,512,512,mla_absorbed,,64,9.788672,0.878,13.718,0.7655,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_flash-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,512,512,mla_absorbed,,64,134.475967,8.176,127.817,7.1326,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,2048,2048,128,1,512,512,mla_absorbed,,64,6.158368,89.27,87.518,4.8838,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,8192,8192,128,1,512,512,mla_absorbed,,64,93.064125,94.516,23.165,1.2927,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,65536,128,1,512,512,mla_absorbed,,64,92.440483,95.154,2.178,0.1215,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,512,131072,128,1,512,512,mla_absorbed,,64,185.099579,95.042,1.45,0.0809,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,65536,128,1,512,512,mla_absorbed,,64,182.972092,96.147,1.834,0.1023,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,context,cudnn,bfloat16,bfloat16,1,1024,131072,128,1,512,512,mla_absorbed,,64,367.306885,95.79,1.096,0.0612,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,1,131072,128,1,512,512,mla_absorbed,,64,1.200192,28.629,112.049,6.2527,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,1,131072,128,1,512,512,mla_absorbed,,64,136.087875,32.318,126.488,7.0585,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,2,131072,128,1,512,512,mla_absorbed,,64,21.225697,3.238,6.348,0.3542,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,2,131072,128,1,512,512,mla_absorbed,,64,2086.57959,4.216,8.266,0.4613,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,3,131072,128,1,512,512,mla_absorbed,,64,21.231903,4.855,6.359,0.3548,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,3,131072,128,1,512,512,mla_absorbed,,64,2102.263184,6.276,8.22,0.4587,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,1,4,131072,128,1,512,512,mla_absorbed,,64,21.28096,6.458,6.356,0.3547,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro,generation,cudnn,bfloat16,bfloat16,128,4,131072,128,1,512,512,mla_absorbed,,64,2115.068115,8.318,8.186,0.4568,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,1,512,512,mla_absorbed,,64,3.142272,87.477,86.095,4.8044,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,1,512,512,mla_absorbed,,64,46.704063,94.168,23.17,1.293,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,64,1,512,512,mla_absorbed,,64,48.986271,89.781,2.74,0.1529,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,64,1,512,512,mla_absorbed,,64,98.074272,89.688,2.053,0.1146,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,1,512,512,mla_absorbed,,64,91.93248,95.68,2.19,0.1222,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,1,512,512,mla_absorbed,,64,184.395935,95.404,1.456,0.0812,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,1,512,512,mla_absorbed,,64,0.62416,27.525,215.247,12.0116,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,1,512,512,mla_absorbed,,64,68.21373,32.237,252.099,14.0681,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,1,512,512,mla_absorbed,,64,12.400288,2.771,10.845,0.6052,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,1,512,512,mla_absorbed,,64,1046.009277,4.205,16.456,0.9183,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,1,512,512,mla_absorbed,,64,12.371392,4.166,10.881,0.6072,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,1,512,512,mla_absorbed,,64,1049.725952,6.285,16.414,0.916,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,1,512,512,mla_absorbed,,64,12.366624,5.557,10.896,0.608,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,1,512,512,mla_absorbed,,64,1054.48291,8.342,16.356,0.9127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,1,512,512,mla_absorbed,,64,1.633408,84.142,83.454,4.657,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,1,512,512,mla_absorbed,,64,23.572449,93.288,23.131,1.2908,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,32,1,512,512,mla_absorbed,,64,24.626017,89.297,4.088,0.2281,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,32,1,512,512,mla_absorbed,,64,49.416256,89.0,3.395,0.1895,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,1,512,512,mla_absorbed,,64,48.834335,90.061,2.748,0.1534,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,1,512,512,mla_absorbed,,64,97.870621,89.875,2.057,0.1148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,1,512,512,mla_absorbed,,64,0.329376,26.079,407.69,22.7506,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,1,512,512,mla_absorbed,,64,36.613857,30.03,469.447,26.1968,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,1,512,512,mla_absorbed,,64,9.82576,1.748,13.673,0.763,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,1,512,512,mla_absorbed,,64,525.860107,4.182,32.702,1.8249,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,1,512,512,mla_absorbed,,64,9.808352,2.627,13.704,0.7647,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,1,512,512,mla_absorbed,,64,527.185852,6.257,32.636,1.8212,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,1,512,512,mla_absorbed,,64,9.81808,3.5,13.697,0.7644,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,1,512,512,mla_absorbed,,64,527.935059,8.331,32.605,1.8195,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,512,512,mla_absorbed,,64,0.875584,78.484,79.04,4.4107,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,512,512,mla_absorbed,,64,12.001056,91.618,23.067,1.2872,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,512,512,mla_absorbed,,64,16.161505,68.033,5.19,0.2896,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,512,512,mla_absorbed,,64,32.473728,67.717,4.65,0.2595,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,512,512,mla_absorbed,,64,24.482656,89.82,4.112,0.2294,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,512,512,mla_absorbed,,64,49.045696,89.672,3.421,0.1909,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,512,512,mla_absorbed,,64,0.220832,19.449,607.93,33.9247,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,512,512,mla_absorbed,,64,18.434143,29.823,932.187,52.0193,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,512,512,mla_absorbed,,64,9.76768,0.879,13.748,0.7672,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,512,512,mla_absorbed,,64,265.514801,4.141,64.736,3.6125,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,512,512,mla_absorbed,,64,9.747168,1.322,13.78,0.769,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,512,512,mla_absorbed,,64,265.525421,6.211,64.749,3.6132,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,512,512,mla_absorbed,,64,9.7616,1.76,13.763,0.768,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +deepseek_v4,dsv4_pro-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,512,512,mla_absorbed,,64,265.796051,8.273,64.699,3.6104,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png old mode 100644 new mode 100755 index dac268bf1..69560440e Binary files a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png and b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_context.png differ diff --git a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png old mode 100644 new mode 100755 index 9ad59afe4..f8610cfbe Binary files a/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png and b/benchmark/attention_inference/results/deepseek_v4/rtxpro6000/deepseek_v4_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260818_125939.csv b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260819_053417.csv similarity index 64% rename from benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260818_125939.csv rename to benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260819_053417.csv index c033e94ab..561d5ff0e 100644 --- a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260818_125939.csv +++ b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_20260819_053417.csv @@ -1,209 +1,209 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.062464,550.073,604.328,7.5541,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.139648,3936.725,1081.254,13.5157,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.442176,19892.741,1365.926,17.0741,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.043552,12622.975,222.707,2.7838,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.042656,25776.247,227.385,2.8423,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.051424,21381.293,372.132,4.6516,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.049792,44164.189,384.329,4.8041,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.040064,107.203,7.463,0.0933,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.295264,1861.913,129.623,1.6203,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.064864,529.72,581.967,7.2746,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.136832,4017.743,1103.506,13.7938,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.435008,20220.532,1388.434,17.3554,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.045728,12022.301,212.109,2.6514,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.0432,25451.657,224.521,2.8065,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.049856,22053.748,383.836,4.7979,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.048672,45180.456,393.173,4.9147,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.03712,115.705,8.055,0.1007,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.295328,1861.509,129.595,1.6199,20,True,,NVIDIA GB200,cudnn 9.26.0, gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.038496,167.354,8.246,0.1031,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.29616,2784.42,137.197,1.715,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.037248,230.615,9.017,0.1127,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.296448,3708.953,145.022,1.8128,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.23952,143.452,157.602,1.97,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.514496,1068.533,293.481,3.6685,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.83696,4788.397,328.793,4.1099,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.267264,2056.977,36.291,0.4536,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.41856,2626.891,23.173,0.2897,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.288512,3810.974,66.328,0.8291,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.444352,4948.832,43.066,0.5383,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.407936,10.529,0.733,0.0092,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,41.985344,13.094,0.912,0.0114,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.409152,15.746,0.776,0.0097,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,42.022335,19.624,0.967,0.0121,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.4096,20.972,0.82,0.0103,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,41.964993,26.201,1.024,0.0128,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.050176,342.392,376.163,4.702,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.08784,3129.302,859.489,10.7436,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.240896,18257.034,1253.611,15.6701,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.045664,6019.576,106.203,1.3275,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.046752,11758.98,103.732,1.2966,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.044608,12324.153,214.496,2.6812,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.047648,23075.713,200.811,2.5101,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.039328,54.604,3.801,0.0475,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.162528,1691.265,117.743,1.4718,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.037888,85.02,4.189,0.0524,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.163104,2527.938,124.56,1.557,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.038176,112.504,4.399,0.055,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.163296,3366.621,131.637,1.6455,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.181152,94.837,104.191,1.3024,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.319168,861.233,236.545,2.9568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.96272,4568.355,313.684,3.9211,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.185376,1482.813,26.161,0.327,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.263776,2084.177,18.386,0.2298,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.18736,2934.222,51.069,0.6384,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.267776,4106.087,35.732,0.4467,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.2512,8.549,0.595,0.0074,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,21.58256,12.736,0.887,0.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.252992,12.733,0.627,0.0078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,21.291872,19.365,0.954,0.0119,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.25392,16.915,0.661,0.0083,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,21.286272,25.827,1.01,0.0126,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.04352,197.379,216.847,2.7106,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.295936,2786.527,137.301,1.7163,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.03856,222.768,8.71,0.1089,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.295904,3715.771,145.289,1.8161,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.229408,149.776,164.548,2.0569,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.513888,1069.797,293.829,3.6729,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.83792,4785.896,328.621,4.1078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.26544,2071.112,36.541,0.4568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.413152,2661.276,23.476,0.2935,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.293408,3747.381,65.222,0.8153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.444032,4952.398,43.097,0.5387,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.404544,10.617,0.739,0.0092,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,41.955872,13.103,0.912,0.0114,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.406848,15.835,0.78,0.0098,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,41.9664,19.65,0.968,0.0121,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.400864,21.429,0.838,0.0105,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,41.969566,26.198,1.024,0.0128,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.049184,349.298,383.75,4.7969,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.08448,3253.763,893.673,11.1709,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.235744,18656.027,1281.008,16.0126,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.042752,6429.592,113.437,1.418,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.04384,12540.051,110.622,1.3828,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.04432,12404.238,215.89,2.6986,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.045184,24334.092,211.762,2.647,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.03888,55.234,3.845,0.0481,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.162432,1692.264,117.812,1.4727,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.039488,81.575,4.019,0.0502,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.164,2514.127,123.879,1.5485,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.039616,108.415,4.239,0.053,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.162528,3382.53,132.259,1.6532,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.182336,94.221,103.514,1.2939,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.310624,884.922,243.051,3.0381,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.961792,4572.763,313.987,3.9248,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.1816,1513.645,26.705,0.3338,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.259456,2118.879,18.692,0.2336,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.19184,2865.7,49.876,0.6235,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.266112,4131.763,35.956,0.4494,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.255584,8.402,0.585,0.0073,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,21.56768,12.745,0.887,0.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.254464,12.659,0.624,0.0078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,21.265152,19.389,0.955,0.0119,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.251392,17.085,0.668,0.0084,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,21.271648,25.845,1.011,0.0126,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.042848,200.475,220.248,2.7531,20,True,,NVIDIA GB200,cudnn 9.26.0, gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.062816,2187.961,600.941,7.5118,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.139584,15754.121,1081.75,13.5219,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.047776,2876.736,50.754,0.6344,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.043168,6367.631,56.172,0.7021,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.0448,6135.668,106.789,1.3349,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.044,12494.45,108.73,1.3591,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.039072,27.481,1.913,0.0239,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.0984,1396.737,97.238,1.2155,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.039008,41.289,2.034,0.0254,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.098656,2089.669,102.965,1.2871,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.042496,50.534,1.976,0.0247,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.097792,2810.842,109.906,1.3738,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.167296,51.346,56.41,0.7051,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.23136,594.048,163.16,2.0395,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.515168,4268.556,293.098,3.6637,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.158624,866.445,15.287,0.1911,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.1864,1474.667,13.009,0.1626,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.162944,1686.947,29.361,0.367,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.188416,2917.777,25.391,0.3174,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.177856,6.037,0.42,0.0053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,10.816224,12.707,0.885,0.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.175616,9.171,0.452,0.0056,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,10.815456,19.061,0.939,0.0117,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.186752,11.499,0.45,0.0056,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,10.8144,25.418,0.994,0.0124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.03984,107.805,118.439,1.4805,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.050048,1373.071,377.125,4.7141,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.085664,12835.166,881.321,11.0165,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.041696,1648.107,29.078,0.3635,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.045888,2995.096,26.421,0.3303,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.044416,3094.357,53.856,0.6732,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.04272,6434.408,55.994,0.6999,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.039904,13.454,0.937,0.0117,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.065696,1046.022,72.822,0.9103,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.040096,20.084,0.99,0.0124,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.064928,1587.593,78.226,0.9778,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.040288,26.652,1.042,0.013,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.065088,2111.587,82.564,1.0321,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.141568,30.339,33.331,0.4166,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.160672,427.7,117.471,1.4684,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.290304,3787.449,260.063,3.2508,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.13792,498.256,8.791,0.1099,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.132832,1034.683,9.127,0.1141,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.13312,1032.444,17.969,0.2246,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.134048,2050.593,17.845,0.2231,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.1344,3.995,0.278,0.0035,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.258976,265.351,18.473,0.2309,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.137024,5.877,0.29,0.0036,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.26192,393.552,19.392,0.2424,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.13584,7.904,0.309,0.0039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.257152,534.466,20.898,0.2612,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.08192,419.43,460.8,5.76,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.59104,930.15,255.473,3.1934,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,8.57888,1025.319,70.403,0.88,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.658208,835.231,216.658,2.7082,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.27728,860.823,216.729,2.7091,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.271904,864.461,118.716,1.4839,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.495968,881.03,114.269,1.4284,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.466272,1.741,108.856,1.3607,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,101.962494,5.392,337.025,4.2128,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.46528,2.613,108.906,1.3613,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,101.985474,8.086,336.97,4.2121,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.464416,3.486,108.951,1.3619,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,101.970078,10.783,337.041,4.213,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.210016,163.605,179.742,2.2468,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.972512,565.295,155.263,1.9408,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,11.731904,749.758,51.482,0.6435,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.009952,544.339,141.201,1.765,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.930976,569.407,143.36,1.792,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.781824,617.071,84.742,1.0593,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,3.469504,633.815,82.206,1.0276,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.927392,2.228,139.291,1.7411,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,206.710846,2.66,166.242,2.078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.921888,3.352,139.698,1.7462,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,206.777374,3.988,166.198,2.0775,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.921504,4.47,139.735,1.7467,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,206.741058,5.318,166.238,2.078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.0592,290.2,318.824,3.9853,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.315712,870.66,239.134,2.9892,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,4.320448,1017.961,69.898,0.8737,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.6552,419.533,108.827,1.3603,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.267072,433.879,109.238,1.3655,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.653728,840.955,115.488,1.4436,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.269088,866.379,112.369,1.4046,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.469504,0.87,54.357,0.6795,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,52.818783,5.204,325.3,4.0663,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.475552,1.301,54.227,0.6778,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,52.835648,7.804,325.216,4.0652,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.469344,1.739,54.367,0.6796,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,52.817986,10.408,325.345,4.0668,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.186304,92.214,101.31,1.2664,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.546592,502.894,138.124,1.7265,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,5.79552,758.87,52.107,0.6513,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.930624,295.369,76.619,0.9577,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.77008,310.582,78.195,0.9774,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.931104,590.434,81.084,1.0135,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.772704,620.245,80.446,1.0056,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.7728,1.211,75.719,0.9465,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,103.855743,2.647,165.441,2.068,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.769568,1.82,75.862,0.9483,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,103.597603,3.98,165.863,2.0733,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.770016,2.427,75.847,0.9481,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,103.553535,5.309,165.944,2.0743,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.054752,156.888,172.362,2.1545,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.179904,763.957,209.827,2.6228,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.103136,1045.592,71.795,0.8974,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.65424,210.074,54.493,0.6812,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.27296,215.936,54.366,0.6796,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.651424,421.965,57.948,0.7244,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.27168,432.307,56.07,0.7009,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.303712,0.466,29.134,0.3642,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,26.43552,5.199,324.979,4.0622,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.304096,0.699,29.131,0.3641,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,26.43792,7.798,324.969,4.0621,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.306752,0.931,29.099,0.3637,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,26.434015,10.399,325.037,4.063,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.168224,51.062,56.099,0.7012,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.344448,399.012,109.592,1.3699,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.935744,749.051,51.433,0.6429,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.90384,152.061,39.445,0.4931,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.695456,162.126,40.819,0.5102,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.906688,303.167,41.634,0.5204,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.69376,324.577,42.098,0.5262,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.693696,0.634,39.628,0.4953,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,51.938305,2.646,165.407,2.0676,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.69728,0.949,39.546,0.4943,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,51.927746,3.97,165.451,2.0681,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.69552,1.267,39.59,0.4949,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,51.941666,5.292,165.417,2.0677,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.057408,74.815,82.194,1.0274,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.115104,597.021,163.977,2.0497,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.076896,1021.001,70.107,0.8763,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.652832,105.264,27.305,0.3413,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.26032,109.051,27.456,0.3432,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.652416,210.662,28.93,0.3616,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.267136,216.928,28.136,0.3517,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.303872,0.233,14.566,0.1821,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,15.111904,4.547,284.246,3.5531,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.30272,0.35,14.574,0.1822,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,15.11232,6.821,284.255,3.5532,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.303072,0.466,14.573,0.1822,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,15.112,9.095,284.278,3.5535,20,True,,NVIDIA GB200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.135712,31.648,34.769,0.4346,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.259904,264.403,72.621,0.9078,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.498912,733.54,50.368,0.6296,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.875232,78.516,20.367,0.2546,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.633664,84.129,21.181,0.2648,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.876864,156.739,21.525,0.2691,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.639456,167.664,21.746,0.2718,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.633728,0.329,20.541,0.2568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,21.57424,3.185,199.103,2.4888,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.633664,0.493,20.543,0.2568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,21.559649,4.781,199.25,2.4906,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.633792,0.657,20.543,0.2568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,21.560064,6.375,199.258,2.4907,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.136128,16154.086,1109.213,13.8652,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.045952,2990.924,52.769,0.6596,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.043488,6320.776,55.759,0.697,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.035616,7717.821,134.325,1.6791,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.042304,12995.362,113.089,1.4136,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.040032,26.822,1.867,0.0233,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.098912,1389.507,96.735,1.2092,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.037824,42.582,2.098,0.0262,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.095936,2148.916,105.884,1.3235,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.03968,54.12,2.116,0.0265,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.097952,2806.251,109.726,1.3716,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.161472,53.198,58.445,0.7306,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.228768,600.779,165.009,2.0626,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.512032,4294.699,294.894,3.6862,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.163776,839.189,14.806,0.1851,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.188288,1459.88,12.878,0.161,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.152192,1806.126,31.435,0.3929,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.185632,2961.536,25.772,0.3222,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.178624,6.011,0.418,0.0052,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,10.803072,12.722,0.886,0.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.178976,8.999,0.443,0.0055,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,10.83376,19.029,0.938,0.0117,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.183168,11.724,0.458,0.0057,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,10.813824,25.419,0.994,0.0124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.04208,102.067,112.134,1.4017,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.049376,1391.759,382.258,4.7782,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.0856,12844.761,881.98,11.0247,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.044384,1548.294,27.317,0.3415,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.04512,3046.076,26.871,0.3359,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.0424,3241.485,56.417,0.7052,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.041632,6602.563,57.457,0.7182,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.03936,13.64,0.95,0.0119,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.06528,1052.688,73.286,0.9161,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.03696,21.789,1.074,0.0134,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.065856,1565.221,77.123,0.964,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.038368,27.985,1.094,0.0137,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.064224,2139.994,83.675,1.0459,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.134144,32.018,35.176,0.4397,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.15968,430.357,118.201,1.4775,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.287584,3823.271,262.523,3.2815,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.12736,539.569,9.52,0.119,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.134784,1019.698,8.995,0.1124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.13696,1003.497,17.465,0.2183,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.133824,2054.025,17.875,0.2234,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.133824,4.012,0.279,0.0035,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.257888,266.47,18.551,0.2319,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.137568,5.854,0.288,0.0036,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.255616,403.258,19.87,0.2484,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.135552,7.921,0.31,0.0039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.255552,537.812,21.029,0.2629,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.07984,430.357,472.805,5.9101,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.583712,941.827,258.681,3.2335,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,8.44144,1042.013,71.549,0.8944,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.66096,831.754,215.756,2.697,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.284096,856.253,215.579,2.6947,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.275872,861.773,118.346,1.4793,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.517984,873.327,113.27,1.4159,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.458336,1.747,109.207,1.3651,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,102.393341,5.369,335.607,4.1951,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.458464,2.621,109.208,1.3651,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,102.400452,8.053,335.604,4.1951,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.460032,3.492,109.145,1.3643,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,102.389763,10.738,335.66,4.1957,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.221216,155.322,170.642,2.133,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.971712,565.76,155.391,1.9424,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,11.673056,753.538,51.741,0.6468,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.008768,544.977,141.367,1.7671,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.929504,569.842,143.469,1.7934,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.788032,614.928,84.448,1.0556,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,3.471456,633.458,82.159,1.027,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.92384,2.232,139.548,1.7444,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,206.474396,2.663,166.432,2.0804,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.925568,3.346,139.431,1.7429,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,206.557053,3.992,166.375,2.0797,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.926368,4.459,139.382,1.7423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,206.563293,5.323,166.381,2.0798,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.051552,333.253,366.123,4.5765,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.31232,880.116,241.731,3.0216,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,4.202112,1046.628,71.866,0.8983,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.65376,420.457,109.066,1.3633,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.283648,428.276,107.827,1.3478,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.660832,831.915,114.246,1.4281,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.270304,865.55,112.262,1.4033,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.467136,0.87,54.409,0.6801,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,52.835903,5.202,325.195,4.0649,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.466432,1.306,54.428,0.6803,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,52.851231,7.801,325.12,4.064,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.46368,1.743,54.492,0.6811,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,52.828735,10.406,325.279,4.066,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.191264,89.823,98.682,1.2335,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.542144,507.02,139.257,1.7407,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,5.866624,749.672,51.476,0.6434,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.932224,294.863,76.487,0.9561,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.769056,310.762,78.241,0.978,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.933376,588.997,80.886,1.0111,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.77488,619.485,80.347,1.0043,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.767616,1.215,75.941,0.9493,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,103.877502,2.646,165.406,2.0676,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.764,1.826,76.101,0.9513,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,103.53331,3.982,165.966,2.0746,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.765536,2.433,76.04,0.9505,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,103.52784,5.31,165.985,2.0748,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.056832,151.146,166.054,2.0757,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.178752,768.881,211.179,2.6397,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.078816,1057.825,72.635,0.9079,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.659232,208.483,54.08,0.676,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.281344,214.523,54.01,0.6751,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.656192,418.899,57.527,0.7191,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.281952,428.843,55.621,0.6953,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.30592,0.466,29.106,0.3638,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,26.53376,5.18,323.776,4.0472,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.306976,0.698,29.095,0.3637,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,26.537249,7.769,323.753,4.0469,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.306368,0.931,29.104,0.3638,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,26.539841,10.357,323.741,4.0468,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.164832,52.113,57.253,0.7157,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.350368,392.27,107.74,1.3468,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.924672,751.887,51.628,0.6453,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.903072,152.19,39.478,0.4935,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.698752,161.812,40.739,0.5092,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.91184,301.454,41.398,0.5175,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.691424,325.025,42.156,0.5269,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.69744,0.633,39.54,0.4943,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,51.943359,2.646,165.391,2.0674,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.694784,0.95,39.605,0.4951,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,51.913376,3.971,165.497,2.0687,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.69808,1.265,39.53,0.4941,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,51.957535,5.29,165.366,2.0671,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.053984,79.56,87.407,1.0926,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.111776,614.796,168.859,2.1107,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.06608,1031.359,70.818,0.8852,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.657056,104.587,27.13,0.3391,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.272128,108.039,27.201,0.34,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.659232,208.483,28.631,0.3579,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.27664,215.314,27.926,0.3491,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.30368,0.233,14.567,0.1821,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,15.1152,4.546,284.184,3.5523,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.30416,0.35,14.565,0.1821,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,15.116544,6.819,284.176,3.5522,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.305632,0.466,14.557,0.182,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,15.116544,9.092,284.193,3.5524,20,True,,NVIDIA GB200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.139904,30.699,33.727,0.4216,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.264992,259.327,71.226,0.8903,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.503264,731.416,50.222,0.6278,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.874592,78.573,20.382,0.2548,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.639456,83.832,21.106,0.2638,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.876352,156.831,21.537,0.2692,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.634208,168.203,21.816,0.2727,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.635232,0.328,20.522,0.2565,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,21.558529,3.188,199.248,2.4906,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.640576,0.491,20.457,0.2557,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,21.547264,4.784,199.364,2.4921,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.636192,0.656,20.513,0.2564,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,21.537088,6.382,199.471,2.4934,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png old mode 100644 new mode 100755 index 759255621..87c574491 Binary files a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png old mode 100644 new mode 100755 index 1a29f2bc8..3dc6e3f61 Binary files a/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/gb200/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260818_130614.csv b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260819_053320.csv similarity index 63% rename from benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260818_130614.csv rename to benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260819_053320.csv index 7a8004e7a..708a7b0c4 100644 --- a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260818_130614.csv +++ b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_20260819_053320.csv @@ -1,209 +1,209 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.059744,575.116,631.841,7.898,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.12848,4278.921,1175.241,14.6905,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.4096,21474.837,1474.56,18.432,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.04288,12820.798,226.197,2.8275,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.045376,24231.128,213.755,2.6719,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.047552,23122.3,402.433,5.0304,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.047392,46400.728,403.792,5.0474,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.033568,127.948,8.908,0.1113,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.293056,1875.941,130.6,1.6325,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.038784,166.111,8.185,0.1023,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.294368,2801.37,138.032,1.7254,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.039136,219.489,8.582,0.1073,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.293088,3751.473,146.685,1.8336,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.226976,151.38,166.312,2.0789,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.5128,1072.067,294.452,3.6806,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.836608,4789.314,328.856,4.1107,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.263296,2087.976,36.838,0.4605,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.417952,2630.713,23.207,0.2901,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.290688,3782.446,65.832,0.8229,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.444736,4944.559,43.029,0.5379,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.407424,10.542,0.734,0.0092,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,41.420544,13.273,0.924,0.0116,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.398528,16.166,0.797,0.01,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,41.404896,19.916,0.981,0.0123,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.401664,21.386,0.836,0.0105,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,41.411903,26.551,1.038,0.013,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.047616,360.8,396.387,4.9548,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.0832,3303.821,907.422,11.3428,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.225312,19519.806,1340.319,16.754,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.043296,6348.806,112.012,1.4001,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.043552,12622.975,111.353,1.3919,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.045504,12081.483,210.273,2.6284,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.042592,25814.98,224.649,2.8081,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.038624,55.6,3.871,0.0484,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.15872,1731.842,120.568,1.5071,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.059456,577.902,634.902,7.9363,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.12288,4473.924,1228.8,15.36,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.384352,22885.514,1571.424,19.6428,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.04432,12404.238,218.848,2.7356,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.043328,25376.469,223.858,2.7982,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.047296,23247.456,404.612,5.0576,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.048448,45389.35,394.991,4.9374,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.038816,110.649,7.703,0.0963,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.293536,1872.874,130.386,1.6298,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.040448,159.277,7.848,0.0981,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.294912,2796.203,137.778,1.7222,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.038848,221.117,8.646,0.1081,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.295136,3725.441,145.667,1.8208,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.225856,152.131,167.136,2.0892,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.512736,1072.2,294.489,3.6811,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.831008,4803.962,329.862,4.1233,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.264128,2081.399,36.722,0.459,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.411616,2671.207,23.564,0.2946,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.289568,3797.076,66.086,0.8261,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.43312,5077.168,44.183,0.5523,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.407264,10.546,0.734,0.0092,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,41.397377,13.28,0.925,0.0116,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.400928,16.069,0.792,0.0099,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,41.456352,19.892,0.98,0.0123,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.405824,21.167,0.828,0.0103,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,41.412735,26.55,1.038,0.013,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.046656,368.224,404.543,5.0568,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.078592,3497.53,960.625,12.0078,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.208544,21089.298,1448.087,18.1011,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.046464,5915.933,104.375,1.3047,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.04224,13015.052,114.812,1.4352,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.043232,12716.409,221.323,2.7665,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.042048,26148.964,227.556,2.8444,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.036768,58.406,4.066,0.0508,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.16032,1714.558,119.364,1.4921,20,True,,NVIDIA GB300,cudnn 9.26.0, gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.0376,85.671,4.221,0.0528,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.160864,2563.139,126.294,1.5787,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.035232,121.905,4.767,0.0596,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.158272,3473.488,135.816,1.6977,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.191552,89.688,98.534,1.2317,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.30784,892.925,245.249,3.0656,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.959456,4583.896,314.751,3.9344,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.18704,1469.621,25.928,0.3241,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.263424,2086.962,18.41,0.2301,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.183872,2989.883,52.038,0.6505,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.273344,4022.446,35.004,0.4376,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.255424,8.408,0.585,0.0073,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,21.23296,12.946,0.901,0.0113,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.252224,12.771,0.629,0.0079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,20.967903,19.664,0.969,0.0121,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.24592,17.465,0.683,0.0085,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,20.955999,26.234,1.026,0.0128,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.047424,181.131,198.996,2.4874,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.060128,2285.773,627.806,7.8476,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.127808,17205.677,1181.42,14.7678,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.035072,3918.766,69.139,0.8642,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.04304,6386.568,56.339,0.7042,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.045152,6087.834,105.956,1.3245,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.043488,12641.552,110.01,1.3751,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.038272,28.056,1.953,0.0244,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.096352,1426.425,99.305,1.2413,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.037152,43.352,2.136,0.0267,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.097728,2109.512,103.942,1.2993,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.037408,57.407,2.245,0.0281,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.100384,2738.264,107.068,1.3383,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.16032,53.58,58.865,0.7358,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.233344,588.997,161.773,2.0222,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.511776,4296.847,295.041,3.688,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.165536,830.266,14.648,0.1831,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.18928,1452.229,12.811,0.1601,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.162816,1688.273,29.384,0.3673,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.191488,2870.967,24.984,0.3123,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.183616,5.848,0.407,0.0051,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,10.665952,12.886,0.897,0.0112,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.178848,9.005,0.444,0.0055,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,10.66912,19.323,0.952,0.0119,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.18368,11.691,0.457,0.0057,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,10.65376,25.801,1.009,0.0126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.043264,99.273,109.065,1.3633,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.047424,1449.044,397.992,4.9749,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.083328,13194.984,906.028,11.3253,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.043648,1574.401,27.777,0.3472,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.033696,4078.791,35.981,0.4498,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.041888,3281.106,57.106,0.7138,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.0416,6607.642,57.502,0.7188,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.038432,13.969,0.973,0.0122,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.159296,2588.369,127.537,1.5942,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.039072,109.924,4.298,0.0537,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.158816,3461.59,135.35,1.6919,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.17728,96.908,106.466,1.3308,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.322048,853.531,234.429,2.9304,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.955488,4602.932,316.058,3.9507,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.172352,1594.863,28.138,0.3517,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.264736,2076.619,18.319,0.229,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.190336,2888.344,50.27,0.6284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.269728,4076.372,35.474,0.4434,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.24864,8.637,0.601,0.0075,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,21.237663,12.943,0.901,0.0113,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.258112,12.48,0.615,0.0077,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,20.980097,19.653,0.968,0.0121,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.248512,17.283,0.676,0.0084,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,20.98192,26.201,1.024,0.0128,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.040832,210.373,231.122,2.889,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.05728,2399.423,659.021,8.2378,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.123296,17835.317,1224.654,15.3082,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.045152,3043.917,53.704,0.6713,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.0344,7990.637,70.489,0.8811,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.03472,7916.99,137.792,1.7224,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.033728,16299.686,141.844,1.7731,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.03904,27.504,1.915,0.0239,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.097984,1402.667,97.651,1.2206,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.034592,46.56,2.294,0.0287,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.0984,2095.106,103.233,1.2904,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.038624,55.6,2.174,0.0272,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.094944,2895.158,113.203,1.415,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.160576,53.495,58.771,0.7346,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.222144,618.693,169.929,2.1241,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.51456,4273.599,293.445,3.6681,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.161632,850.32,15.002,0.1875,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.180704,1521.15,13.419,0.1677,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.160416,1713.532,29.823,0.3728,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.182112,3018.779,26.27,0.3284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.179424,5.984,0.417,0.0052,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,10.67024,12.881,0.897,0.0112,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.171648,9.383,0.462,0.0058,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,10.662944,19.334,0.953,0.0119,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.17488,12.28,0.48,0.006,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,10.6656,25.772,1.008,0.0126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.04256,100.916,110.869,1.3859,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.0472,1455.921,399.881,4.9985,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.076896,14298.685,981.813,12.2727,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.044864,1531.729,27.024,0.3378,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.04368,3146.496,27.757,0.347,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.043488,3160.388,55.005,0.6876,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.04272,6434.408,55.994,0.6999,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.03808,14.099,0.982,0.0123,20,True,,NVIDIA GB300,cudnn 9.26.0, gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.063904,1075.355,74.864,0.9358,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.038816,20.747,1.022,0.0128,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.064064,1609.004,79.281,0.991,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.040448,26.546,1.038,0.013,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.063712,2157.191,84.348,1.0543,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.141856,30.277,33.263,0.4158,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.14992,458.374,125.896,1.5737,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.293216,3749.835,257.481,3.2185,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.134368,511.427,9.023,0.1128,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.1424,965.161,8.514,0.1064,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.133088,1032.692,17.974,0.2247,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.142848,1924.269,16.746,0.2093,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.138592,3.874,0.27,0.0034,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.259008,265.318,18.471,0.2309,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.129824,6.203,0.306,0.0038,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.264672,389.46,19.19,0.2399,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.137632,7.802,0.305,0.0038,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.257696,533.338,20.854,0.2607,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.071104,483.232,530.895,6.6362,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.470368,1168.778,321.014,4.0127,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,6.593824,1333.99,91.598,1.145,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.515264,1066.94,276.764,3.4595,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,0.99712,1102.687,277.624,3.4703,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,0.99584,1104.105,151.626,1.8953,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,1.945056,1130.571,146.635,1.8329,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.46496,1.742,108.914,1.3614,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,106.212608,5.176,323.539,4.0442,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.462816,2.616,109.015,1.3627,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,106.210114,7.764,323.566,4.0446,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.464416,3.486,108.951,1.3619,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,106.218941,10.351,323.559,4.0445,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.21728,158.136,173.733,2.1717,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.848896,647.613,177.872,2.2234,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,9.750272,902.138,61.945,0.7743,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.882624,622.865,161.571,2.0196,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.679104,654.82,164.864,2.0608,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.534144,716.694,98.423,1.2303,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.95872,743.235,96.397,1.205,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.667296,2.576,161.02,2.0128,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,177.047256,3.105,194.095,2.4262,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.66256,3.875,161.489,2.0186,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,177.063965,4.657,194.088,2.4261,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.65664,5.185,162.076,2.0259,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,177.049347,6.21,194.116,2.4265,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.053536,320.903,352.555,4.4069,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.253088,1086.096,298.305,3.7288,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,3.24912,1353.612,92.945,1.1618,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.514144,534.632,138.683,1.7335,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,0.992576,553.868,139.447,1.7431,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.509984,1077.986,148.039,1.8505,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,0.991296,1109.166,143.858,1.7982,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.462624,0.872,54.509,0.6814,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,53.259521,5.161,322.608,4.0326,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.461088,1.309,54.546,0.6818,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,53.265217,7.741,322.594,4.0324,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.460096,1.746,54.571,0.6821,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,53.256321,10.323,322.667,4.0333,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.196448,87.453,96.078,1.201,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.491104,559.714,153.73,1.9216,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,4.929792,892.136,61.258,0.7657,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.79696,344.908,89.469,1.1184,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.518432,362.055,91.155,1.1394,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.808448,680.014,93.386,1.1673,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.520448,723.15,93.792,1.1724,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.51792,1.415,88.433,1.1054,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,89.439072,3.073,192.108,2.4014,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.516224,2.125,88.537,1.1067,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,89.160416,4.624,192.72,2.409,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.5144,2.836,88.649,1.1081,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,89.194496,6.164,192.658,2.4082,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.0496,173.184,190.266,2.3783,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.146144,940.435,258.298,3.2287,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,1.64752,1334.748,91.65,1.1456,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.513632,267.583,69.411,0.8676,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,0.991424,277.256,69.805,0.8726,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.5152,533.536,73.27,0.9159,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,0.988896,555.929,72.104,0.9013,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.264512,0.474,29.639,0.3705,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,27.704224,4.961,310.097,3.8762,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.267456,0.71,29.602,0.37,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,27.712992,7.439,310.017,3.8752,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.267488,0.947,29.603,0.37,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,27.707872,9.921,310.094,3.8762,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.176928,48.55,53.339,0.6667,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.31664,434.054,119.217,1.4902,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.458848,894.331,61.409,0.7676,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.787136,174.606,45.293,0.5662,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.44288,190.506,47.964,0.5995,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.783552,350.81,48.176,0.6022,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.45568,377.663,48.983,0.6123,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.437472,0.747,46.691,0.5836,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,44.908928,3.06,191.298,2.3912,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.438848,1.119,46.649,0.5831,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,44.887711,4.593,191.4,2.3925,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.428736,1.503,46.982,0.5873,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,44.894943,6.123,191.381,2.3923,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.049568,86.648,95.194,1.1899,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.0936,734.182,201.649,2.5206,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,0.848256,1296.203,89.003,1.1125,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.514016,133.691,34.679,0.4335,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,0.982112,139.942,35.233,0.4404,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.51168,268.603,36.887,0.4611,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,0.98592,278.803,36.161,0.452,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.26464,0.237,14.818,0.1852,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,13.999424,4.909,306.833,3.8354,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.26752,0.355,14.801,0.185,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,13.997856,7.364,306.887,3.8361,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.265568,0.474,14.814,0.1852,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,13.999168,9.818,306.877,3.836,20,True,,NVIDIA GB300,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.153472,27.985,30.746,0.3843,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.22592,304.176,83.544,1.0443,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.277792,860.478,59.084,0.7386,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.753824,91.161,23.647,0.2956,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.389376,98.921,24.905,0.3113,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.748512,183.616,25.216,0.3152,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.385856,198.345,25.725,0.3216,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.39088,0.386,24.128,0.3016,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,17.866049,3.846,240.428,3.0053,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.386432,0.581,24.206,0.3026,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,17.844448,5.777,240.733,3.0092,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.385056,0.775,24.232,0.3029,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,17.85696,7.697,240.579,3.0072,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.03888,20.713,1.021,0.0128,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.06368,1618.706,79.759,0.997,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.039424,27.236,1.065,0.0133,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.061888,2220.769,86.834,1.0854,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.131456,32.672,35.895,0.4487,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.14816,463.819,127.392,1.5924,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.28096,3913.41,268.713,3.3589,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.13008,528.286,9.321,0.1165,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.125088,1098.738,9.693,0.1212,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.134752,1019.94,17.752,0.2219,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.139136,1975.606,17.192,0.2149,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.125632,4.273,0.298,0.0037,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.249856,275.036,19.148,0.2393,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.125024,6.441,0.317,0.004,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.248352,415.053,20.451,0.2556,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.127808,8.401,0.328,0.0041,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.2496,550.637,21.53,0.2691,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.072576,473.431,520.127,6.5016,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.46848,1173.488,322.308,4.0289,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,6.58656,1335.461,91.699,1.1462,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.543328,1011.83,262.468,3.2809,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.046304,1050.853,264.573,3.3072,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.041312,1055.891,145.005,1.8126,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.048512,1073.473,139.229,1.7404,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.4648,1.743,108.921,1.3615,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,106.177536,5.178,323.646,4.0456,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.466112,2.612,108.87,1.3609,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,106.181633,7.766,323.653,4.0457,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.465504,3.484,108.903,1.3613,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,106.185951,10.355,323.66,4.0457,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.21088,162.935,179.006,2.2376,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,0.851136,645.908,177.404,2.2176,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,9.759424,901.292,61.887,0.7736,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,0.878048,626.111,162.413,2.0302,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,1.671168,657.93,165.647,2.0706,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,1.523968,721.479,99.08,1.2385,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,2.960128,742.881,96.351,1.2044,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.657504,2.591,161.971,2.0246,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,177.084793,3.104,194.054,2.4257,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.663808,3.872,161.368,2.0171,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,177.093155,4.656,194.056,2.4257,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.659968,5.175,161.751,2.0219,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,177.068924,6.21,194.095,2.4262,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.052832,325.179,357.253,4.4657,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.25152,1092.867,300.165,3.7521,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,3.224448,1363.969,93.656,1.1707,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.538016,510.91,132.53,1.6566,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.039264,528.986,133.183,1.6648,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.540064,1017.946,139.794,1.7474,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.046208,1050.949,136.308,1.7038,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.462816,0.872,54.504,0.6813,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,53.258911,5.161,322.612,4.0327,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.465504,1.307,54.448,0.6806,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,53.249058,7.743,322.691,4.0336,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.4632,1.744,54.502,0.6813,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,53.235905,10.327,322.791,4.0349,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.1776,96.733,106.275,1.3284,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.489152,561.948,154.344,1.9293,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,4.861344,904.698,62.121,0.7765,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.796032,345.31,89.573,1.1197,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.515872,362.666,91.309,1.1414,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,0.803808,683.939,93.925,1.1741,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,1.51552,725.501,94.097,1.1762,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.514464,1.418,88.635,1.1079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,89.498306,3.071,191.981,2.3998,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.509568,2.134,88.928,1.1116,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,89.230751,4.621,192.568,2.4071,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.503232,2.857,89.308,1.1163,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,89.223228,6.162,192.596,2.4075,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.050432,170.327,187.127,2.3391,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.146656,937.152,257.396,3.2175,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,1.627584,1351.097,92.772,1.1597,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.535168,256.815,66.618,0.8327,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.04304,263.535,66.35,0.8294,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.540288,508.762,69.868,0.8733,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.043168,527.006,68.353,0.8544,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.268256,0.473,29.59,0.3699,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,27.705088,4.961,310.087,3.8761,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.267008,0.71,29.608,0.3701,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,27.718336,7.438,309.958,3.8745,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.269728,0.946,29.574,0.3697,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,27.70912,9.92,310.08,3.876,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.15904,54.011,59.338,0.7417,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.31344,438.486,120.434,1.5054,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,2.455744,895.461,61.486,0.7686,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.785152,175.048,45.407,0.5676,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.431584,192.01,48.342,0.6043,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.765344,359.156,49.323,0.6165,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.435008,383.103,49.688,0.6211,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.437408,0.747,46.693,0.5837,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,44.91312,3.06,191.28,2.391,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.429024,1.127,46.97,0.5871,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,44.879425,4.594,191.435,2.3929,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.439424,1.492,46.633,0.5829,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,44.934113,6.117,191.214,2.3902,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.050176,85.598,94.041,1.1755,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.093824,732.43,201.168,2.5146,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,0.838208,1311.741,90.07,1.1259,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.540096,127.236,33.005,0.4126,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.03968,132.194,33.282,0.416,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.538048,255.44,35.079,0.4385,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.040128,264.273,34.276,0.4285,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.266912,0.237,14.804,0.185,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,14.043616,4.893,305.868,3.8233,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.267872,0.355,14.798,0.185,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,14.058816,7.332,305.556,3.8194,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.266784,0.474,14.806,0.1851,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,14.060736,9.775,305.533,3.8192,20,True,,NVIDIA GB300,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.142688,30.1,33.069,0.4134,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.237824,288.951,79.363,0.992,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,1.274496,862.703,59.237,0.7405,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.747616,91.918,23.844,0.298,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.384064,99.301,25.001,0.3125,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.75328,182.454,25.056,0.3132,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.395168,197.021,25.554,0.3194,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.383232,0.388,24.261,0.3033,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,17.878719,3.844,240.257,3.0032,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.38208,0.583,24.283,0.3035,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,17.87808,5.766,240.28,3.0035,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.380064,0.778,24.32,0.304,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,17.852961,7.698,240.633,3.0079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png old mode 100644 new mode 100755 index 4d278947e..dcd1b7697 Binary files a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png old mode 100644 new mode 100755 index 4044a883a..638219726 Binary files a/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/gb300/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260818_140856.csv b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260819_070056.csv similarity index 56% rename from benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260818_140856.csv rename to benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260819_070056.csv index 6e57fed69..4209e4c42 100644 --- a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260818_140856.csv +++ b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_20260819_070056.csv @@ -1,105 +1,105 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.064736,530.767,583.118,12.1483,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.163136,3369.923,925.577,19.2829,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.54912,16018.526,1099.905,22.9147,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.046016,11947.058,210.782,4.3913,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.043872,25061.808,221.082,4.6059,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.051296,21434.647,373.061,7.7721,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.051392,42789.213,372.364,7.7576,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.043232,99.347,6.916,0.1441,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.355744,1545.369,107.586,2.2414,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.045056,142.988,7.045,0.1468,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.35856,2299.849,113.321,2.3609,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.04416,194.518,7.606,0.1585,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.358112,3070.301,120.051,2.5011,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.048512,354.136,389.066,8.1055,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.099136,2772.735,761.555,15.8657,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.290848,15121.461,1038.308,21.6314,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.0424,6482.97,114.379,2.3829,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.043104,12754.172,112.511,2.344,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.044544,12341.86,214.805,4.4751,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.043424,25320.368,220.345,4.5905,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.044096,48.7,3.39,0.0706,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.200992,1367.606,95.21,1.9835,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.04336,74.29,3.661,0.0763,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.206656,1995.185,98.309,2.0481,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.043584,98.545,3.853,0.0803,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.199872,2750.539,107.548,2.2406,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.046496,184.746,202.968,4.2285,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.064864,2118.879,581.967,12.1243,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.16288,13500.879,927.032,19.3132,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.0416,3303.821,58.289,1.2144,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.045728,6011.151,53.027,1.1047,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.043776,6279.192,109.287,2.2768,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.043968,12503.544,108.809,2.2669,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.043808,24.51,1.706,0.0355,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.119104,1153.941,80.335,1.6737,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.043744,36.819,1.814,0.0378,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.120288,1713.874,84.448,1.7593,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.042464,50.572,1.977,0.0412,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.118816,2313.475,90.458,1.8845,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.041472,103.563,113.778,2.3704,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.048576,1414.68,388.553,8.0949,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.0984,11173.899,767.251,15.9844,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.0432,1590.729,28.065,0.5847,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.044416,3094.357,27.297,0.5687,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.044096,3116.812,54.247,1.1301,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.042752,6429.592,55.952,1.1657,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.043648,12.3,0.856,0.0178,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.075872,905.729,63.055,1.3137,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.04624,17.416,0.858,0.0179,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.075904,1358.021,66.914,1.394,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.043008,24.966,0.976,0.0203,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.075168,1828.424,71.493,1.4894,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.127584,269.311,295.874,6.164,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.238272,443.97,121.94,2.5404,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,19.186016,458.464,31.48,0.6558,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.182944,464.735,120.552,2.5115,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,2.3264,472.624,118.992,2.479,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,2.305312,476.947,65.499,1.3646,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,4.593088,478.768,62.096,1.2937,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.170912,3.668,229.281,4.7767,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,71.568642,7.682,480.153,10.0032,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.173056,5.492,228.876,4.7683,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,71.564384,11.523,480.211,10.0044,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.173312,7.321,228.84,4.7675,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,71.710434,15.333,479.263,9.9846,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.084928,202.287,222.24,4.63,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.659552,416.765,114.468,2.3847,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,9.691936,453.784,31.159,0.6491,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.609472,451.01,116.992,2.4373,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.184832,463.995,116.82,2.4337,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.1784,466.527,64.068,1.3347,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,2.317952,474.346,61.523,1.2817,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.169504,1.836,114.779,2.3912,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,36.356544,7.561,472.596,9.8458,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.172384,2.748,114.504,2.3855,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,36.091969,11.424,476.09,9.9185,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.176576,3.65,114.103,2.3771,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,36.360737,15.119,472.599,9.8458,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.063168,135.986,149.398,3.1125,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.36992,371.537,102.046,2.126,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,4.822176,456.023,31.313,0.6523,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.602432,228.14,59.179,1.2329,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.170976,234.743,59.101,1.2313,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.608672,451.603,62.018,1.292,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.186464,463.357,60.097,1.252,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.1704,0.917,57.345,1.1947,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,18.084896,7.6,475.036,9.8966,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.169152,1.378,57.41,1.196,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,18.079103,11.403,475.218,9.9004,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.171072,1.834,57.319,1.1942,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,18.128384,15.163,473.955,9.8741,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.049888,86.092,94.584,1.9705,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.224128,306.608,84.212,1.7544,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.421056,454.145,31.184,0.6497,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.60272,114.016,29.576,0.6162,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.167328,117.738,29.643,0.6176,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.600928,228.711,31.409,0.6543,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.16784,235.373,30.528,0.636,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.167744,0.46,28.738,0.5987,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,9.069568,7.577,473.616,9.867,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.168704,0.689,28.716,0.5983,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,9.14128,11.276,469.929,9.7902,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.168832,0.919,28.715,0.5982,20,True,,NVIDIA H200,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,9.078752,15.139,473.195,9.8582,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.06352,540.928,594.281,12.3809,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.159584,3444.931,946.178,19.7121,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.543456,16185.474,1111.368,23.1535,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.044256,12422.175,219.164,4.5659,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.045216,24316.87,214.511,4.469,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.050976,21569.201,375.402,7.8209,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.051104,43030.353,374.462,7.8013,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.04752,90.382,6.292,0.1311,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.357216,1539.001,107.143,2.2321,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.044544,144.631,7.126,0.1485,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.356224,2314.93,114.064,2.3763,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.042624,201.528,7.88,0.1642,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.356608,3083.25,120.557,2.5116,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.048224,356.251,391.39,8.1539,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.097792,2810.842,772.021,16.0838,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.28896,15220.26,1045.092,21.7728,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.043488,6320.776,111.517,2.3233,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.042976,12792.159,112.846,2.351,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.044,12494.45,217.46,4.5304,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.044608,24648.305,214.496,4.4687,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.042848,50.119,3.489,0.0727,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.200416,1371.537,95.484,1.9892,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.042656,75.516,3.721,0.0775,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.199712,2064.557,101.727,2.1193,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.04272,100.538,3.931,0.0819,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.198688,2766.93,108.189,2.2539,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.042272,203.206,223.249,4.651,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.064096,2144.267,588.941,12.2696,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.16176,13594.357,933.45,19.4469,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.043232,3179.102,56.089,1.1685,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.042048,6537.241,57.668,1.2014,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.043808,6274.605,109.207,2.2751,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.042432,12956.161,112.748,2.3489,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.042144,25.478,1.774,0.037,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.117792,1166.794,81.23,1.6923,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.043968,36.631,1.805,0.0376,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.117664,1752.094,86.331,1.7986,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.043648,49.2,1.924,0.0401,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.117504,2339.307,91.468,1.9056,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.04192,102.456,112.562,2.345,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.04848,1417.481,389.323,8.1109,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.097856,11236.016,771.516,16.0733,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.042528,1615.864,28.509,0.5939,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.044832,3065.644,27.044,0.5634,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.042912,3202.809,55.743,1.1613,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.043168,6367.631,55.413,1.1544,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.043072,12.464,0.868,0.0181,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.075616,908.795,63.269,1.3181,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.043936,18.329,0.903,0.0188,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.076192,1352.888,66.661,1.3888,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.042848,25.059,0.98,0.0204,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.075648,1816.822,71.039,1.48,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.123936,277.238,304.583,6.3455,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.228288,447.579,122.931,2.5611,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,19.567072,449.535,30.867,0.6431,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,1.171072,469.447,121.774,2.537,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,2.333984,471.088,118.606,2.471,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,2.29952,478.148,65.664,1.368,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,4.764544,461.539,59.861,1.2471,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.163648,3.691,230.713,4.8065,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,89.584892,6.137,383.591,7.9915,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.152896,5.588,232.878,4.8516,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,93.836288,8.788,366.234,7.6299,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.160384,7.403,231.39,4.8206,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,97.199326,11.312,353.584,7.3663,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.083904,204.756,224.952,4.6865,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.660192,416.361,114.357,2.3824,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,9.774432,449.954,30.896,0.6437,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,0.601696,456.839,118.504,2.4688,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,1.164,472.299,118.911,2.4773,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.163776,472.39,64.873,1.3515,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,2.283264,481.553,62.457,1.3012,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.163552,1.846,115.366,2.4035,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,36.980129,7.433,464.627,9.6797,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.151008,2.799,116.63,2.4298,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,41.315842,9.98,415.894,8.6645,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.160416,3.701,115.692,2.4102,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,41.272385,13.32,416.357,8.6741,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.062688,137.027,150.542,3.1363,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.366912,374.583,102.882,2.1434,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,4.9488,444.355,30.511,0.6357,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.595936,230.627,59.825,1.2463,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.160352,236.892,59.642,1.2425,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,0.602624,456.135,62.641,1.305,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,1.167136,471.03,61.092,1.2728,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.160032,0.926,57.858,1.2054,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,18.082048,7.601,475.111,9.8981,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.163328,1.384,57.698,1.202,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,18.410721,11.198,466.658,9.722,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.152832,1.863,58.226,1.213,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,18.454016,14.895,465.591,9.6998,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.04944,86.872,95.441,1.9883,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.222752,308.502,84.733,1.7653,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.401472,457.849,31.438,0.655,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.596768,115.153,29.871,0.6223,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.16096,118.384,29.806,0.6209,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.597984,229.837,31.563,0.6576,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.161696,236.618,30.689,0.6394,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.16096,0.462,28.906,0.6022,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,9.14144,7.517,469.892,9.7894,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.16,0.694,28.932,0.6027,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,9.003584,11.449,477.116,9.9399,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.162272,0.924,28.877,0.6016,20,True,,NVIDIA H200,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,9.235136,14.882,465.182,9.6913,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png old mode 100644 new mode 100755 index 13a3f143d..9d81572d7 Binary files a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png old mode 100644 new mode 100755 index 55f2d10a6..30aa42750 Binary files a/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/h200/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260818_142442.csv b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260819_065633.csv similarity index 63% rename from benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260818_142442.csv rename to benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260819_065633.csv index 054290278..4ef2fba3f 100644 --- a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260818_142442.csv +++ b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_20260819_065633.csv @@ -1,209 +1,209 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.056192,611.47,671.781,37.4878,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.182464,3012.955,827.533,46.1793,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.658048,13366.948,917.835,51.2185,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.034976,15718.086,277.314,15.4751,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.057152,601.199,660.497,36.8581,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.18704,2939.242,807.287,45.0495,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,0.675424,13023.069,894.223,49.9008,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.035552,15463.429,272.821,15.2244,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.035424,31038.606,273.807,15.2794,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.056224,19555.913,340.362,18.9934,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.056992,38584.77,335.775,18.7375,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.021376,200.925,13.988,0.7806,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.215744,2548.186,177.4,9.8996,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.021376,301.387,14.85,0.8287,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.218304,3777.456,186.127,10.3866,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.022144,387.913,15.168,0.8464,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.217792,5048.448,197.398,11.0155,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.116704,294.418,323.457,18.0501,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.384384,1430.226,392.823,21.9209,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.554848,5657.205,388.449,21.6769,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.270432,2032.88,35.866,2.0015,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.46464,2366.373,20.875,1.1649,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.290144,3789.538,65.955,3.6805,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.487424,4511.52,39.261,2.1909,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.452992,9.481,0.66,0.0368,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,51.771328,10.619,0.739,0.0413,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.451584,14.266,0.703,0.0392,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,51.776798,15.927,0.785,0.0438,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.451584,19.022,0.744,0.0415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,51.774593,21.237,0.83,0.0463,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.037248,461.229,506.722,28.2769,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.097984,2805.335,770.508,42.9971,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.34672,12684.721,870.991,48.6044,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.03456,7953.643,140.326,7.8307,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.035232,15603.878,137.649,7.6813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.035776,15366.609,267.449,14.9246,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.036256,30326.335,263.908,14.727,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.022496,95.461,6.646,0.3709,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.118784,2314.099,161.103,8.9901,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.021472,150.02,7.392,0.4125,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.123296,3344.122,164.775,9.1951,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.022528,190.65,7.455,0.416,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.123008,4469.269,174.751,9.7517,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.09216,186.414,204.8,11.4286,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.204896,1341.548,368.467,20.5618,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.783584,5612.731,385.396,21.5065,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.131424,2091.535,36.901,2.0592,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.255168,2154.486,19.006,1.0606,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.161024,3414.123,59.421,3.3159,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.262656,4186.128,36.429,2.0329,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.247552,8.675,0.604,0.0337,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,25.492577,10.783,0.751,0.0419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.248352,12.97,0.639,0.0357,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,25.579329,16.119,0.794,0.0443,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.248288,17.298,0.676,0.0377,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,25.580544,21.491,0.84,0.0469,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.02944,291.778,320.557,17.8882,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.060128,2285.773,627.806,35.0338,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.188672,11655.27,800.304,44.6598,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.035776,3841.652,67.778,3.7823,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.035072,7837.532,69.139,3.8582,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.036,7635.498,132.892,7.4159,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.035136,15646.511,136.16,7.5982,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.02192,48.985,3.41,0.1903,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.071648,1918.252,133.545,7.4523,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.0216,74.565,3.674,0.205,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.072096,2859.499,140.897,7.8625,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.021568,99.568,3.893,0.2173,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.072512,3790.792,148.222,8.2713,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.075968,113.073,124.226,6.9322,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.116352,1181.234,324.436,18.1047,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.393792,5584.225,383.438,21.3972,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.071264,1928.589,34.026,1.8988,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.129024,2130.44,18.794,1.0488,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.078208,3514.703,61.172,3.4136,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.131072,4194.304,36.5,2.0368,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.108864,9.863,0.687,0.0383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,12.682784,10.837,0.754,0.0421,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.109568,14.7,0.724,0.0404,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,12.684384,16.253,0.801,0.0447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.10864,19.767,0.773,0.0431,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,12.68336,21.672,0.847,0.0473,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.022432,191.466,210.351,11.7383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.037024,1856.079,509.787,28.448,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.097664,11258.106,773.033,43.138,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.035328,1945.184,34.319,1.9151,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.0352,3904.516,34.444,1.9221,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.03488,3940.337,68.58,3.827,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.035936,7649.095,66.565,3.7145,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.02256,23.797,1.657,0.0925,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.045248,1518.73,105.731,5.9002,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.021952,36.685,1.808,0.1009,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.045792,2251.031,110.915,6.1895,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.02144,50.081,1.958,0.1093,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.045632,3011.898,117.767,6.5718,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.061024,70.382,77.324,4.3149,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.07952,864.179,237.354,13.2452,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.186144,5906.78,405.586,22.6332,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.059456,1155.804,20.392,1.1379,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.06016,2284.557,20.153,1.1246,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.058624,2344.414,40.803,2.277,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.059488,4620.729,40.211,2.2439,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.071808,7.476,0.52,0.029,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.085856,800.404,55.723,3.1095,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.060544,13.301,0.655,0.0366,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.085728,1202.398,59.246,3.3061,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.059936,17.915,0.7,0.0391,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.086432,1590.14,62.175,3.4696,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.164128,209.347,229.996,12.8346,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.715168,320.526,88.035,4.9127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,25.462591,345.452,23.72,1.3237,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.337824,235.157,61.0,3.404,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.648992,236.505,59.545,3.3228,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,4.64096,236.915,32.535,1.8156,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,9.269792,237.225,30.768,1.717,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.492224,1.723,107.722,6.0113,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,98.148384,5.601,350.122,19.5381,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.492544,2.585,107.715,6.0109,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,98.141312,8.403,350.169,19.5407,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.057856,19004.28,330.761,18.4576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.057664,38135.115,331.862,18.5191,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.021792,197.089,13.721,0.7657,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,0.221344,2483.717,172.912,9.6491,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.021536,299.148,14.74,0.8225,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,0.221088,3729.889,183.783,10.2558,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.021824,393.6,15.39,0.8588,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,0.22144,4965.28,194.146,10.834,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,128.0,16,0.114016,301.359,331.083,18.4756,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,128.0,16,0.390336,1408.417,386.833,21.5867,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,128.0,16,1.576704,5578.785,383.065,21.3764,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,128.0,16,0.268928,2044.249,36.067,2.0126,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,128.0,16,0.465344,2362.793,20.843,1.1631,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,128.0,16,0.291616,3770.409,65.622,3.662,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,128.0,16,0.491456,4474.507,38.938,2.1729,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,128.0,16,0.449792,9.549,0.665,0.0371,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,128.0,16,51.766815,10.62,0.739,0.0413,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,128.0,16,0.450496,14.301,0.705,0.0393,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,128.0,16,51.769569,15.929,0.785,0.0438,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,128.0,16,0.451424,19.029,0.744,0.0415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,128.0,16,51.771423,21.238,0.83,0.0463,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.036992,464.421,510.228,28.4726,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.096064,2861.404,785.908,43.8565,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.355616,12367.403,849.202,47.3885,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.034848,7887.91,139.166,7.766,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.03504,15689.379,138.404,7.7234,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.036,15270.995,265.785,14.8317,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.035744,30760.732,267.688,14.938,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.021888,98.112,6.83,0.3812,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,0.120864,2274.275,158.331,8.8354,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.020992,153.45,7.561,0.4219,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,0.121152,3403.302,167.691,9.3578,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.020864,205.855,8.049,0.4492,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,0.120544,4560.624,178.323,9.9511,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,128.0,16,0.09056,189.707,208.418,11.6305,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,128.0,16,0.202656,1356.377,372.54,20.7891,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,128.0,16,0.787104,5587.631,383.672,21.4103,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,128.0,16,0.130144,2112.106,37.264,2.0795,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,128.0,16,0.25568,2150.171,18.968,1.0585,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,128.0,16,0.159968,3436.661,59.814,3.3378,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,128.0,16,0.262272,4192.257,36.482,2.0358,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,128.0,16,0.24784,8.665,0.603,0.0337,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,128.0,16,25.492544,10.783,0.751,0.0419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,128.0,16,0.246816,13.051,0.643,0.0359,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,128.0,16,25.577248,16.12,0.794,0.0443,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,128.0,16,0.247008,17.388,0.68,0.0379,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,128.0,16,25.577345,21.494,0.84,0.0469,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.028608,300.263,329.879,18.4084,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.0584,2353.407,646.382,36.0704,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.185984,11823.723,811.871,45.3053,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.0352,3904.516,68.887,3.8442,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.035136,7823.256,69.013,3.8512,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.035232,7801.939,135.789,7.5775,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.035104,15660.775,136.284,7.6052,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.0216,49.71,3.461,0.1931,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,0.070528,1948.715,135.666,7.5707,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.021984,73.263,3.61,0.2014,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,0.070528,2923.072,144.029,8.0373,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.021152,101.526,3.97,0.2215,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,0.070016,3925.93,153.506,8.5662,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,128.0,16,0.078208,109.834,120.668,6.7337,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,128.0,16,0.115424,1190.731,327.044,18.2502,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,128.0,16,0.389536,5645.238,387.628,21.631,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,128.0,16,0.07424,1851.279,32.662,1.8227,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,128.0,16,0.127872,2149.633,18.963,1.0582,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,128.0,16,0.077024,3568.731,62.112,3.4661,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,128.0,16,0.130976,4197.378,36.527,2.0383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,128.0,16,0.108384,9.907,0.69,0.0385,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,128.0,16,12.684704,10.835,0.754,0.0421,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,128.0,16,0.107648,14.962,0.737,0.0411,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,128.0,16,12.68256,16.255,0.801,0.0447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,128.0,16,0.109088,19.686,0.77,0.043,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,128.0,16,12.685088,21.669,0.847,0.0473,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.021632,198.547,218.13,12.1724,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.036384,1888.728,518.755,28.9484,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.096992,11336.106,778.389,43.4369,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.035104,1957.597,34.538,1.9273,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.034912,3936.725,34.728,1.9379,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.035072,3918.766,68.204,3.806,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.03488,7880.674,68.58,3.827,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.021984,24.421,1.7,0.0949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.043968,1562.943,108.809,6.0719,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.021632,37.228,1.834,0.1024,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.044608,2310.779,113.859,6.3538,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.022112,48.559,1.899,0.106,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.044384,3096.588,121.079,6.7566,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,128.0,16,0.06128,70.088,77.001,4.2969,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,128.0,16,0.085536,803.398,220.66,12.3136,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,128.0,16,0.184672,5953.862,408.819,22.8136,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,128.0,16,0.05904,1163.948,20.536,1.146,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,128.0,16,0.059424,2312.853,20.403,1.1385,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,128.0,16,0.059776,2299.233,40.017,2.2331,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,128.0,16,0.059776,4598.466,40.017,2.2331,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,128.0,16,0.060096,8.934,0.622,0.0347,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,128.0,16,0.085984,799.212,55.64,3.1049,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,128.0,16,0.0592,13.603,0.67,0.0374,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,128.0,16,0.084992,1212.811,59.759,3.3348,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,128.0,16,0.059808,17.953,0.702,0.0392,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_swa-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,128.0,16,0.085152,1614.043,63.11,3.5218,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.161184,213.171,234.197,13.069,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.685056,326.254,89.608,5.0005,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,24.767584,355.145,24.386,1.3608,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.341888,234.749,60.894,3.3981,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.655296,236.185,59.464,3.3183,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,4.642016,236.861,32.528,1.8152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,9.27456,237.103,30.752,1.7161,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,2.528768,1.698,106.166,5.9244,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,98.214783,5.597,349.886,19.5249,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,2.528704,2.548,106.175,5.9249,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,98.220291,8.396,349.887,19.525,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,2.492448,3.446,107.726,6.0115,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,98.14032,11.203,350.194,19.5421,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.18176,189.039,207.685,11.5895,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.626944,337.907,92.809,5.1791,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,23.576639,373.085,25.618,1.4296,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.115712,259.844,67.403,3.7614,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.346528,252.963,63.689,3.554,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,3.819616,287.859,39.531,2.206,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,7.482912,293.873,38.115,2.127,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.5872,2.706,169.146,9.4389,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,136.240677,4.035,252.23,14.0753,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.587232,4.059,169.153,9.4393,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,136.232346,6.053,252.26,14.077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.58752,5.411,169.132,9.4382,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,136.230011,8.071,252.28,14.0781,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.097408,176.37,193.766,10.8128,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.889984,308.857,84.83,4.7338,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,12.827712,342.855,23.542,1.3137,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,2.331008,117.922,30.589,1.707,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,4.644,118.38,29.804,1.6632,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,2.33376,235.567,32.35,1.8053,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,4.652736,236.315,30.65,1.7104,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.4912,0.862,53.883,3.0069,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,49.181664,5.589,349.357,19.4954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.490656,1.293,53.898,3.0077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,49.17728,8.384,349.41,19.4983,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.490656,1.724,53.902,3.0079,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,49.181599,11.178,349.4,19.4978,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.134464,127.766,140.367,7.833,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.824896,333.227,91.524,5.1073,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,11.641536,377.789,25.941,1.4476,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,1.24112,221.476,57.451,3.206,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,2.587936,212.43,53.484,2.9846,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.8192,302.196,41.5,2.3159,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,3.673408,299.316,38.821,2.1664,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.37472,1.562,97.645,5.4489,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,67.753441,4.057,253.595,14.1515,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.373152,2.346,97.762,5.4555,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,67.838333,6.078,253.294,14.1347,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.370752,3.133,97.939,5.4654,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,67.833984,8.104,253.325,14.1365,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.069216,124.103,136.344,7.6085,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.501632,273.984,75.252,4.1993,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,6.3592,345.802,23.744,1.325,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,2.33024,58.981,15.3,0.8538,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,4.642784,59.205,14.906,0.8318,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,2.330816,117.932,16.196,0.9038,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,4.645152,118.35,15.35,0.8566,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.568256,0.418,26.133,1.4583,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,24.688416,5.567,347.976,19.4183,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.492064,0.646,26.934,1.503,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,24.690559,8.35,347.967,19.4178,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.489952,0.862,26.958,1.5044,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,24.691681,11.132,347.973,19.4181,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,98.220192,11.194,349.909,19.5262,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,64,64,gqa,,16,0.185536,185.192,203.458,11.3537,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,64,64,gqa,,16,1.671136,328.971,90.355,5.0421,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,64,64,gqa,,16,22.962656,383.061,26.303,1.4678,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,64,64,gqa,,16,2.105216,261.14,67.74,3.7801,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,64,64,gqa,,16,4.354304,252.511,63.575,3.5477,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,64,64,gqa,,16,3.82624,287.361,39.463,2.2022,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,64,64,gqa,,16,7.48912,293.629,38.084,2.1252,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,64,64,gqa,,16,1.606464,2.674,167.117,9.3258,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,64,64,gqa,,16,135.993027,4.043,252.689,14.1009,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,64,64,gqa,,16,1.593856,4.042,168.45,9.4001,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,64,64,gqa,,16,136.00531,6.063,252.682,14.1005,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,64,64,gqa,,16,1.588032,5.409,169.078,9.4351,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,64,64,gqa,,16,135.996643,8.085,252.713,14.1023,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.097856,175.563,192.879,10.7633,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.888704,309.302,84.952,4.7406,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,12.375296,355.389,24.403,1.3618,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,2.330688,117.939,30.593,1.7072,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,4.645152,118.35,29.797,1.6628,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,2.336224,235.318,32.316,1.8033,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,4.653408,236.281,30.646,1.7101,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,2.52544,0.85,53.153,2.9661,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,49.224545,5.584,349.053,19.4784,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,2.527808,1.274,53.106,2.9635,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,49.22179,8.377,349.094,19.4807,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,2.491424,1.724,53.885,3.007,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,49.222721,11.169,349.108,19.4815,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,64,64,gqa,,16,0.134112,128.101,140.736,7.8536,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,64,64,gqa,,16,0.853728,321.974,88.433,4.9349,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,64,64,gqa,,16,11.243104,391.177,26.86,1.4989,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,64,64,gqa,,16,1.241152,221.47,57.449,3.2059,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,64,64,gqa,,16,2.590976,212.181,53.421,2.9811,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,64,64,gqa,,16,1.817952,302.404,41.529,2.3175,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,64,64,gqa,,16,3.67488,299.197,38.806,2.1655,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,64,64,gqa,,16,1.38992,1.545,96.577,5.3893,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,64,64,gqa,,16,67.671967,4.062,253.901,14.1686,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,64,64,gqa,,16,1.390944,2.316,96.512,5.3857,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,64,64,gqa,,16,67.758141,6.085,253.593,14.1514,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,64,64,gqa,,16,1.392928,3.083,96.38,5.3784,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,64,64,gqa,,16,67.754684,8.114,253.622,14.153,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.07072,121.464,133.444,7.4467,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.499808,274.983,75.526,4.2146,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,6.355488,346.004,23.758,1.3258,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,2.36032,58.229,15.105,0.8429,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,4.646464,59.159,14.894,0.8312,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,2.33184,117.88,16.188,0.9034,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,4.645728,118.336,15.348,0.8565,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,2.524608,0.425,26.585,1.4835,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,24.708128,5.562,347.699,19.4028,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,2.52544,0.638,26.578,1.4831,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,24.709503,8.343,347.701,19.4029,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,2.524832,0.851,26.586,1.4836,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,24.711967,11.123,347.687,19.4022,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,64,64,gqa,,16,0.099456,86.369,94.888,5.2951,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.435488,315.598,86.681,4.8371,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,5.516864,398.6,27.37,1.5273,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.63232,217.357,56.382,3.1463,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.237568,222.111,55.921,3.1206,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,1.191136,230.77,31.691,1.7685,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,2.353024,233.638,30.303,1.691,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.263808,0.85,53.107,2.9636,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,33.871521,4.058,253.634,14.1537,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.255872,1.282,53.446,2.9825,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,33.870529,6.087,253.657,14.155,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.263712,1.699,53.118,2.9641,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,33.867359,8.116,253.697,14.1572,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.053536,80.226,88.139,4.9185,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.320704,214.277,58.853,3.2842,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,3.3472,328.487,22.555,1.2587,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,2.33088,29.482,7.648,0.4268,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,4.643904,29.596,7.451,0.4158,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,2.329632,58.996,8.102,0.4521,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,4.642144,59.214,7.68,0.4286,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.529216,0.212,13.268,0.7404,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,13.530496,5.079,317.467,17.7158,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.48992,0.323,13.479,0.7522,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,13.530912,7.618,317.477,17.7164,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.490176,0.431,13.478,0.7521,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,13.532032,10.157,317.47,17.716,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.070208,61.175,67.209,3.7505,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.28496,241.155,66.235,3.6962,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.771232,396.759,27.243,1.5203,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.626784,109.638,28.44,1.5871,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.204576,114.097,28.726,1.603,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.641632,214.202,29.416,1.6415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.221184,225.091,29.194,1.6291,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.20032,0.447,27.958,1.5602,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,11.66496,5.891,368.239,20.549,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.200928,0.671,27.946,1.5595,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,11.66368,8.838,368.302,20.5526,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.199456,0.895,27.982,1.5615,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,11.664704,11.782,368.292,20.552,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,64,64,gqa,,16,0.44512,308.768,84.806,4.7325,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,64,64,gqa,,16,5.505728,399.406,27.425,1.5304,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,64,64,gqa,,16,0.648288,212.003,54.993,3.0688,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,64,64,gqa,,16,1.259392,218.262,54.952,3.0665,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,64,64,gqa,,16,1.23344,222.855,30.604,1.7078,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,64,64,gqa,,16,2.348864,234.052,30.356,1.694,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,64,64,gqa,,16,1.23712,0.868,54.253,3.0275,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,64,64,gqa,,16,33.871967,4.058,253.631,14.1535,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,64,64,gqa,,16,1.235584,1.304,54.323,3.0314,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,64,64,gqa,,16,33.866688,6.087,253.686,14.1566,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,64,64,gqa,,16,1.233856,1.74,54.403,3.0359,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,64,64,gqa,,16,33.865887,8.117,253.708,14.1578,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.05344,80.37,88.297,4.9273,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.313504,219.198,60.205,3.3596,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,3.344928,328.71,22.571,1.2595,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,2.363488,29.075,7.542,0.4209,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,4.707392,29.196,7.351,0.4102,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,2.361824,58.192,7.991,0.446,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,4.706816,58.4,7.574,0.4227,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,2.528256,0.212,13.273,0.7407,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,13.535456,5.077,317.351,17.7093,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,2.525952,0.319,13.286,0.7414,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,13.534592,7.616,317.391,17.7115,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,2.526464,0.425,13.284,0.7413,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,13.53584,10.154,317.381,17.711,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,64,64,gqa,,16,0.070816,60.65,66.632,3.7183,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,64,64,gqa,,16,0.280512,244.979,67.285,3.7548,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,64,64,gqa,,16,2.761152,398.208,27.343,1.5258,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,64,64,gqa,,16,0.61728,111.326,28.878,1.6115,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,64,64,gqa,,16,1.181888,116.288,29.278,1.6338,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,64,64,gqa,,16,0.631968,217.478,29.866,1.6666,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,64,64,gqa,,16,1.199296,229.199,29.727,1.6589,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,64,64,gqa,,16,1.181312,0.454,28.408,1.5853,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,64,64,gqa,,16,11.662464,5.892,368.318,20.5534,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,64,64,gqa,,16,1.180416,0.682,28.431,1.5866,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,64,64,gqa,,16,11.665088,8.837,368.257,20.5501,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,64,64,gqa,,16,1.179584,0.91,28.453,1.5878,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +gpt_oss,gpt_oss_full-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,64,64,gqa,,16,11.66528,11.782,368.274,20.551,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png old mode 100644 new mode 100755 index b05935fb9..515726f2b Binary files a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png and b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_context.png differ diff --git a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png old mode 100644 new mode 100755 index 356c44e16..54eae09db Binary files a/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png and b/benchmark/attention_inference/results/gpt_oss/rtxpro6000/gpt_oss_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260818_124034.csv b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260819_054736.csv similarity index 59% rename from benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260818_124034.csv rename to benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260819_054736.csv index 139aefe8f..8658064bc 100644 --- a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260818_124034.csv +++ b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_20260819_054736.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.645952,42.388,234.079,2.926,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,68.169281,51.412,283.912,3.5489,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.54096,1.919,5.305,0.0663,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2382.912109,2.942,8.133,0.1017,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.53664,2.878,5.313,0.0664,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2382.477783,4.413,8.146,0.1018,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.532896,3.838,5.321,0.0665,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2380.615479,5.889,8.164,0.102,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.359776,38.052,419.982,5.2498,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,37.622913,46.577,514.068,6.4258,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.794208,2.14,11.818,0.1477,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1133.448608,3.092,17.075,0.2134,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.788096,3.212,11.832,0.1479,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1133.631104,4.637,17.084,0.2136,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.77776,4.286,11.85,0.1481,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1132.288574,6.19,17.117,0.214,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.25904,26.425,583.104,7.2888,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.585793,32.956,727.232,9.0904,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.748736,1.074,11.852,0.1482,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,561.145264,3.123,34.467,0.4308,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.733792,1.613,11.87,0.1484,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,561.032715,4.685,34.485,0.4311,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.732192,2.15,11.876,0.1484,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,560.009644,6.258,34.56,0.432,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.152672,22.418,989.186,12.3648,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.000192,31.291,1380.745,17.2593,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.723584,0.538,11.871,0.1484,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,285.567413,3.068,67.704,0.8463,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.710304,0.808,11.886,0.1486,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,285.075317,4.61,67.833,0.8479,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.710304,1.077,11.888,0.1486,20,True,,NVIDIA GB200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,285.309204,6.142,67.789,0.8474,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.648672,42.21,233.098,2.9137,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,68.220062,51.373,283.701,3.5463,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.567425,1.917,5.3,0.0663,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2383.392578,2.941,8.132,0.1016,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.546368,2.877,5.311,0.0664,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2381.427246,4.415,8.15,0.1019,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.535263,3.838,5.321,0.0665,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2381.261963,5.887,8.161,0.102,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.361344,37.887,418.159,5.227,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,37.635426,46.561,513.897,6.4237,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.794368,2.14,11.818,0.1477,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1133.383667,3.092,17.076,0.2135,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.789312,3.211,11.831,0.1479,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1133.621582,4.637,17.085,0.2136,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.786016,4.283,11.842,0.148,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1132.897705,6.187,17.107,0.2138,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.26288,26.039,574.586,7.1823,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.631359,32.9,725.988,9.0748,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.772544,1.072,11.83,0.1479,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,560.801575,3.125,34.488,0.4311,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.756352,1.61,11.849,0.1481,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,560.475159,4.69,34.52,0.4315,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.751584,2.147,11.858,0.1482,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,560.293884,6.255,34.543,0.4318,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.156416,21.881,965.509,12.0689,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.005152,31.28,1380.256,17.2532,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.738272,0.537,11.858,0.1482,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,285.479218,3.069,67.725,0.8466,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.726592,0.807,11.871,0.1484,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,284.811493,4.614,67.895,0.8487,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.723712,1.076,11.875,0.1484,20,True,,NVIDIA GB200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,284.934418,6.15,67.878,0.8485,20,True,,NVIDIA GB200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_context.png b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_context.png old mode 100644 new mode 100755 diff --git a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png old mode 100644 new mode 100755 index a6709ab3f..800bf3804 Binary files a/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/gb200/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260818_124631.csv b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260819_054557.csv similarity index 57% rename from benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260818_124631.csv rename to benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260819_054557.csv index 3bb2b36b6..4e2bf98fd 100644 --- a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260818_124631.csv +++ b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_20260819_054557.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.632864,43.264,238.92,2.9865,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,66.739746,52.513,289.993,3.6249,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.193184,1.942,5.371,0.0671,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2350.77002,2.982,8.244,0.1031,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.179457,2.915,5.381,0.0673,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2348.12915,4.478,8.265,0.1033,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.180672,3.886,5.388,0.0673,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2346.813721,5.974,8.281,0.1035,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.353024,38.78,428.014,5.3502,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,37.026817,47.326,522.344,6.5293,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.802208,2.139,11.811,0.1476,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1118.749146,3.133,17.3,0.2162,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.79728,3.209,11.823,0.1478,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1119.354614,4.696,17.302,0.2163,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.793632,4.28,11.835,0.1479,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1118.605103,6.266,17.326,0.2166,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.256928,26.642,587.897,7.3487,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.131872,33.529,739.864,9.2483,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.738976,1.075,11.861,0.1483,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,553.474365,3.166,34.944,0.4368,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.72672,1.614,11.877,0.1485,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,552.815613,4.755,34.998,0.4375,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.71552,2.153,11.891,0.1486,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,552.73291,6.341,35.015,0.4377,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.149344,22.917,1011.23,12.6404,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,13.573472,32.275,1424.153,17.8019,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.710304,0.539,11.884,0.1485,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,281.895721,3.108,68.586,0.8573,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.697728,0.809,11.898,0.1487,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,281.512268,4.669,68.691,0.8586,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.690432,1.079,11.907,0.1488,20,True,,NVIDIA GB300,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,281.619934,6.222,68.677,0.8585,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.636096,43.044,237.706,2.9713,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,66.632004,52.598,290.462,3.6308,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.157185,1.945,5.377,0.0672,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2345.772949,2.988,8.262,0.1033,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.141985,2.919,5.388,0.0673,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2343.797363,4.486,8.28,0.1035,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.13792,3.892,5.396,0.0674,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2343.576904,5.982,8.293,0.1037,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.354816,38.584,425.853,5.3232,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,36.863167,47.537,524.663,6.5583,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.81632,2.136,11.798,0.1475,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1117.765869,3.135,17.315,0.2164,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,12.809056,3.206,11.813,0.1477,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1118.273682,4.701,17.319,0.2165,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.803968,4.277,11.825,0.1478,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1115.760986,6.282,17.37,0.2171,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.257344,26.599,586.947,7.3368,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,26.157087,33.497,739.151,9.2394,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.725344,1.076,11.874,0.1484,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,553.244812,3.167,34.959,0.437,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.721536,1.614,11.882,0.1485,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,552.736938,4.755,35.003,0.4375,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.71392,2.154,11.893,0.1487,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,552.290161,6.346,35.043,0.438,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.15328,22.329,985.263,12.3158,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,13.54928,32.333,1426.695,17.8337,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.709504,0.539,11.885,0.1486,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,282.027161,3.107,68.554,0.8569,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.695456,0.809,11.9,0.1487,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,281.535736,4.668,68.685,0.8586,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.690208,1.079,11.907,0.1488,20,True,,NVIDIA GB300,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,281.346924,6.228,68.743,0.8593,20,True,,NVIDIA GB300,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_context.png b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_context.png old mode 100644 new mode 100755 diff --git a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png old mode 100644 new mode 100755 index 4551c878a..85889d9e7 Binary files a/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/gb300/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260818_141810.csv b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260819_071246.csv similarity index 57% rename from benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260818_141810.csv rename to benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260819_071246.csv index e9fe698e8..8e8ea17a1 100644 --- a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260818_141810.csv +++ b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_20260819_071246.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.6832,40.077,221.317,4.6108,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,79.962273,43.829,242.04,5.0425,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.968609,1.89,5.227,0.1089,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,2812.320068,2.492,6.891,0.1436,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,28.948896,2.837,5.238,0.1091,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2814.22168,3.736,6.896,0.1437,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,28.935841,3.785,5.247,0.1093,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2814.230469,4.981,6.906,0.1439,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.392096,34.915,385.363,8.0284,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,42.462208,41.268,455.481,9.4892,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,13.101344,2.09,11.541,0.2404,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,1348.157104,2.6,14.356,0.2991,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,13.013568,3.156,11.627,0.2422,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,1347.359375,3.902,14.374,0.2995,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,12.9944,4.214,11.652,0.2428,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1347.975342,5.2,14.378,0.2995,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.277056,24.707,545.186,11.3581,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,27.491327,31.871,703.278,14.6516,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.841728,1.066,11.766,0.2451,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,657.905518,2.664,29.397,0.6124,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,12.82224,1.602,11.788,0.2456,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,657.248169,3.999,29.437,0.6133,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.814048,2.137,11.8,0.2458,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,656.560425,5.338,29.478,0.6141,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.161504,21.192,935.092,19.4811,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.892832,29.416,1297.987,27.0414,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.773504,0.536,11.825,0.2464,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,332.638824,2.634,58.123,1.2109,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.760416,0.805,11.839,0.2467,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,332.633331,3.951,58.134,1.2111,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.742208,1.074,11.858,0.247,20,True,,NVIDIA H200,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,332.625031,5.268,58.146,1.2114,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,0.680704,40.224,222.129,4.6277,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,80.247169,43.674,241.181,5.0246,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,28.816288,1.9,5.254,0.1095,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,3916.308594,1.79,4.949,0.1031,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,31.733408,2.588,4.778,0.0995,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,4108.262695,2.559,4.724,0.0984,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,30.155935,3.632,5.035,0.1049,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,4119.929688,3.403,4.717,0.0983,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.392864,34.847,384.61,8.0127,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,53.684032,32.642,360.27,7.5056,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,12.98304,2.109,11.646,0.2426,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,2089.413818,1.677,9.263,0.193,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,13.559232,3.029,11.159,0.2325,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,2093.641602,2.511,9.251,0.1927,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,13.494464,4.058,11.22,0.2338,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,2111.4104,3.32,9.179,0.1912,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.281728,24.297,536.145,11.1697,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,28.809471,30.413,671.1,13.9813,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,12.695776,1.078,11.902,0.2479,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,1039.270996,1.686,18.61,0.3877,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,13.11552,1.566,11.525,0.2401,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,1054.755249,2.492,18.343,0.3821,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,12.654944,2.164,11.948,0.2489,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,1075.122437,3.26,18.002,0.375,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.164672,20.784,917.102,19.1063,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,14.805216,29.59,1305.668,27.2014,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,12.627136,0.542,11.962,0.2492,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,522.705444,1.676,36.988,0.7706,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,12.608736,0.814,11.982,0.2496,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,532.256775,2.469,36.331,0.7569,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,12.588064,1.088,12.003,0.2501,20,True,,NVIDIA H200,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,541.935791,3.233,35.688,0.7435,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_context.png b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_context.png old mode 100644 new mode 100755 diff --git a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png old mode 100644 new mode 100755 index dd4ec943f..141fb53d6 Binary files a/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/h200/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260818_143146.csv b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260819_070326.csv similarity index 53% rename from benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260818_143146.csv rename to benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260819_070326.csv index 6d62c5a82..170edfb59 100644 --- a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260818_143146.csv +++ b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_20260819_070326.csv @@ -1,33 +1,33 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,1.144512,23.923,132.112,7.3723,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,119.706367,29.277,161.68,9.0223,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,27.3848,2.0,5.529,0.3085,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,1917.439697,3.656,10.108,0.564,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,27.369761,3.001,5.54,0.3091,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,2022.694336,5.198,9.595,0.5354,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,27.380352,4.0,5.545,0.3094,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,2073.32373,6.761,9.374,0.5231,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.562752,24.327,268.501,14.9833,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,62.271042,28.141,310.589,17.332,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,15.601344,1.755,9.692,0.5408,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,963.890137,3.636,20.079,1.1205,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,15.604832,2.632,9.696,0.5411,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,981.849365,5.354,19.725,1.1008,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,15.61264,3.507,9.698,0.5412,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,1006.457642,6.964,19.256,1.0746,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.36672,18.666,411.887,22.9848,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,41.286304,21.222,468.292,26.1324,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,13.09776,1.045,11.536,0.6438,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,483.153564,3.627,40.03,2.2338,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,13.100096,1.568,11.538,0.6439,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,486.625458,5.402,39.758,2.2187,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,13.098656,2.09,11.543,0.6442,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,489.718292,7.157,39.521,2.2054,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.252832,13.537,597.318,33.3325,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,21.435328,20.438,901.815,50.3245,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,13.18128,0.519,11.459,0.6395,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,243.404739,3.6,79.432,4.4326,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,13.18144,0.779,11.461,0.6396,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,244.031036,5.386,79.241,4.422,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,13.181248,1.039,11.463,0.6397,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,245.65271,7.133,78.732,4.3935,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,1,131072,96,1,576,512,mla_absorbed,,32,1.12624,24.311,134.255,7.4919,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,1,131072,96,1,576,512,mla_absorbed,,32,119.661949,29.288,161.74,9.0257,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,2,131072,96,1,576,512,mla_absorbed,,32,27.316671,2.005,5.543,0.3093,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,2,131072,96,1,576,512,mla_absorbed,,32,1913.762329,3.663,10.127,0.5651,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,3,131072,96,1,576,512,mla_absorbed,,32,27.343519,3.004,5.545,0.3094,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,3,131072,96,1,576,512,mla_absorbed,,32,1915.981445,5.488,10.129,0.5653,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,1,4,131072,96,1,576,512,mla_absorbed,,32,27.361183,4.003,5.549,0.3097,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3,generation,cudnn,bfloat16,bfloat16,128,4,131072,96,1,576,512,mla_absorbed,,32,1917.9823,7.309,10.133,0.5654,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,48,1,576,512,mla_absorbed,,32,0.570208,24.009,264.99,14.7874,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,48,1,576,512,mla_absorbed,,32,62.220417,28.164,310.842,17.3461,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,48,1,576,512,mla_absorbed,,32,15.59088,1.756,9.698,0.5412,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,48,1,576,512,mla_absorbed,,32,955.971924,3.666,20.245,1.1298,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,48,1,576,512,mla_absorbed,,32,15.59232,2.634,9.704,0.5415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,48,1,576,512,mla_absorbed,,32,957.051941,5.493,20.237,1.1293,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,48,1,576,512,mla_absorbed,,32,15.596672,3.511,9.708,0.5417,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,48,1,576,512,mla_absorbed,,32,958.145081,7.316,20.227,1.1288,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,24,1,576,512,mla_absorbed,,32,0.366784,18.662,411.815,22.9807,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,24,1,576,512,mla_absorbed,,32,41.300991,21.214,468.125,26.1231,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,24,1,576,512,mla_absorbed,,32,13.077632,1.047,11.554,0.6448,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,24,1,576,512,mla_absorbed,,32,481.625519,3.638,40.157,2.2409,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,24,1,576,512,mla_absorbed,,32,13.077664,1.57,11.558,0.645,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,24,1,576,512,mla_absorbed,,32,482.275085,5.45,40.117,2.2387,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,24,1,576,512,mla_absorbed,,32,13.068832,2.095,11.57,0.6456,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,24,1,576,512,mla_absorbed,,32,482.830627,7.259,40.085,2.2369,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,12,1,576,512,mla_absorbed,,32,0.25456,13.445,593.263,33.1062,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,12,1,576,512,mla_absorbed,,32,21.409632,20.462,902.897,50.3849,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,12,1,576,512,mla_absorbed,,32,13.173472,0.52,11.466,0.6398,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,12,1,576,512,mla_absorbed,,32,243.045319,3.605,79.549,4.4391,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,12,1,576,512,mla_absorbed,,32,13.172064,0.78,11.469,0.64,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,12,1,576,512,mla_absorbed,,32,243.324219,5.401,79.472,4.4348,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,12,1,576,512,mla_absorbed,,32,13.172544,1.039,11.471,0.6401,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +kimi_k3,kimi_k3-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,12,1,576,512,mla_absorbed,,32,243.548187,7.195,79.412,4.4315,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_context.png b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_context.png old mode 100644 new mode 100755 diff --git a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png old mode 100644 new mode 100755 index f5d607584..278416369 Binary files a/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png and b/benchmark/attention_inference/results/kimi_k3/rtxpro6000/kimi_k3_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260818_130938.csv b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260819_053557.csv similarity index 60% rename from benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260818_130938.csv rename to benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260819_053557.csv index cbbfe40eb..314c6a6cf 100644 --- a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260818_130938.csv +++ b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_20260819_053557.csv @@ -1,148 +1,148 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.091392,751.92,826.084,10.3261,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.737408,1491.049,409.529,5.1191,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,12.67456,1387.992,95.306,1.1913,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.785952,1398.955,362.888,4.5361,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.472096,1493.804,376.095,4.7012,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.50656,1459.632,200.45,2.5056,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.061088,1436.759,186.347,2.3293,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.103648,41.438,5180.068,64.7508,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.100768,42.622,2664.058,33.3007,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,9.005376,61.048,7631.405,95.3926,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.19424,3.915,244.703,3.0588,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,1.262176,6.806,212.703,2.6588,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,83.90387,13.104,819.126,10.2391,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.193664,5.874,244.782,3.0598,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,1.25936,10.231,213.191,2.6649,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,83.903748,19.657,819.177,10.2397,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.193088,7.834,244.861,3.0608,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,1.26128,13.621,212.88,2.661,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,83.911263,26.207,819.154,10.2394,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.228768,300.389,330.018,4.1252,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.160672,947.306,260.185,3.2523,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,13.677056,1286.255,88.32,1.104,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.163296,945.169,245.176,3.0647,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,2.24048,981.497,247.111,3.0889,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.964544,1119.356,153.72,1.9215,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.844672,1143.933,148.368,1.8546,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.177376,1.973,246.583,3.0823,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,249.974686,2.199,274.923,3.4365,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.217312,3.874,242.156,3.027,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,251.968643,4.364,272.764,3.4095,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.213152,5.822,242.626,3.0328,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,251.305283,6.563,273.5,3.4188,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.217952,7.746,242.116,3.0265,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,251.351486,8.749,273.467,3.4183,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.059328,579.149,636.272,7.9534,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.400224,1373.62,377.276,4.716,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,6.34944,1385.334,95.123,1.189,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.72016,763.38,198.02,2.4753,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.399552,785.617,197.795,2.4724,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.768832,1430.106,196.395,2.4549,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.477248,1488.595,193.07,2.4134,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.068544,31.33,3916.489,48.9561,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.074208,28.939,1808.78,22.6097,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.52352,60.766,7596.26,94.9533,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.299808,119.522,7470.588,93.3823,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.19168,1.96,122.494,1.5312,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,1.2592,3.411,106.603,1.3325,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,42.249569,13.012,813.356,10.1669,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,33.183521,16.567,517.786,6.4723,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.189856,2.942,122.604,1.5325,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,1.25888,5.118,106.636,1.333,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,42.243103,19.521,813.53,10.1691,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,33.042271,24.957,520.031,6.5004,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.189824,3.923,122.613,1.5327,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,1.257984,6.828,106.719,1.334,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,42.243935,26.028,813.564,10.1695,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,33.588482,32.735,511.606,6.3951,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.182016,188.773,207.392,2.5924,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.6288,874.294,240.132,3.0016,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,6.754944,1302.171,89.413,1.1177,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,1.003616,547.775,142.093,1.7762,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.92688,570.618,143.664,1.7958,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.012608,1085.822,149.115,1.8639,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.940768,1133.069,146.959,1.837,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.883136,1.14,142.556,1.7819,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,124.00576,2.217,277.099,3.4637,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.918592,2.239,139.93,1.7491,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,125.102013,4.394,274.687,3.4336,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.923968,3.349,139.547,1.7443,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,125.131844,6.59,274.639,3.433,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.92096,4.472,139.774,1.7472,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,124.894142,8.804,275.178,3.4397,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.057728,297.6,326.953,4.0869,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.21776,1262.298,346.7,4.3338,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.577792,1706.129,117.151,1.4644,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.720128,381.707,99.015,1.2377,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.399456,392.835,98.904,1.2363,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.72336,760.003,104.371,1.3046,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.400928,784.845,101.794,1.2724,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.0504,21.304,2663.213,33.2902,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.06464,16.611,1038.257,12.9782,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.2696,60.556,7570.02,94.6253,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.18176,116.3,7269.208,90.8651,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.18512,0.983,61.431,0.7679,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,1.24912,1.719,53.731,0.6716,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.793184,12.613,788.41,9.8551,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,16.913248,16.252,507.944,6.3493,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.185184,1.474,61.433,0.7679,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,1.249856,2.577,53.703,0.6713,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.790815,18.922,788.544,9.8568,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,16.964737,24.304,506.433,6.3304,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.18528,1.965,61.434,0.7679,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,1.248032,3.441,53.785,0.6723,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.797825,25.221,788.338,9.8542,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,16.923391,32.485,507.702,6.3463,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.16032,107.16,117.729,1.4716,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.360448,762.601,209.455,2.6182,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,3.159648,1391.942,95.577,1.1947,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.925152,297.116,77.072,0.9634,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.774784,309.759,77.988,0.9749,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.924384,594.727,81.673,1.0209,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.779392,617.914,80.143,1.0018,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.732864,0.62,77.459,0.9682,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,61.714687,2.227,278.393,3.4799,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.772032,1.212,75.752,0.9469,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,62.1264,4.424,276.565,3.4571,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.776256,1.813,75.576,0.9447,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,62.130783,6.636,276.562,3.457,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.771104,2.425,75.8,0.9475,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,62.157505,8.845,276.46,3.4557,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.054368,157.996,173.58,2.1697,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.132384,1038.184,285.146,3.5643,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.33136,1651.712,113.414,1.4177,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.717472,191.56,49.691,0.6211,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.39296,197.334,49.683,0.621,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.720352,381.588,52.403,0.655,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.399456,392.835,50.951,0.6369,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.049376,10.873,1359.222,16.9903,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.063168,8.499,531.226,6.6403,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.140992,60.228,7528.938,94.1117,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.862496,0.577,36.036,0.4505,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,1.248128,0.86,26.887,0.3361,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.366848,12.091,755.793,9.4474,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,8.52272,16.126,504.005,6.3001,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.8616,0.865,36.056,0.4507,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,1.248128,1.29,26.889,0.3361,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.368832,18.134,755.707,9.4463,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,8.495584,24.267,505.646,6.3206,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.860736,1.154,36.075,0.4509,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,1.249984,1.718,26.85,0.3356,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.366368,24.183,755.917,9.449,20,True,,NVIDIA GB200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,8.496128,32.353,505.644,6.3205,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.126592,67.855,74.548,0.9319,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.270112,508.822,139.752,1.7469,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.594144,1379.438,94.719,1.184,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.86864,158.223,41.043,0.513,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.648992,166.695,41.969,0.5246,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.87184,315.285,43.298,0.5412,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.647808,333.629,43.272,0.5409,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.580672,0.34,42.458,0.5307,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,21.524832,3.193,399.095,4.9887,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.649952,0.651,40.678,0.5085,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,21.807648,6.302,393.944,4.9243,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.64496,0.979,40.804,0.5101,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,21.811808,9.452,393.893,4.9237,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.647104,1.304,40.753,0.5094,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,21.815424,12.6,393.851,4.9231,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.09648,712.267,782.519,9.7815,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.75744,1451.616,398.698,4.9837,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,11.916,1476.35,101.373,1.2672,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.754816,1456.662,377.857,4.7232,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.403264,1567.077,394.543,4.9318,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.470016,1495.918,205.433,2.5679,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,2.849952,1543.2,200.153,2.5019,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.103584,41.464,5183.269,64.7909,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.099456,43.185,2699.202,33.74,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,8.922688,61.613,7702.127,96.2766,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.210016,3.887,242.956,3.0369,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,1.261952,6.807,212.74,2.6593,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,84.338844,13.037,814.902,10.1863,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.210016,5.83,242.971,3.0371,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,1.262528,10.206,212.656,2.6582,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,84.329948,19.557,815.037,10.188,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.207936,7.781,243.214,3.0402,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,1.262176,13.611,212.729,2.6591,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,84.35453,26.069,814.85,10.1856,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.226304,303.66,333.611,4.1701,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.167552,941.724,258.652,3.2332,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,13.050848,1347.973,92.558,1.157,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.173568,936.896,243.03,3.0379,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,2.258496,973.667,245.14,3.0643,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.975168,1113.335,152.893,1.9112,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.840672,1145.124,148.522,1.8565,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.19984,1.952,244.065,3.0508,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,248.644196,2.211,276.394,3.4549,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.23712,3.84,240.012,3.0002,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,250.610489,4.387,274.242,3.428,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.239552,5.753,239.766,2.9971,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,250.605057,6.581,274.264,3.4283,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.228992,7.707,240.917,3.0115,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,250.700989,8.771,274.176,3.4272,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.068896,498.719,547.909,6.8489,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.418432,1313.847,360.859,4.5107,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,5.864704,1499.836,102.986,1.2873,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.720896,762.601,197.818,2.4727,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.375136,799.566,201.307,2.5163,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.747168,1471.572,202.09,2.5261,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.45312,1513.312,196.276,2.4535,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.069664,30.826,3853.523,48.169,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.07632,28.138,1758.725,21.9841,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.485568,61.281,7660.532,95.7566,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.283168,120.393,7525.034,94.0629,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.209056,1.944,121.531,1.5191,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,1.260352,3.408,106.505,1.3313,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,42.363457,12.977,811.169,10.1396,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,32.85968,16.73,522.889,6.5361,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.205408,2.921,121.739,1.5217,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,1.264032,5.097,106.202,1.3275,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,42.388863,19.454,810.733,10.1342,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,32.891041,25.072,522.422,6.5303,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.207808,3.891,121.614,1.5202,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,1.2624,6.804,106.345,1.3293,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,42.401344,25.931,810.543,10.1318,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,32.857185,33.463,522.993,6.5374,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.183584,187.161,205.621,2.5703,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.626976,876.837,240.83,3.0104,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,6.7016,1312.536,90.125,1.1266,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,1.006752,546.069,141.65,1.7706,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.937024,567.629,142.912,1.7864,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.016,1082.196,148.617,1.8577,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.9504,1127.473,146.233,1.8279,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.893856,1.134,141.749,1.7719,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,124.524063,2.207,275.945,3.4493,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.928608,2.227,139.203,1.74,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,125.620255,4.376,273.554,3.4194,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.936832,3.326,138.62,1.7328,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,125.605507,6.565,273.603,3.42,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.93072,4.449,139.068,1.7383,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,125.618019,8.753,273.592,3.4199,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.057312,299.76,329.327,4.1166,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.2512,1094.259,300.547,3.7568,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.686816,1636.899,112.397,1.405,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.707968,388.263,100.715,1.2589,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.369504,401.427,101.067,1.2633,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.717792,765.899,105.18,1.3148,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.379936,796.785,103.343,1.2918,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.052544,20.435,2554.543,31.9318,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.064128,16.744,1046.547,13.0818,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.249024,61.11,7639.277,95.491,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.170112,117.458,7341.57,91.7696,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.193568,0.979,61.194,0.7649,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,1.253248,1.714,53.554,0.6694,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.823008,12.596,787.333,9.8417,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,16.941792,16.225,507.088,6.3386,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.19408,1.468,61.184,0.7648,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,1.251232,2.574,53.644,0.6706,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.819263,18.897,787.516,9.8439,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,16.962433,24.308,506.502,6.3313,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.194368,1.957,61.18,0.7647,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,1.2504,3.435,53.683,0.671,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.814367,25.202,787.741,9.8468,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,16.94368,32.446,507.094,6.3387,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.15728,109.231,120.005,1.5001,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.362112,759.096,208.492,2.6062,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,3.192224,1377.737,94.602,1.1825,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.929728,295.654,76.693,0.9587,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.784032,308.154,77.584,0.9698,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.928256,592.246,81.333,1.0167,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.788,614.939,79.757,0.997,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.736992,0.618,77.275,0.9659,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,62.27824,2.207,275.874,3.4484,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.776192,1.209,75.574,0.9447,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,62.865696,4.372,273.312,3.4164,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.777152,1.813,75.538,0.9442,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,62.856255,6.56,273.37,3.4171,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.774752,2.42,75.645,0.9456,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,62.835838,8.749,273.476,3.4184,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.056288,152.607,167.659,2.0957,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.16352,840.502,230.851,2.8856,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.495552,1470.376,100.963,1.262,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.702144,195.742,50.775,0.6347,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.36352,201.594,50.755,0.6344,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.70928,387.545,53.221,0.6653,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.374016,400.109,51.894,0.6487,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.049824,10.775,1347.001,16.8375,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.062112,8.644,540.258,6.7532,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.135904,60.498,7562.663,94.5333,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.861248,0.577,36.06,0.4508,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,1.250816,0.858,26.829,0.3354,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.40208,12.054,753.458,9.4182,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,8.510592,16.149,504.723,6.309,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.863136,0.864,36.026,0.4503,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,1.252544,1.286,26.794,0.3349,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.409408,18.069,753.02,9.4127,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,8.514144,24.214,504.543,6.3068,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.864672,1.152,35.998,0.45,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,1.244256,1.726,26.974,0.3372,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.402304,24.107,753.535,9.4192,20,True,,NVIDIA GB200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,8.513312,32.288,504.623,6.3078,20,True,,NVIDIA GB200,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.12544,68.478,75.233,0.9404,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.268704,511.488,140.484,1.7561,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.5912,1381.99,94.894,1.1862,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.864864,158.914,41.222,0.5153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.643104,167.292,42.119,0.5265,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.871488,315.412,43.315,0.5414,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.650592,333.066,43.199,0.54,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.58288,0.339,42.399,0.53,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,21.464384,3.202,400.219,5.0027,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.642112,0.654,40.872,0.5109,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,21.783808,6.309,394.375,4.9297,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.642112,0.981,40.875,0.5109,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,21.788095,9.462,394.321,4.929,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.64144,1.308,40.894,0.5112,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,21.795551,12.612,394.21,4.9276,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png old mode 100644 new mode 100755 index 1594aae43..6da03a713 Binary files a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png old mode 100644 new mode 100755 index a3c8d5961..f9d69e0e5 Binary files a/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/gb200/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260818_131600.csv b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260819_053435.csv similarity index 62% rename from benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260818_131600.csv rename to benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260819_053435.csv index 7ae74dcf5..ee4f33ec4 100644 --- a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260818_131600.csv +++ b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_20260819_053435.csv @@ -1,151 +1,151 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.077664,884.83,972.104,12.1513,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.590272,1862.72,511.611,6.3951,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,9.369152,1877.671,128.929,1.6116,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.622432,1766.477,458.223,5.7278,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.207136,1821.686,458.646,5.7331,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.22368,1797.057,246.788,3.0849,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,2.418784,1818.288,235.831,2.9479,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.10592,40.549,5068.955,63.3619,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.095712,44.874,2804.788,35.0598,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,9.127456,60.231,7529.335,94.1167,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.187392,3.927,245.469,3.0684,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,0.961824,8.931,279.124,3.4891,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,83.423615,13.18,823.842,10.298,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,64,8,128,128,gqa,,16,50.839554,21.627,675.929,8.4491,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.189824,5.884,245.211,3.0651,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,0.964416,13.36,278.391,3.4799,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,83.422531,19.77,823.903,10.2988,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,64,8,128,128,gqa,,16,50.842335,32.439,675.933,8.4492,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.188608,7.85,245.362,3.067,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,0.963104,17.838,278.787,3.4848,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,83.419838,26.361,823.98,10.2997,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,64,8,128,128,gqa,,16,50.845886,43.249,675.927,8.4491,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.22064,311.455,342.175,4.2772,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.033216,1064.164,292.281,3.6535,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,10.947488,1606.961,110.341,1.3793,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.035072,1062.256,275.549,3.4444,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.944384,1130.961,284.742,3.5593,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.694624,1297.647,178.205,2.2276,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.246784,1354.586,175.689,2.1961,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,1.859936,2.309,288.668,3.6083,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,216.288773,2.542,317.74,3.9718,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.912096,4.492,280.81,3.5101,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,216.997986,5.067,316.721,3.959,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.913824,6.733,280.574,3.5072,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,216.990952,7.601,316.751,3.9594,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.905888,9.014,281.759,3.522,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,216.979782,10.135,316.786,3.9598,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.054112,634.974,697.604,8.72,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.31072,1769.297,485.952,6.0744,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,4.223392,2082.708,143.008,1.7876,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.604288,909.758,235.991,2.9499,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.168096,941.285,236.987,2.9623,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.61184,1797.057,246.788,3.0849,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.195872,1838.845,238.498,2.9812,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.069632,30.84,3855.294,48.1912,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.069184,31.04,1940.13,24.2516,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.643264,59.199,7400.363,92.5045,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.361984,116.376,7273.935,90.9242,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.1904,1.961,122.566,1.5321,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,0.9616,4.466,139.595,1.7449,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,41.936031,13.109,819.437,10.243,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,25.126623,21.879,683.815,8.5477,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.190464,2.941,122.57,1.5321,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,0.961984,6.697,139.547,1.7443,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,41.910271,19.676,819.991,10.2499,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,25.160513,32.775,682.936,8.5367,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.189536,3.923,122.629,1.5329,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,0.96432,8.908,139.218,1.7402,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,41.934048,26.22,819.576,10.2447,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,25.169888,43.684,682.723,8.534,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.184672,186.058,204.41,2.5551,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.550048,999.469,274.512,3.4314,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,5.415552,1624.228,111.527,1.3941,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.869312,632.403,164.045,2.0506,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.6248,676.706,170.374,2.1297,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.877216,1253.41,172.13,2.1516,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.648896,1333.634,172.972,2.1621,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.566752,1.371,171.343,2.1418,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,107.05027,2.568,320.988,4.0123,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.617856,2.655,165.941,2.0743,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,107.254944,5.126,320.395,4.0049,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.617536,3.983,165.984,2.0748,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,107.606239,7.663,319.368,3.9921,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.618176,5.308,165.928,2.0741,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,107.711105,10.208,319.077,3.9885,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.051968,330.586,363.192,4.5399,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.169248,1624.113,446.076,5.576,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.124064,2070.581,142.176,1.7772,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.603744,455.289,118.102,1.4763,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.167264,470.978,118.578,1.4822,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.603168,911.447,125.168,1.5646,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.167296,941.93,122.168,1.5271,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.05312,20.214,2526.843,31.5855,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.063104,17.015,1063.529,13.2941,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.336768,58.816,7352.428,91.9053,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.233632,111.41,6963.551,87.0444,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.184128,0.983,61.459,0.7682,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,0.95424,2.25,70.336,0.8792,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.855232,12.577,786.172,9.8271,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,12.845792,21.398,668.778,8.3597,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.18528,1.474,61.43,0.7679,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,0.961792,3.349,69.788,0.8723,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.860704,18.861,786.023,9.8253,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,12.84672,32.095,668.771,8.3596,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.182464,1.968,61.513,0.7689,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,0.955072,4.497,70.283,0.8785,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.860256,25.149,786.087,9.8261,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,12.848704,42.787,668.708,8.3589,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.154144,111.453,122.446,1.5306,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.325088,845.549,232.237,2.903,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.664896,1650.363,113.321,1.4165,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.793568,346.382,89.851,1.1231,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.4744,372.867,93.877,1.1735,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.798272,688.682,94.576,1.1822,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.472416,746.74,96.852,1.2106,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.418304,0.757,94.638,1.183,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,53.024418,2.592,324.019,4.0502,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.473504,1.457,91.099,1.1387,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,53.762943,5.113,319.588,3.9948,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.470816,2.19,91.271,1.1409,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,53.738689,7.673,319.751,3.9969,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.470752,2.92,91.28,1.141,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,53.768318,10.225,319.595,3.9949,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.049696,172.85,189.898,2.3737,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.104832,1311.04,360.088,4.5011,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.088288,2020.626,138.745,1.7343,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.601536,228.48,59.268,0.7408,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.163936,236.162,59.459,0.7432,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.601856,456.717,62.721,0.784,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.165728,471.599,61.166,0.7646,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.046208,11.619,1452.41,18.1551,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.0624,8.604,537.764,6.7221,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.225472,56.076,7009.919,87.624,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.827584,0.588,36.724,0.4591,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,0.95456,1.125,35.156,0.4395,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.2064,12.264,766.614,9.5827,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,6.447648,21.316,666.211,8.3276,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.825568,0.882,36.767,0.4596,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,0.95536,1.686,35.129,0.4391,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.207008,18.395,766.619,9.5827,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,6.448832,31.968,666.129,8.3266,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.8256,1.176,36.769,0.4596,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,0.955968,2.246,35.109,0.4389,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.206304,24.529,766.714,9.5839,20,True,,NVIDIA GB300,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,6.448768,42.625,666.176,8.3272,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.125728,68.322,75.06,0.9383,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.236384,581.422,159.692,1.9962,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.349664,1629.312,111.876,1.3984,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.736448,186.624,48.41,0.6051,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.327008,207.141,52.152,0.6519,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.744832,369.047,50.681,0.6335,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.349024,407.521,52.855,0.6607,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.270112,0.423,52.84,0.6605,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,17.482656,3.931,491.37,6.1421,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.316992,0.815,50.962,0.637,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,17.612288,7.804,487.783,6.0973,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.322272,1.218,50.762,0.6345,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,17.58032,11.727,488.7,6.1088,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.322624,1.624,50.752,0.6344,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,17.576736,15.639,488.83,6.1104,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.083104,826.909,908.47,11.3559,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,0.617184,1781.497,489.303,6.1163,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,9.526112,1846.733,126.805,1.5851,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,0.617088,1781.774,462.191,5.7774,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.195264,1839.78,463.202,5.79,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.215744,1808.788,248.399,3.105,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,2.443008,1800.259,233.493,2.9187,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.106752,40.233,5029.448,62.8681,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,64,8,128,128,gqa,,16,0.095616,44.919,2807.604,35.095,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,9.173312,59.93,7491.697,93.6462,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.186944,3.928,245.519,3.069,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,0.96752,8.878,277.481,3.4685,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,83.289436,13.201,825.169,10.3146,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,64,8,128,128,gqa,,16,50.636673,21.714,678.637,8.483,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.189024,5.886,245.301,3.0663,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,0.9656,13.344,278.05,3.4756,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,83.298592,19.799,825.129,10.3141,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,64,8,128,128,gqa,,16,50.979713,32.351,674.112,8.4264,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.188928,7.849,245.326,3.0666,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,0.965408,17.795,278.122,3.4765,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,83.315613,26.394,825.01,10.3126,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,64,8,128,128,gqa,,16,50.694847,43.378,677.941,8.4743,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.217952,315.296,346.395,4.3299,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.031328,1066.112,292.817,3.6602,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,10.86432,1619.263,111.186,1.3898,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.0192,1078.799,279.84,3.498,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,1.92176,1144.276,288.094,3.6012,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,1.688768,1302.146,178.823,2.2353,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,3.23856,1358.025,176.135,2.2017,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,1.855776,2.314,289.315,3.6164,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,215.184189,2.555,319.371,3.9921,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.904768,4.51,281.891,3.5236,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,216.191803,5.086,317.902,3.9738,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.906656,6.758,281.629,3.5204,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,216.08905,7.632,318.073,3.9759,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.89936,9.045,282.728,3.5341,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,215.932632,10.184,318.323,3.979,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.063264,543.117,596.686,7.4586,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.335872,1636.802,449.561,5.6195,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,4.573312,1923.353,132.066,1.6508,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.6032,911.399,236.416,2.9552,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.16608,942.913,237.397,2.9675,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.613536,1792.09,246.106,3.0763,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.18896,1849.535,239.884,2.9986,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.0712,30.161,3770.391,47.1299,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,32,4,128,128,gqa,,16,0.073632,29.165,1822.929,22.7866,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,4.616288,59.545,7443.607,93.0451,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,32,4,128,128,gqa,,16,2.356576,116.643,7290.628,91.1328,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.187968,1.963,122.702,1.5338,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,0.962368,4.463,139.483,1.7435,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,41.872864,13.129,820.673,10.2584,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,32,4,128,128,gqa,,16,24.988928,22.0,687.583,8.5948,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.189536,2.942,122.622,1.5328,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,0.96256,6.693,139.464,1.7433,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,41.867359,19.696,820.831,10.2604,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,32,4,128,128,gqa,,16,25.081247,32.878,685.094,8.5637,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.187616,3.927,122.737,1.5342,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,0.96208,8.929,139.542,1.7443,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,41.869663,26.26,820.836,10.2605,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,32,4,128,128,gqa,,16,25.063072,43.87,685.633,8.5704,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.173696,197.815,217.326,2.7166,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.548416,1002.443,275.329,3.4416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,5.423424,1621.871,111.365,1.3921,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.8608,638.657,165.667,2.0708,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.621792,677.961,170.69,2.1336,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,0.879456,1250.218,171.691,2.1461,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,1.62784,1350.884,175.209,2.1901,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,1.565248,1.372,171.508,2.1438,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,106.57901,2.579,322.407,4.0301,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.618656,2.653,165.859,2.0732,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,107.219681,5.127,320.5,4.0063,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.61584,3.987,166.158,2.077,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,107.010818,7.706,321.145,4.0143,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.615104,5.319,166.244,2.078,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,107.176094,10.259,320.67,4.0084,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.053184,323.027,354.888,4.4361,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.200288,1372.413,376.945,4.7118,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.257664,1948.052,133.762,1.672,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.601792,456.766,118.485,1.4811,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.163616,472.455,118.95,1.4869,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.602816,911.979,125.241,1.5655,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.168768,940.744,122.014,1.5252,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.051776,20.738,2592.435,32.4054,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,16,2,128,128,gqa,,16,0.058816,18.256,1141.066,14.2633,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,2.312544,59.432,7429.444,92.8681,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,1,131072,16,2,128,128,gqa,,16,1.20496,114.061,7129.248,89.1156,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.185088,0.983,61.432,0.7679,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,0.955968,2.246,70.208,0.8776,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,21.792896,12.613,788.421,9.8553,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,12.837312,21.412,669.22,8.3652,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.184,1.475,61.466,0.7683,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,0.956448,3.368,70.178,0.8772,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,21.796864,18.916,788.325,9.8541,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,12.833824,32.127,669.443,8.368,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.184992,1.966,61.442,0.768,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,0.955808,4.494,70.229,0.8779,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,21.792255,25.227,788.54,9.8567,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,12.837024,42.826,669.316,8.3665,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.1528,112.434,123.523,1.544,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.323648,849.311,233.27,2.9159,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,2.713248,1620.953,111.302,1.3913,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.79872,344.148,89.272,1.1159,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.474784,372.77,93.852,1.1732,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.797728,689.152,94.641,1.183,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.475872,744.991,96.625,1.2078,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,1.415328,0.759,94.837,1.1855,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,53.005569,2.593,324.134,4.0517,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.467328,1.464,91.482,1.1435,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,53.606625,5.128,320.519,4.0065,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.465984,2.197,91.571,1.1446,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,53.717888,7.676,319.875,3.9984,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.472512,2.917,91.171,1.1396,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,53.675838,10.242,320.145,4.0018,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.052128,164.785,181.039,2.263,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.130848,1050.371,288.493,3.6062,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.209024,1818.842,124.89,1.5611,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.600736,228.784,59.347,0.7418,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.164,236.149,59.455,0.7432,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.60224,456.426,62.681,0.7835,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.166368,471.34,61.133,0.7642,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.048128,11.155,1394.468,17.4309,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,1,131072,8,1,128,128,gqa,,16,0.06128,8.761,547.593,6.8449,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.169664,58.751,7344.382,91.8048,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.82672,0.588,36.742,0.4593,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,0.961376,1.117,34.907,0.4363,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,11.121056,12.358,772.497,9.6562,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,6.438592,21.346,667.148,8.3393,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.82784,0.881,36.722,0.459,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,0.954656,1.687,35.155,0.4394,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,11.122464,18.535,772.446,9.6556,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,6.445504,31.985,666.473,8.3309,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.830304,1.173,36.674,0.4584,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,0.957376,2.243,35.057,0.4382,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,11.119712,24.72,772.685,9.6586,20,True,,NVIDIA GB300,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,6.448192,42.629,666.236,8.3279,20,True,,NVIDIA GB300,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.120096,71.526,78.58,0.9823,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.22368,614.445,168.762,2.1095,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,1.344416,1635.672,112.313,1.4039,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.729376,188.434,48.88,0.611,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.329952,206.683,52.036,0.6505,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.733664,374.665,51.452,0.6432,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.337856,410.923,53.297,0.6662,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,1.264672,0.425,53.067,0.6633,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,17.3736,3.955,494.455,6.1807,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.32096,0.813,50.809,0.6351,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,17.483809,7.861,491.368,6.1421,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.313184,1.226,51.113,0.6389,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,17.563135,11.738,489.178,6.1147,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.323712,1.622,50.71,0.6339,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,17.510336,15.698,490.683,6.1335,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png old mode 100644 new mode 100755 index 42a5c7b90..ce15ae8a7 Binary files a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png old mode 100644 new mode 100755 index e16c409ac..d0e9a5db5 Binary files a/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/gb300/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260818_142507.csv b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260819_071907.csv similarity index 55% rename from benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260818_142507.csv rename to benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260819_071907.csv index 5c511421c..afdc98c60 100644 --- a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260818_142507.csv +++ b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_20260819_071907.csv @@ -1,61 +1,61 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.16608,413.773,454.585,9.4705,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.724768,637.484,175.09,3.6477,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,28.892481,608.885,41.809,0.871,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.783648,616.44,159.904,3.3313,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,3.640896,603.979,152.064,3.168,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,3.52144,624.467,85.757,1.7866,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,7.762688,566.562,73.483,1.5309,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.153952,27.898,3487.474,72.6557,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,15.240672,36.072,4509.228,93.9423,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.542688,5.568,348.053,7.2511,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,103.439552,10.63,664.425,13.8422,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.526528,8.441,351.759,7.3283,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,103.702019,15.904,662.784,13.808,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.527232,11.249,351.618,7.3254,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,103.984482,21.148,661.024,13.7713,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.104352,329.268,361.744,7.5363,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.907008,606.12,166.476,3.4682,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,14.58096,603.259,41.422,0.863,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.900704,610.362,158.328,3.2985,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.767264,622.155,156.64,3.2633,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.760192,624.654,85.783,1.7872,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,3.5184,625.007,81.063,1.6888,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.093184,23.046,2880.879,60.0183,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,7.614496,36.099,4512.687,94.0143,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.54016,2.789,174.312,3.6315,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,52.070625,10.558,659.949,13.7489,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.524512,4.226,176.112,3.669,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,52.180607,15.803,658.598,13.7208,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.539936,5.578,174.359,3.6325,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,52.198017,21.064,658.418,13.717,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.07616,225.576,247.825,5.163,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.500704,548.983,150.783,3.1413,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,7.35264,598.159,41.072,0.8557,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.90112,305.04,79.127,1.6485,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.765792,311.337,78.385,1.633,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.915264,600.653,82.487,1.7185,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.785664,615.744,79.862,1.6638,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.06352,16.904,2113.128,44.0235,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,3.778304,36.376,4547.256,94.7345,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.54112,1.393,87.102,1.8146,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,25.687649,10.701,668.88,13.935,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.541408,2.09,87.091,1.8144,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,25.783968,15.991,666.422,13.8838,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.54016,2.789,87.167,1.816,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,25.781216,21.324,666.534,13.8861,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.055776,154.008,169.198,3.525,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.29856,460.339,126.436,2.6341,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,3.480672,631.781,43.381,0.9038,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.897824,153.08,39.709,0.8273,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.762976,155.917,39.255,0.8178,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.90112,305.04,41.891,0.8727,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.768384,310.88,40.321,0.84,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.053952,9.951,1243.938,25.9154,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.901536,36.139,4517.642,94.1175,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.537184,0.699,43.662,0.9096,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,12.841184,10.703,669.018,13.9379,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.523136,1.057,44.068,0.9181,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,12.918528,15.958,665.053,13.8553,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.534816,1.399,43.735,0.9111,20,True,,NVIDIA H200,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,12.855424,21.382,668.358,13.9241,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.16448,417.798,459.007,9.5626,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,1.77488,619.485,170.147,3.5447,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,46.987839,374.399,25.708,0.5356,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,1.829984,600.831,155.855,3.247,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,3.979488,552.59,139.125,2.8984,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,4.010144,548.365,75.306,1.5689,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,8.832928,497.915,64.579,1.3454,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.152928,28.085,3510.827,73.1422,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,1,131072,64,8,128,128,gqa,,16,15.806944,34.779,4347.689,90.5768,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,1.519744,5.652,353.307,7.3606,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,2,131072,64,8,128,128,gqa,,16,268.220642,4.099,256.236,5.3383,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,1.518144,8.487,353.701,7.3688,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,3,131072,64,8,128,128,gqa,,16,296.149078,5.569,232.086,4.8351,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,1.519488,11.306,353.41,7.3627,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,128,4,131072,64,8,128,128,gqa,,16,296.260284,7.423,232.013,4.8336,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.103296,332.634,365.442,7.6134,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,0.927104,592.982,162.867,3.3931,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,19.981377,440.215,30.227,0.6297,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,0.918496,598.539,155.261,3.2346,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,1.8168,605.191,152.369,3.1744,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,1.811808,606.859,83.339,1.7362,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,3.964608,554.663,71.94,1.4987,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.094816,22.649,2831.293,58.9853,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,7.628288,36.034,4504.528,93.8443,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,1.518304,2.829,176.821,3.6838,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,80.140511,6.86,428.796,8.9333,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,1.517344,4.246,176.944,3.6863,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,95.804222,8.607,358.711,7.4731,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,1.517376,5.661,176.951,3.6865,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,98.318817,11.183,349.558,7.2825,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.075936,226.241,248.556,5.1783,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.505952,543.289,149.219,3.1087,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,7.566496,581.253,39.911,0.8315,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,0.884832,310.655,80.584,1.6788,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,1.735968,316.685,79.732,1.6611,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,0.911264,603.289,82.849,1.726,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,1.797856,611.568,79.32,1.6525,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.067168,15.986,1998.361,41.6325,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,3.769792,36.458,4557.524,94.9484,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,1.527872,1.406,87.857,1.8304,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,31.745888,8.659,541.234,11.2757,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,1.531392,2.103,87.66,1.8263,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,33.760834,12.213,508.963,10.6034,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,1.517984,2.829,88.44,1.8425,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,34.641376,15.87,496.056,10.3345,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.055584,154.54,169.782,3.5371,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.295744,464.723,127.64,2.6592,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,3.516512,625.342,42.939,0.8946,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,0.89088,154.273,40.018,0.8337,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,1.749696,157.1,39.553,0.824,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,0.891872,308.203,42.325,0.8818,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,1.747872,314.529,40.794,0.8499,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.05424,9.898,1237.333,25.7778,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,1.90272,36.116,4514.831,94.059,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,1.52576,0.704,43.989,0.9164,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,13.273088,10.355,647.248,13.4843,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,1.523712,1.057,44.051,0.9177,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,13.310016,15.489,645.492,13.4477,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,1.532384,1.401,43.804,0.9126,20,True,,NVIDIA H200,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,13.366816,20.564,642.788,13.3914,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png old mode 100644 new mode 100755 index c8f4b4168..e4e0be164 Binary files a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png old mode 100644 new mode 100755 index d00dc7eaf..55625b7ad Binary files a/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/h200/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260818_144157.csv b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260819_071357.csv similarity index 64% rename from benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260818_144157.csv rename to benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260819_071357.csv index 9b3ad6031..db7229e42 100644 --- a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260818_144157.csv +++ b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_20260819_071357.csv @@ -1,131 +1,131 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.271264,253.331,278.317,15.5311,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.134464,350.781,96.345,5.3764,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,48.161983,365.271,25.081,1.3996,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,4.296704,255.897,66.379,3.7042,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,8.568,256.655,64.618,3.6059,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.418336,342.616,47.051,2.6256,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,12.81824,343.108,44.501,2.4833,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.368864,11.644,1455.56,81.2254,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.387424,3.598,224.902,12.5503,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,2.703936,3.177,99.288,5.5406,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.38832,5.395,224.831,12.5464,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,2.706912,4.76,99.185,5.5349,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.38752,7.196,224.92,12.5514,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,2.705568,6.35,99.24,5.538,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.32144,213.786,234.873,13.1067,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.142944,349.835,96.085,5.3619,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,45.15712,389.577,26.75,1.4928,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,3.482848,315.693,81.891,4.5698,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,6.994496,314.393,79.155,4.4171,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.513184,337.626,46.366,2.5874,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,13.60944,323.161,41.914,2.3389,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.809248,1.529,191.12,10.6652,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.807968,3.059,191.219,10.6707,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.807616,4.589,191.255,10.6727,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.808032,6.118,191.238,10.6718,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.174016,197.452,216.927,12.1053,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.662784,330.624,90.809,5.0674,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,24.054913,365.667,25.108,1.4011,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.162112,254.268,65.957,3.6806,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.294848,256.007,64.455,3.5968,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,4.283392,256.692,35.251,1.9671,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,8.551648,257.146,33.352,1.8611,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.195296,10.996,1374.59,76.707,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,22.386368,12.279,1534.945,85.6554,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.384288,1.801,112.599,6.2834,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,2.697888,1.592,49.755,2.7765,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,49.125793,11.191,699.509,39.0351,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.382432,2.704,112.694,6.2887,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,2.696512,2.389,49.784,2.7781,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,49.200066,16.761,698.496,38.9785,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.382016,3.606,112.72,6.2902,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,2.696064,3.186,49.795,2.7787,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,49.310303,22.298,696.977,38.8938,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.185184,185.544,203.844,11.3752,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.568928,350.402,96.241,5.3706,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,22.476768,391.342,26.871,1.4995,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.260448,243.207,63.088,3.5205,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.46128,246.457,62.05,3.4626,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,3.278272,335.394,46.059,2.5703,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,6.550336,335.712,43.542,2.4298,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,2.412224,0.89,111.288,6.2103,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,226.565735,1.213,151.664,8.4634,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.407008,1.784,111.536,6.2241,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,132.844452,4.138,258.678,14.4352,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.406144,2.678,111.583,6.2267,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,132.834854,6.208,258.712,14.4371,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.4056,3.571,111.615,6.2285,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,132.847015,8.277,258.705,14.4366,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.111616,153.919,169.101,9.4364,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.917408,299.624,82.294,4.5923,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,12.028096,365.648,25.107,1.4011,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,2.152,127.731,33.133,1.849,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,4.286208,128.262,32.292,1.802,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.156512,254.928,35.009,1.9536,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.292544,256.145,33.222,1.8539,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.048224,22.266,2783.384,155.3228,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,11.181312,12.292,1536.574,85.7463,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.3568,0.911,56.956,3.1784,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,2.562912,0.838,26.188,1.4614,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,24.627808,11.161,697.665,38.9322,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,29.450209,9.334,291.712,16.2786,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.359744,1.365,56.889,3.1746,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,2.706304,1.19,24.802,1.384,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,24.679359,16.707,696.25,38.8533,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,29.63344,13.914,289.926,16.1789,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.357184,1.822,56.954,3.1782,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,2.70112,1.59,24.851,1.3868,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,24.731745,22.229,694.818,38.7733,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,29.664991,18.532,289.635,16.1627,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.123488,139.122,152.844,8.5292,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.800928,343.199,94.262,5.2602,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,11.10224,396.14,27.201,1.5179,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,1.114272,246.688,63.991,3.5709,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,2.232896,246.208,61.988,3.4591,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.148672,255.858,35.137,1.9608,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.241248,259.242,33.624,1.8763,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,2.233856,0.481,60.087,3.3531,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,109.101181,1.26,157.477,8.7878,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.223904,0.966,60.36,3.3683,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,66.232323,4.15,259.42,14.4765,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.22384,1.448,60.365,3.3686,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,66.192833,6.229,259.59,14.4861,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.224224,1.931,60.358,3.3682,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,66.200127,8.304,259.578,14.4854,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.083552,102.809,112.95,6.303,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.547008,251.256,69.009,3.851,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,6.29632,349.255,23.981,1.3383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,2.15136,63.885,16.572,0.9248,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,4.28528,64.145,16.15,0.9012,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,2.152704,127.69,17.535,0.9785,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,4.287072,128.236,16.632,0.9281,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.039168,13.707,1713.464,95.6174,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,5.599296,12.273,1534.203,85.614,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.347936,0.457,28.586,1.5952,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,2.697728,0.398,12.44,0.6942,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,12.990688,10.58,661.319,36.9039,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,15.90176,8.643,270.127,15.074,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.34784,0.686,28.588,1.5953,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,2.695968,0.597,12.448,0.6947,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,12.972544,15.892,662.284,36.9578,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,15.900768,12.965,270.16,15.0759,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.348,0.915,28.588,1.5953,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,2.695936,0.797,12.449,0.6947,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,13.024384,21.105,659.688,36.813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,15.889408,17.299,270.37,15.0876,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.083328,103.086,113.253,6.3199,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.44304,310.218,85.204,4.7547,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,5.34704,411.26,28.239,1.5758,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,1.041984,131.901,34.215,1.9093,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,2.039328,134.788,33.936,1.8937,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,1.099008,250.115,34.348,1.9167,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,2.051968,267.916,34.749,1.9391,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,2.045504,0.262,32.81,1.8309,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,42.740768,1.608,200.99,11.2159,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.040864,0.526,32.887,1.8352,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,22.516768,6.104,381.537,21.2911,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.040224,0.789,32.899,1.8359,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,22.507839,9.159,381.712,21.3009,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.0408,1.052,32.892,1.8355,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], -llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,22.518433,12.207,381.555,21.2922,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.270272,254.26,279.339,15.5881,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.131808,351.079,96.427,5.381,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,47.013214,374.197,25.694,1.4338,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,4.29712,255.872,66.373,3.7038,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,8.565184,256.74,64.639,3.6071,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.419072,342.576,47.046,2.6253,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,context,cudnn,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,12.81808,343.113,44.502,2.4833,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,0.369056,11.638,1454.803,81.1832,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.382048,3.606,225.41,12.5787,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,64,8,128,128,gqa,,16,2.703648,3.177,99.299,5.5412,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.381568,5.41,225.469,12.582,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,64,8,128,128,gqa,,16,2.70384,4.765,99.298,5.5412,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.381632,7.213,225.476,12.5824,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,64,8,128,128,gqa,,16,2.705824,6.349,99.231,5.5374,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,64,8,128,128,gqa,,16,0.329024,208.859,229.459,12.8046,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,64,8,128,128,gqa,,16,3.141344,350.013,96.134,5.3646,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,64,8,128,128,gqa,,16,44.145473,398.505,27.363,1.527,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,64,8,128,128,gqa,,16,3.520704,312.299,81.01,4.5207,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,64,8,128,128,gqa,,16,6.939424,316.888,79.783,4.4522,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,64,8,128,128,gqa,,16,6.493728,338.638,46.505,2.5951,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,64,8,128,128,gqa,,16,13.293472,330.843,42.91,2.3945,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,64,8,128,128,gqa,,16,2.84816,1.508,188.509,10.5195,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,64,8,128,128,gqa,,16,2.844288,3.02,188.777,10.5344,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,64,8,128,128,gqa,,16,2.839328,4.538,189.118,10.5535,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,64,8,128,128,gqa,,16,2.84176,6.046,188.968,10.5451,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.171072,200.85,220.66,12.3136,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.671232,328.952,90.349,5.0418,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,23.284449,377.767,25.939,1.4475,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.187328,251.337,65.197,3.6382,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.29648,255.91,64.43,3.5954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,4.284,256.655,35.246,1.9669,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,8.55024,257.188,33.357,1.8615,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,0.19504,11.01,1376.394,76.8077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,22.389343,12.277,1534.741,85.644,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.4104,1.782,111.379,6.2154,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,32,4,128,128,gqa,,16,2.733344,1.571,49.11,2.7405,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,48.055874,11.44,715.083,39.9042,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.410752,2.672,111.37,6.2148,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,32,4,128,128,gqa,,16,2.730688,2.359,49.161,2.7433,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,48.153152,17.125,713.682,39.826,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.410144,3.564,111.405,6.2168,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,32,4,128,128,gqa,,16,2.732768,3.143,49.126,2.7414,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,48.240799,22.792,712.429,39.7561,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,4,128,128,gqa,,16,0.187872,182.889,200.928,11.2125,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,4,128,128,gqa,,16,1.621984,338.94,93.093,5.1949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,4,128,128,gqa,,16,21.996193,399.892,27.458,1.5323,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,4,128,128,gqa,,16,2.35728,233.216,60.496,3.3759,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,4,128,128,gqa,,16,4.461568,246.441,62.046,3.4624,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,4,128,128,gqa,,16,3.28208,335.005,46.006,2.5673,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,4,128,128,gqa,,16,6.504768,338.063,43.847,2.4468,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,4,128,128,gqa,,16,2.4648,0.871,108.914,6.0778,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,4,128,128,gqa,,16,226.582718,1.213,151.652,8.4628,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,4,128,128,gqa,,16,2.447776,1.755,109.678,6.1204,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,4,128,128,gqa,,16,133.074722,4.131,258.23,14.4102,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,4,128,128,gqa,,16,2.459328,2.62,109.17,6.0921,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,4,128,128,gqa,,16,133.054596,6.198,258.285,14.4132,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,4,128,128,gqa,,16,2.453504,3.501,109.436,6.1069,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,4,128,128,gqa,,16,133.005249,8.267,258.397,14.4195,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.119104,144.243,158.47,8.8432,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.955488,287.683,79.015,4.4093,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,11.980864,367.089,25.206,1.4066,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,2.184128,125.852,32.646,1.8218,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,4.294144,128.025,32.233,1.7987,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.184,251.72,34.568,1.929,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.301984,255.582,33.149,1.8498,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,0.05296,20.275,2534.477,141.4329,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,11.176192,12.297,1537.278,85.7856,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.393024,0.897,56.094,3.1302,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,16,2,128,128,gqa,,16,2.514784,0.854,26.689,1.4893,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,24.067616,11.421,713.904,39.8384,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,16,2,128,128,gqa,,16,29.301311,9.381,293.194,16.3613,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.391552,1.347,56.132,3.1324,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,16,2,128,128,gqa,,16,2.741152,1.175,24.486,1.3664,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,24.136703,17.083,711.904,39.7268,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,16,2,128,128,gqa,,16,29.23344,14.104,293.893,16.4003,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.390944,1.796,56.15,3.1333,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,16,2,128,128,gqa,,16,2.737248,1.569,24.523,1.3685,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,24.14448,22.769,711.718,39.7164,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp4,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,16,2,128,128,gqa,,16,29.189089,18.834,294.358,16.4262,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,2,128,128,gqa,,16,0.144384,118.987,130.723,7.2948,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,2,128,128,gqa,,16,0.832768,330.077,90.658,5.0591,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,2,128,128,gqa,,16,10.757568,408.833,28.072,1.5665,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,2,128,128,gqa,,16,1.157504,237.475,61.601,3.4375,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,2,128,128,gqa,,16,2.242912,245.108,61.711,3.4437,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,2,128,128,gqa,,16,2.225792,246.993,33.919,1.8928,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,2,128,128,gqa,,16,4.256832,258.293,33.501,1.8695,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,2,128,128,gqa,,16,2.276384,0.472,58.965,3.2904,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,2,128,128,gqa,,16,110.371361,1.245,155.665,8.6866,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,2,128,128,gqa,,16,2.26544,0.948,59.253,3.3065,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,2,128,128,gqa,,16,66.247551,4.149,259.36,14.4732,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,2,128,128,gqa,,16,2.261824,1.424,59.351,3.312,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,2,128,128,gqa,,16,66.214653,6.227,259.505,14.4813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,2,128,128,gqa,,16,2.261408,1.899,59.366,3.3128,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,2,128,128,gqa,,16,66.222916,8.302,259.488,14.4804,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.086144,99.716,109.551,6.1134,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.568864,241.602,66.358,3.703,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,6.283136,349.988,24.032,1.3411,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,2.180224,63.039,16.352,0.9125,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,4.340064,63.335,15.946,0.8898,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,2.152384,127.709,17.538,0.9787,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,4.285408,128.286,16.639,0.9285,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,0.03728,14.401,1800.24,100.4598,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,5.599904,12.272,1534.037,85.6047,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.381696,0.451,28.18,1.5726,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,2,131072,8,1,128,128,gqa,,16,2.733216,0.393,12.278,0.6852,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,12.667328,10.85,678.2,37.846,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,2,131072,8,1,128,128,gqa,,16,15.899872,8.644,270.159,15.0758,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.38224,0.676,28.176,1.5723,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,3,131072,8,1,128,128,gqa,,16,2.733568,0.589,12.277,0.6851,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,12.69968,16.233,676.514,37.7519,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,3,131072,8,1,128,128,gqa,,16,15.906528,12.961,270.062,15.0704,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.38096,0.902,28.193,1.5732,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,1,4,131072,8,1,128,128,gqa,,16,2.7312,0.786,12.289,0.6857,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,12.73952,21.577,674.439,37.6361,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +llama3.1,llama3.1-tp8,generation,cudnn,bfloat16,fp8_e4m3,128,4,131072,8,1,128,128,gqa,,16,15.898112,17.29,270.222,15.0793,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn fp8 9.26.0, +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,128,128,gqa,,16,0.08528,100.726,110.661,6.1753,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,128,128,gqa,,16,0.459584,299.051,82.137,4.5835,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,128,128,gqa,,16,5.351712,410.901,28.214,1.5745,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,128,128,gqa,,16,1.064608,129.098,33.488,1.8687,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,128,128,gqa,,16,2.07504,132.469,33.352,1.8611,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,128,128,gqa,,16,1.083904,253.6,34.827,1.9435,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,128,128,gqa,,16,2.056672,267.304,34.669,1.9347,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,128,128,gqa,,16,2.076096,0.259,32.327,1.8039,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,128,128,gqa,,16,42.774178,1.607,200.833,11.2072,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,128,128,gqa,,16,2.068832,0.519,32.442,1.8104,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,128,128,gqa,,16,22.535809,6.099,381.215,21.2731,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,128,128,gqa,,16,2.072256,0.777,32.39,1.8075,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,128,128,gqa,,16,22.514912,9.157,381.592,21.2942,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,128,128,gqa,,16,2.071776,1.037,32.4,1.808,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=128; cga=1)], +llama3.1,llama3.1-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,128,128,gqa,,16,22.533249,12.199,381.305,21.2782,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png old mode 100644 new mode 100755 index 9917715fe..3ba967774 Binary files a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png and b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_context.png differ diff --git a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png old mode 100644 new mode 100755 index f1749152f..dba5eedd8 Binary files a/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png and b/benchmark/attention_inference/results/llama3.1/rtxpro6000/llama3.1_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260818_131428.csv b/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260819_060531.csv similarity index 60% rename from benchmark/attention_inference/results/qwen35/gb200/qwen35_20260818_131428.csv rename to benchmark/attention_inference/results/qwen35/gb200/qwen35_20260819_060531.csv index ac0758a17..5632ee270 100644 --- a/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260818_131428.csv +++ b/benchmark/attention_inference/results/qwen35/gb200/qwen35_20260819_060531.csv @@ -1,121 +1,121 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.08208,837.226,868.703,10.8588,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.666784,1648.977,427.744,5.3468,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,11.146496,1578.271,102.351,1.2794,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.666048,1650.799,226.703,2.8338,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.279136,1719.147,222.973,2.7872,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.256544,1750.057,133.519,1.669,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.503552,1756.723,120.625,1.5078,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.102208,42.022,2626.685,32.8336,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.682432,63.318,3957.869,49.4734,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.91296,1.243,38.84,0.4855,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,188.968826,5.818,181.872,2.2734,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.90896,1.865,38.867,0.4858,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,188.396729,8.754,182.446,2.2806,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.910912,2.486,38.861,0.4858,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,188.862595,11.644,182.019,2.2752,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.217632,315.76,327.632,4.0954,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.024672,1073.038,278.345,3.4793,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,11.87568,1481.362,96.066,1.2008,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.85232,1290.022,177.158,2.2145,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.596448,1377.448,178.655,2.2332,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.458112,1508.131,115.061,1.4383,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.851904,1542.144,105.891,1.3236,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.448224,2.966,185.378,2.3172,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,107.157982,5.13,320.685,4.0086,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.466848,5.856,183.046,2.2881,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,107.561539,10.222,319.521,3.994,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.466688,8.785,183.089,2.2886,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,106.601791,15.471,322.437,4.0305,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.469248,11.693,182.792,2.2849,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,105.920418,20.761,324.55,4.0569,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.083808,819.963,850.792,10.6349,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.64832,1695.94,439.926,5.4991,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,10.784896,1631.187,105.782,1.3223,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.646496,1700.725,233.559,2.9195,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.236128,1778.961,230.731,2.8841,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.276448,1722.768,131.437,1.643,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.448544,1796.188,123.334,1.5417,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.106208,40.439,2527.759,31.597,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.524704,64.49,4031.1,50.3887,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.549824,1.311,40.994,0.5124,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,191.447678,5.743,179.517,2.244,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.552544,1.966,40.982,0.5123,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,191.323608,8.62,179.655,2.2457,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.549984,2.623,41.003,0.5125,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,191.444839,11.486,179.564,2.2445,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.216448,317.487,329.424,4.1178,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.014848,1083.425,281.04,3.513,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,11.584416,1518.608,98.481,1.231,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.831936,1321.63,181.498,2.2687,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.546496,1421.939,184.425,2.3053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.439744,1527.371,116.529,1.4566,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.812096,1563.974,107.39,1.3424,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.464704,2.932,183.292,2.2911,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,105.461441,5.213,325.844,4.073,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.482784,5.793,181.079,2.2635,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,105.358788,10.436,326.201,4.0775,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.48272,8.69,181.109,2.2639,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,106.243134,15.524,323.525,4.0441,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.482272,11.59,181.186,2.2648,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,105.328285,20.878,326.375,4.0797,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.059232,580.087,601.897,7.5237,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.358592,1533.096,397.684,4.9711,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.492928,1601.349,103.847,1.2981,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.581152,945.976,129.91,1.6239,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.120896,980.922,127.225,1.5903,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.650816,1689.435,128.894,1.6112,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.261408,1743.309,119.703,1.4963,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.093824,22.888,1430.701,17.8838,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.257056,64.57,4036.114,50.4514,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.494464,0.661,20.672,0.2584,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,96.140862,5.718,178.738,2.2342,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.494176,0.992,20.675,0.2584,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,96.160576,8.576,178.724,2.234,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.495072,1.323,20.675,0.2584,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,95.935936,11.461,179.164,2.2395,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.145888,235.521,244.376,3.0547,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.532992,1031.452,267.558,3.3445,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.602304,1570.085,101.82,1.2727,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.649184,846.841,116.296,1.4537,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.199008,917.018,118.937,1.4867,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.721152,1524.66,116.322,1.454,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.285792,1710.248,117.433,1.4679,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.141696,1.881,117.574,1.4697,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,36.238655,7.585,474.134,5.9267,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.185888,3.622,113.207,1.4151,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,36.53075,15.049,470.4,5.88,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.181664,5.452,113.625,1.4203,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,36.322464,22.703,473.155,5.9144,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.18464,7.251,113.354,1.4169,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,36.37075,30.231,472.585,5.9073,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.057952,296.45,325.69,4.0711,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.201024,1367.389,375.564,4.6946,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.322784,1893.438,130.012,1.6252,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.57904,474.713,123.14,1.5393,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.122976,489.553,123.255,1.5407,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.582784,943.327,129.546,1.6193,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.123904,978.297,126.885,1.5861,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.092224,11.643,1455.434,18.1929,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.256672,32.288,4036.233,50.4529,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.497728,0.33,20.659,0.2582,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,52.081055,5.278,329.908,4.1239,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.493504,0.496,20.673,0.2584,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,52.085857,7.916,329.898,4.1237,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.493344,0.661,20.675,0.2584,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,52.026176,10.567,330.296,4.1287,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.12848,133.716,146.905,1.8363,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.297536,923.848,253.742,3.1718,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.617344,1680.347,115.38,1.4423,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.644064,426.787,110.708,1.3839,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.1904,461.824,116.274,1.4534,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.64768,848.808,116.566,1.4571,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.1912,923.029,119.717,1.4965,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.137888,0.944,117.961,1.4745,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,18.708769,7.346,918.335,11.4792,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.18416,1.814,113.358,1.417,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,18.746592,14.663,916.538,11.4567,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.181728,2.726,113.598,1.42,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,18.820513,21.908,912.994,11.4124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.183264,3.63,113.458,1.4182,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,18.771233,29.287,915.447,11.4431,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.053216,161.416,197.041,2.463,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.12336,1114.129,340.005,4.2501,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.183936,1857.384,141.707,1.7713,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.580704,236.676,119.176,1.4897,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.122368,244.909,121.453,1.5182,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.581184,472.962,122.686,1.5336,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.12192,490.013,123.371,1.5421,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.092192,5.823,1455.894,18.1987,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.257248,16.142,4035.563,50.4445,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.313408,0.17,21.26,0.2658,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.525248,4.502,562.843,7.0355,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.304736,0.255,21.29,0.2661,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.532896,6.752,562.719,7.034,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.303424,0.341,21.295,0.2662,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.502239,9.012,563.302,7.0413,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.128448,66.875,81.634,1.0204,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.233504,588.594,179.625,2.2453,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.342528,1637.972,124.967,1.5621,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.646816,212.485,106.995,1.3374,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.185632,231.841,114.972,1.4372,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.647488,424.53,110.123,1.3765,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.186624,463.294,116.644,1.458,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.14608,0.468,117.114,1.4639,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,10.124544,6.787,1696.905,21.2113,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.188,0.904,112.985,1.4123,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,9.86192,13.936,1742.147,21.7768,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.188352,1.355,112.955,1.4119,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,10.106176,20.399,1700.093,21.2512,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.189728,1.805,112.828,1.4103,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,10.055936,27.335,1708.639,21.358,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.350752,1567.363,406.573,5.0822,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.179936,1698.108,110.122,1.3765,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.58528,939.304,128.994,1.6124,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.12832,974.468,126.388,1.5799,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.642848,1710.376,130.491,1.6311,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.19568,1839.14,126.284,1.5785,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.09584,22.407,1400.606,17.5076,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.278784,64.242,4015.619,50.1952,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.547936,0.656,20.503,0.2563,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,96.535583,5.695,178.008,2.2251,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.546944,0.984,20.508,0.2564,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,96.343231,8.559,178.385,2.2298,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.545632,1.312,20.515,0.2564,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,96.393539,11.406,178.313,2.2289,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.152384,225.481,233.959,2.9245,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.526016,1045.131,271.106,3.3888,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,5.315936,1654.665,107.305,1.3413,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.652768,842.192,115.657,1.4457,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.195776,919.496,119.258,1.4907,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.709984,1548.643,118.152,1.4769,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.28928,1705.621,117.116,1.4639,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.142656,1.879,117.476,1.4684,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,34.655617,7.932,495.792,6.1974,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.188608,3.613,112.948,1.4118,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,35.104286,15.661,489.515,6.1189,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.185824,5.433,113.227,1.4153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,35.186878,23.436,488.425,6.1053,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.185344,7.247,113.286,1.4161,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,35.077343,31.345,490.01,6.1251,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.05568,308.547,338.979,4.2372,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.19968,1376.592,378.092,4.7262,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.270016,1937.452,133.034,1.6629,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.583584,471.017,122.181,1.5273,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.126816,487.884,122.835,1.5354,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.583648,941.93,129.354,1.6169,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.127392,975.27,126.492,1.5812,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.093344,11.503,1437.97,17.9746,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.276544,32.138,4017.477,50.2185,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.543584,0.328,20.514,0.2564,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,52.671646,5.219,326.209,4.0776,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.5432,0.492,20.516,0.2565,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,52.660671,7.83,326.297,4.0787,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.542592,0.656,20.519,0.2565,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,52.654976,10.441,326.352,4.0794,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.128736,133.45,146.613,1.8327,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.296864,925.939,254.317,3.179,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.55328,1722.508,118.275,1.4784,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.648416,423.922,109.965,1.3746,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.18832,462.633,116.477,1.456,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.6536,841.12,115.51,1.4439,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.193152,921.519,119.521,1.494,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.142176,0.94,117.518,1.469,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,18.151199,7.572,946.544,11.8318,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.186176,1.81,113.165,1.4146,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,18.152639,15.143,946.527,11.8316,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.18608,2.716,113.181,1.4148,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,18.341856,22.48,936.82,11.7102,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.184736,3.625,113.317,1.4165,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,18.324991,30.0,937.739,11.7217,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.054464,157.718,192.526,2.4066,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.122624,1120.816,342.046,4.2756,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.191232,1846.008,140.839,1.7605,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.581632,236.299,118.986,1.4873,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.1248,244.379,121.19,1.5149,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.584256,470.475,122.041,1.5255,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.127488,487.593,122.761,1.5345,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.094624,5.674,1418.476,17.7309,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.278048,16.063,4015.942,50.1993,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.549088,0.164,20.495,0.2562,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.41264,4.519,564.927,7.0616,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.547744,0.246,20.5,0.2563,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.402624,6.781,565.13,7.0641,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.547424,0.328,20.502,0.2563,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.388737,9.045,565.406,7.0676,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.127232,67.514,82.414,1.0302,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.231296,594.212,181.339,2.2667,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.347072,1632.447,124.546,1.5568,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.646176,212.696,107.101,1.3388,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.189248,231.136,114.623,1.4328,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.648992,423.546,109.868,1.3733,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.192736,460.92,116.046,1.4506,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.142304,0.47,117.501,1.4688,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,9.717088,7.072,1768.06,22.1007,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.184768,0.906,113.293,1.4162,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,9.716,14.146,1768.312,22.1039,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.190208,1.353,112.779,1.4097,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,9.64752,21.369,1780.918,22.2615,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.188,1.808,112.992,1.4124,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,9.723712,28.269,1767.017,22.0877,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png b/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png old mode 100644 new mode 100755 index 3985c57c6..4acd51d54 Binary files a/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/gb200/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png old mode 100644 new mode 100755 index 334c19b72..9b1ed896c Binary files a/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/gb200/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260818_132128.csv b/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260819_060443.csv similarity index 61% rename from benchmark/attention_inference/results/qwen35/gb300/qwen35_20260818_132128.csv rename to benchmark/attention_inference/results/qwen35/gb300/qwen35_20260819_060443.csv index d12372038..274a59ee5 100644 --- a/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260818_132128.csv +++ b/benchmark/attention_inference/results/qwen35/gb300/qwen35_20260819_060443.csv @@ -1,121 +1,121 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.07584,906.111,940.179,11.7522,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.569248,1931.516,501.034,6.2629,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.133024,1926.217,124.915,1.5614,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.586976,1873.18,257.242,3.2155,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.133952,1939.256,251.521,3.144,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.165792,1886.291,143.913,1.7989,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.290784,1919.887,131.828,1.6479,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.100768,42.622,2664.221,33.3028,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.582336,64.057,4004.03,50.0504,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.727328,1.277,39.912,0.4989,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,186.702393,5.889,184.08,2.301,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.722016,1.917,39.948,0.4994,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,188.030624,8.771,182.802,2.285,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.72208,2.556,39.953,0.4994,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,187.845276,11.707,183.004,2.2876,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.204928,335.335,347.943,4.3493,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.920576,1194.374,309.82,3.8727,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.864576,1783.37,115.651,1.4456,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.789344,1392.944,191.292,2.3911,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.4808,1485.024,192.607,2.4076,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.37008,1605.033,122.454,1.5307,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.651936,1658.429,113.875,1.4234,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.439936,2.983,186.445,2.3306,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,100.468605,5.472,342.037,4.2755,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.461184,5.879,183.756,2.2969,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,100.799454,10.908,340.955,4.2619,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.464704,8.797,183.337,2.2917,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,100.972801,16.334,340.412,4.2551,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.459904,11.768,183.962,2.2995,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,100.389954,21.905,342.43,4.2804,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.054048,635.726,659.628,8.2454,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.301152,1825.509,473.536,5.9192,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.428832,1986.098,128.798,1.61,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.57872,949.951,130.456,1.6307,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.109984,990.565,128.476,1.606,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.585472,1877.992,143.279,1.791,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.132576,1941.612,133.32,1.6665,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.09136,23.506,1469.288,18.3661,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.21168,65.266,4079.599,50.995,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.477216,0.663,20.727,0.2591,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,95.103294,5.781,180.688,2.2586,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.474176,0.995,20.739,0.2592,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,95.062019,8.675,180.789,2.2599,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.475392,1.327,20.737,0.2592,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,94.968163,11.578,180.99,2.2624,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.144992,236.977,245.887,3.0736,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.477312,1151.775,298.77,3.7346,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.903936,1793.68,116.32,1.454,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.64352,854.295,117.32,1.4665,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.18464,928.14,120.379,1.5047,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.6584,1669.975,127.409,1.5926,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.205888,1823.572,125.215,1.5652,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.13936,1.885,117.815,1.4727,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,31.770241,8.652,540.82,6.7602,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.180544,3.638,113.719,1.4215,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,31.808767,17.283,540.23,6.7529,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.181632,5.452,113.628,1.4204,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,31.80784,25.925,540.312,6.7539,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.183424,7.259,113.47,1.4184,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,31.829344,34.544,540.013,6.7502,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.050912,337.442,370.725,4.6341,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.1688,1628.424,447.26,5.5907,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.064992,2129.813,146.243,1.828,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.575296,477.803,123.942,1.5493,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.115808,492.697,124.046,1.5506,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.579264,949.059,130.333,1.6292,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.11664,984.661,127.71,1.5964,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.089696,11.971,1496.454,18.7057,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.209184,32.652,4081.769,51.0221,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.078272,877.957,910.966,11.3871,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.567456,1937.616,502.616,6.2827,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.0648,1940.714,125.855,1.5732,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.58352,1884.274,258.766,3.2346,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.231008,1786.36,231.69,2.8961,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.152544,1907.973,145.567,1.8196,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.290336,1920.263,131.854,1.6482,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.101984,42.114,2632.454,32.9057,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,8.600672,63.92,3995.494,49.9437,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.741088,1.274,39.831,0.4979,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,186.664963,5.89,184.117,2.3015,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.738176,1.912,39.853,0.4982,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,187.686234,8.787,183.137,2.2892,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.7392,2.549,39.851,0.4981,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,186.383865,11.798,184.439,2.3055,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.20016,343.323,356.231,4.4529,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,0.920064,1195.038,309.992,3.8749,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,9.832704,1789.15,116.026,1.4503,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,0.783008,1404.215,192.84,2.4105,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,1.469664,1496.276,194.067,2.4258,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,1.360352,1616.51,123.33,1.5416,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,2.630272,1672.088,114.813,1.4352,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,1.43952,2.984,186.498,2.3312,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,100.87133,5.45,340.671,4.2584,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,1.46048,5.882,183.844,2.2981,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,100.641281,10.925,341.491,4.2686,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,1.459904,8.826,183.939,2.2992,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,100.581474,16.397,341.736,4.2717,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,1.454624,11.811,184.63,2.3079,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,101.075104,21.756,340.109,4.2514,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.056832,604.584,627.315,7.8414,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.302688,1816.246,471.133,5.8892,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.361184,2016.905,130.796,1.6349,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.576896,952.955,130.868,1.6359,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.116096,985.141,127.772,1.5972,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.582656,1887.068,143.972,1.7996,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.202592,1828.57,125.558,1.5695,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.093152,23.054,1441.022,18.0128,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.212288,65.256,4079.01,50.9876,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.475392,0.663,20.732,0.2592,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,94.890015,5.794,181.095,2.2637,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.476416,0.995,20.732,0.2591,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,94.982109,8.682,180.941,2.2618,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.474208,1.327,20.741,0.2593,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,94.874977,11.589,181.167,2.2646,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.139296,246.667,255.941,3.1993,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.471968,1164.816,302.153,3.7769,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,4.688576,1876.069,121.663,1.5208,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,0.644032,853.616,117.226,1.4653,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,1.185888,927.163,120.253,1.5032,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,0.6504,1690.516,128.976,1.6122,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,1.198656,1834.574,125.97,1.5746,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,1.137696,1.888,117.988,1.4748,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,31.387903,8.757,547.407,6.8426,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,1.179904,3.64,113.781,1.4223,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,31.414207,17.5,547.016,6.8377,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,1.1816,5.452,113.631,1.4204,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,31.421568,26.244,546.954,6.8369,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,1.17792,7.292,114.0,1.425,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,31.415648,34.999,547.124,6.8391,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.051872,331.197,363.864,4.5483,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.170432,1612.83,442.977,5.5372,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.048896,2146.544,147.392,1.8424,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.576384,476.901,123.708,1.5463,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.116192,492.528,124.004,1.55,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.577408,952.11,130.752,1.6344,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.116352,984.915,127.743,1.5968,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.091712,11.708,1463.559,18.2945,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.208576,32.657,4082.359,51.0295,20,True,,NVIDIA GB300,cudnn 9.26.0, qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.476672,0.332,20.726,0.2591,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.699905,5.317,332.34,4.1543,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.476928,0.497,20.726,0.2591,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.630718,7.986,332.806,4.1601,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.473824,0.663,20.737,0.2592,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,51.627262,10.649,332.849,4.1606,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.132928,129.242,141.989,1.7749,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.275136,999.062,274.401,3.43,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.409184,1825.534,125.349,1.5669,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.643232,427.339,110.851,1.3856,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.18224,465.012,117.076,1.4635,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.643552,854.252,117.314,1.4664,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.184192,928.491,120.425,1.5053,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.136032,0.945,118.153,1.4769,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,16.317184,8.423,1052.934,13.1617,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.18336,1.815,113.435,1.4179,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,16.351328,16.811,1050.799,13.135,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.179392,2.731,113.823,1.4228,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,16.35664,25.208,1050.522,13.1315,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.1816,3.635,113.618,1.4202,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,16.366432,33.59,1049.958,13.1245,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.052928,162.295,198.114,2.4764,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.1048,1311.44,400.22,5.0027,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.059808,2074.926,158.304,1.9788,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.57712,238.146,119.916,1.499,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.116544,246.186,122.086,1.5261,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.575968,477.245,123.797,1.5475,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.115008,493.051,124.135,1.5517,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.090048,5.962,1490.559,18.632,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.209952,16.323,4080.9,51.0113,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.258048,0.172,21.449,0.2681,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.455488,4.513,564.132,7.0517,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.253216,0.258,21.466,0.2683,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.456768,6.769,564.126,7.0516,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.253952,0.343,21.464,0.2683,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.506592,9.01,563.221,7.0403,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.12944,66.362,81.009,1.0126,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.212768,645.957,197.13,2.4641,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.228448,1790.082,136.572,1.7072,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.638752,215.168,108.346,1.3543,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.181696,232.613,115.355,1.4419,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.645344,425.94,110.489,1.3811,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.186176,463.469,116.688,1.4586,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.1392,0.471,117.821,1.4728,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,8.434944,8.147,2036.812,25.4601,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.185056,0.906,113.265,1.4158,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,8.489984,16.188,2023.669,25.2959,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.184224,1.36,113.349,1.4169,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,8.500128,24.254,2021.316,25.2664,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.181376,1.818,113.625,1.4203,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,8.460192,32.491,2030.919,25.3865,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.652447,5.322,332.646,4.1581,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.47104,0.498,20.745,0.2593,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.629951,7.986,332.811,4.1601,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.473504,0.663,20.738,0.2592,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,51.54781,10.665,333.362,4.167,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.124448,138.049,151.665,1.8958,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.266144,1032.816,283.672,3.5459,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,2.36,1863.579,127.962,1.5995,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,0.639808,429.626,111.445,1.3931,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,1.182528,464.899,117.048,1.4631,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,0.647104,849.563,116.67,1.4584,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,1.181376,930.704,120.712,1.5089,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,1.138112,0.943,117.937,1.4742,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,16.204384,8.482,1060.264,13.2533,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,1.176256,1.826,114.12,1.4265,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,16.326401,16.836,1052.404,13.155,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,1.178208,2.734,113.938,1.4242,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,16.190529,25.467,1061.3,13.2663,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,1.178048,3.646,113.96,1.4245,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,16.225376,33.882,1059.086,13.2386,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.051712,166.111,202.772,2.5347,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.105472,1303.085,397.67,4.9709,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.045184,2103.958,160.519,2.0065,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.575456,238.835,120.263,1.5033,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.115104,246.504,122.244,1.5281,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.575936,477.272,123.804,1.5475,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.115808,492.697,124.046,1.5506,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.092576,5.799,1449.856,18.1232,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.20848,16.329,4082.328,51.0291,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.238304,0.172,21.516,0.269,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,30.444256,4.514,564.34,7.0543,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.234272,0.258,21.531,0.2691,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,30.363647,6.79,565.856,7.0732,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.238656,0.344,21.517,0.269,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,30.456863,9.025,564.141,7.0518,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.124448,69.024,84.258,1.0532,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.213152,644.793,196.775,2.4597,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,1.212352,1813.849,138.386,1.7298,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,0.641056,214.395,107.956,1.3495,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,1.17776,233.39,115.741,1.4468,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,0.638944,430.207,111.595,1.3949,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,1.182336,464.974,117.067,1.4633,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,1.139232,0.471,117.818,1.4727,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,8.207808,8.372,2093.177,26.1647,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,1.181728,0.909,113.584,1.4198,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,8.373696,16.413,2051.772,25.6472,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,1.18224,1.362,113.539,1.4192,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,8.421088,24.481,2040.288,25.5036,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,1.179648,1.82,113.792,1.4224,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,8.372128,32.833,2052.282,25.6535,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d256[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png b/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png old mode 100644 new mode 100755 index 0d80f1297..b722966bd Binary files a/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/gb300/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png old mode 100644 new mode 100755 index 1ce337742..fac57855d Binary files a/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/gb300/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/h200/qwen35_20260818_143154.csv b/benchmark/attention_inference/results/qwen35/h200/qwen35_20260819_072512.csv similarity index 55% rename from benchmark/attention_inference/results/qwen35/h200/qwen35_20260818_143154.csv rename to benchmark/attention_inference/results/qwen35/h200/qwen35_20260819_072512.csv index 601dc02ff..0cdf25e07 100644 --- a/benchmark/attention_inference/results/qwen35/h200/qwen35_20260818_143154.csv +++ b/benchmark/attention_inference/results/qwen35/h200/qwen35_20260819_072512.csv @@ -1,61 +1,61 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.16768,409.825,425.234,8.859,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.658016,663.149,172.02,3.5838,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,26.680321,659.369,42.76,0.8908,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,1.496384,734.779,100.907,2.1022,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,2.947328,746.107,96.77,2.016,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,2.945344,746.61,56.962,1.1867,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,6.539008,672.586,46.183,0.9621,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.114336,37.564,2348.064,48.918,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,9.375232,58.639,3665.396,76.3624,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,2.86688,2.996,93.656,1.9512,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,97.472,11.28,352.595,7.3457,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,2.865696,4.496,93.706,1.9522,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,97.656189,16.889,351.973,7.3328,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,2.838112,6.053,94.629,1.9714,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,97.448929,22.566,352.764,7.3493,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.113408,302.975,314.366,6.5493,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.874176,628.885,163.132,3.3986,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,13.419552,655.468,42.507,0.8856,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,1.43552,382.966,52.592,1.0957,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,2.830976,388.386,50.374,1.0494,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,1.490496,737.682,56.281,1.1725,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,2.942336,747.373,51.318,1.0691,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.097888,21.938,1371.303,28.5688,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.697984,58.51,3657.306,76.1939,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,2.864896,1.499,46.861,0.9763,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,48.394817,11.36,355.081,7.3975,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,2.863872,2.25,46.883,0.9767,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,48.21904,17.102,356.419,7.4254,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,2.836736,3.028,47.337,0.9862,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,48.591839,22.627,353.727,7.3693,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.075968,226.146,248.452,5.1761,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.50688,542.294,148.945,3.103,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,6.823168,644.575,44.259,0.9221,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,1.429152,192.336,49.892,1.0394,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,2.828352,194.373,48.937,1.0195,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,1.42944,384.595,52.816,1.1003,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,2.83024,388.487,50.387,1.0497,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.09808,10.948,1368.535,28.5111,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.688864,29.312,3664.196,76.3374,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,2.860544,0.751,46.926,0.9776,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,24.501535,11.219,701.261,14.6096,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,2.86096,1.126,46.922,0.9775,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,24.493504,16.834,701.534,14.6153,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,2.861888,1.501,46.91,0.9773,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,24.445984,22.489,702.94,14.6446,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.074784,114.863,140.214,2.9211,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.327552,419.594,128.05,2.6677,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,3.265248,673.463,51.381,1.0704,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,1.429024,96.177,48.429,1.0089,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,2.829728,97.139,48.172,1.0036,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,1.42912,192.341,49.893,1.0394,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,2.825376,194.578,48.989,1.0206,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.097024,5.533,1383.388,28.8206,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.696992,14.631,3657.744,76.203,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,2.861024,0.375,46.915,0.9774,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,12.507168,10.989,1373.686,28.6185,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,2.858592,0.563,46.957,0.9783,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,12.636832,16.314,1359.632,28.3257,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,2.858432,0.751,46.961,0.9783,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,12.629184,21.765,1360.497,28.3437,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.165696,414.732,430.325,8.9651,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,1.650432,666.196,172.811,3.6002,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,27.550528,638.543,41.409,0.8627,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,1.530048,718.613,98.686,2.056,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,3.25968,674.613,87.497,1.8229,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,3.253632,675.867,51.565,1.0743,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,6.586496,667.737,45.85,0.9552,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.11584,37.077,2317.578,48.2829,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,9.580896,57.38,3586.714,74.7232,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,2.842816,3.022,94.449,1.9677,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,140.739868,7.812,244.196,5.0874,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,2.822336,4.565,95.146,1.9822,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,190.674393,8.65,180.267,3.7556,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,2.822208,6.087,95.162,1.9825,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,196.00592,11.219,175.385,3.6539,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.112768,304.694,316.15,6.5865,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,0.890976,617.027,160.056,3.3345,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,14.140064,622.069,40.341,0.8404,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,1.42336,386.238,53.042,1.105,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,2.812576,390.927,50.703,1.0563,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,1.534784,716.395,54.657,1.1387,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,3.262304,674.071,46.285,0.9643,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.09968,21.544,1346.65,28.0552,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,4.79152,57.368,3585.911,74.7065,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,2.842432,1.511,47.231,0.984,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,68.701248,8.002,250.127,5.211,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,2.849184,2.261,47.125,0.9818,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,71.762428,11.491,239.487,4.9893,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,2.8408,3.024,47.27,0.9848,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,72.784096,15.106,236.154,4.9199,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.075456,227.681,250.137,5.2112,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,0.511168,537.745,147.696,3.077,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,6.943776,633.38,43.491,0.9061,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,1.42128,193.402,50.168,1.0452,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,2.8064,195.894,49.32,1.0275,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,1.4184,387.589,53.227,1.1089,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,2.804,392.123,50.858,1.0595,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.099104,10.834,1354.395,28.2166,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,4.69776,29.256,3657.257,76.1929,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,2.838656,0.757,47.288,0.9852,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,26.017824,10.565,660.392,13.7582,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,2.841248,1.134,47.248,0.9843,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,26.383295,15.628,651.284,13.5684,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,2.841984,1.511,47.238,0.9841,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,26.413889,20.813,650.569,13.5535,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.0752,114.228,139.438,2.905,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.326368,421.117,128.515,2.6774,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,3.523488,624.104,47.615,0.992,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,1.418528,96.888,48.787,1.0164,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,2.803072,98.063,48.631,1.0131,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,1.420224,193.545,50.206,1.0459,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,2.800576,196.301,49.423,1.0296,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.098624,5.444,1360.945,28.353,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,4.678112,14.69,3672.506,76.5105,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,2.836768,0.379,47.316,0.9858,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,13.11792,10.477,1309.729,27.286,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,2.841248,0.567,47.243,0.9842,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,13.242464,15.568,1297.451,27.0302,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,2.840288,0.756,47.261,0.9846,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,13.312704,20.648,1290.644,26.8884,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png b/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png old mode 100644 new mode 100755 index 3d4b236cf..8d3ad1838 Binary files a/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/h200/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png old mode 100644 new mode 100755 index 0524e499d..af273d4ea Binary files a/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/h200/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260818_145215.csv b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260819_072406.csv similarity index 62% rename from benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260818_145215.csv rename to benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260819_072406.csv index 964135fc9..969966be9 100644 --- a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260818_145215.csv +++ b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_20260819_072406.csv @@ -1,121 +1,121 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.57344,119.837,124.343,6.9388,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,7.248064,151.697,39.35,2.1959,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,111.513054,157.759,10.231,0.5709,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,8.34096,131.821,18.103,1.0102,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,16.7848,131.013,16.992,0.9482,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,14.777728,148.807,11.353,0.6335,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,29.878912,147.196,10.107,0.564,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.213568,20.111,1257.062,70.1486,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,22.321119,24.629,1539.526,85.911,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,7.010112,1.225,38.302,2.1374,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,165.801437,6.631,207.285,11.5672,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,7.009472,1.838,38.31,2.1378,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,167.107681,9.869,205.69,11.4782,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,7.010272,2.451,38.31,2.1379,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,167.928192,13.095,204.71,11.4235,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.378432,181.59,188.417,10.5144,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,3.400224,323.364,83.881,4.6808,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,50.009216,351.779,22.813,1.273,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,5.851936,187.889,25.803,1.4399,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,11.693184,188.06,24.391,1.3611,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,8.662464,253.857,19.368,1.0808,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,17.319551,253.935,17.436,0.973,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,6.815424,0.63,39.391,2.1982,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,402.956085,1.364,85.28,4.7589,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.056064,1.418,44.336,2.4741,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,241.215424,4.558,142.479,7.9508,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.056896,2.127,44.335,2.4741,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,241.674179,6.824,142.226,7.9367,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.05712,2.836,44.339,2.4743,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,241.829849,9.093,142.152,7.9326,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.315968,108.744,112.833,6.2965,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,3.782784,145.331,37.699,2.1037,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,55.97187,157.152,10.191,0.5687,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,4.353312,126.284,17.343,0.9678,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,8.74416,125.742,16.309,0.9101,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,8.734112,125.887,9.604,0.536,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,16.757313,131.228,9.011,0.5028,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.081856,26.235,1639.881,91.5112,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,12.077408,22.76,1422.653,79.3891,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.018304,0.714,22.307,1.2448,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,83.603806,6.576,205.542,11.47,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.019168,1.07,22.307,1.2448,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,83.83194,9.837,205.007,11.4401,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.022496,1.426,22.297,1.2442,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,84.138557,13.068,204.285,11.3998,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.209984,163.63,169.782,9.4745,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,1.685888,326.093,84.588,4.7203,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,24.845505,354.032,22.959,1.2812,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,2.874304,191.266,26.266,1.4658,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,5.694272,193.091,25.044,1.3975,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,5.643872,194.815,14.863,0.8294,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,11.28512,194.86,13.38,0.7467,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,6.346976,0.338,21.149,1.1802,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,174.928452,1.571,98.223,5.4812,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,5.69856,0.754,23.559,1.3147,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,96.262497,5.711,178.513,9.9616,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,5.696896,1.131,23.568,1.3152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,96.428383,8.552,178.227,9.9457,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,5.698048,1.508,23.567,1.3151,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,96.522499,11.391,178.075,9.9372,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.195744,87.767,96.424,5.3808,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,2.032896,135.215,37.138,2.0724,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,28.438721,154.65,10.619,0.5926,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,3.263712,84.222,21.847,1.2192,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,6.679712,82.302,20.721,1.1563,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,4.331072,126.933,17.432,0.9727,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,8.720512,126.083,16.353,0.9126,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.081952,13.102,1637.86,91.3985,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,12.115488,11.344,1418.095,79.1348,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.018112,0.357,22.305,1.2447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,44.665951,6.154,384.677,21.4664,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.01856,0.535,22.305,1.2447,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,44.690079,9.226,384.493,21.4561,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.037792,0.711,22.235,1.2408,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,44.710114,12.296,384.344,21.4478,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.141856,121.108,133.053,7.4248,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,1.100352,249.809,68.612,3.8288,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,12.331008,356.666,24.49,1.3666,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,2.869056,95.808,24.852,1.3869,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,5.69328,96.562,24.311,1.3567,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,2.871168,191.475,26.295,1.4674,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,5.692096,193.165,25.053,1.3981,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,6.370752,0.169,21.069,1.1757,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,86.380417,1.591,198.898,11.0992,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,5.69632,0.377,23.565,1.315,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.141762,5.375,335.967,18.7482,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,5.69696,0.565,23.564,1.3149,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.170815,8.058,335.797,18.7387,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,5.69632,0.754,23.568,1.3152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,51.136192,10.751,336.045,18.7525,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.146208,58.751,71.718,4.0021,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,1.181344,116.341,35.505,1.9813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,14.663648,149.964,11.441,0.6385,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,3.196704,42.994,21.649,1.2081,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,6.639552,41.4,20.531,1.1457,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,3.261152,84.289,21.864,1.2201,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,6.673056,82.384,20.742,1.1575,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.081568,6.582,1645.521,91.8259,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,12.071936,5.692,1423.168,79.4179,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.025632,0.178,22.276,1.2431,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,23.676767,5.805,725.645,40.4936,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.041568,0.267,22.218,1.2398,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,23.718975,8.692,724.375,40.4227,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.038304,0.356,22.23,1.2405,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,23.692768,11.602,725.199,40.4687,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.13648,62.939,76.83,4.2874,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.608128,226.003,68.971,3.8488,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,5.985184,367.411,28.031,1.5642,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,2.866592,47.945,24.142,1.3472,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,5.692384,48.289,23.947,1.3363,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,2.868256,95.835,24.859,1.3872,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,5.691008,96.601,24.321,1.3572,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,6.351296,0.085,21.133,1.1793,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,39.744961,1.729,432.266,24.122,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,5.69504,0.189,23.569,1.3152,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,16.979776,8.094,1011.846,56.4646,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,5.69616,0.283,23.565,1.315,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,16.981249,12.14,1011.789,56.4614,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,5.696,0.377,23.566,1.3151,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], -qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,16.981407,16.187,1011.811,56.4626,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.559904,122.734,127.349,7.1065,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,7.252736,151.6,39.325,2.1945,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,111.491325,157.79,10.233,0.571,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,8.352992,131.631,18.077,1.0087,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,16.774879,131.09,17.002,0.9488,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,14.813056,148.452,11.326,0.632,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,29.842367,147.376,10.12,0.5647,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,0.216608,19.828,1239.42,69.164,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,22.329281,24.62,1538.963,85.8796,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,7.026144,1.223,38.215,2.1325,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,165.35498,6.649,207.845,11.5985,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.99744,1.841,38.376,2.1415,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,165.550659,9.962,207.624,11.5862,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.996768,2.455,38.384,2.142,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,generation,cudnn,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,165.655228,13.275,207.518,11.5803,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,32,2,256,256,gqa,,16,0.379392,181.131,187.941,10.4878,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,32,2,256,256,gqa,,16,3.40592,322.824,83.74,4.673,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,32,2,256,256,gqa,,16,48.812225,360.405,23.372,1.3043,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,32,2,256,256,gqa,,16,5.867168,187.401,25.736,1.4361,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,32,2,256,256,gqa,,16,11.713376,187.736,24.349,1.3588,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,32,2,256,256,gqa,,16,8.67296,253.549,19.344,1.0795,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,32,2,256,256,gqa,,16,17.339745,253.64,17.416,0.9719,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,32,2,256,256,gqa,,16,6.906592,0.622,38.871,2.1692,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,32,2,256,256,gqa,,16,403.069458,1.364,85.256,4.7576,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,32,2,256,256,gqa,,16,6.064256,1.416,44.276,2.4708,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,32,2,256,256,gqa,,16,239.03981,4.6,143.776,8.0232,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,32,2,256,256,gqa,,16,6.060736,2.126,44.307,2.4725,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,32,2,256,256,gqa,,16,239.028091,6.9,143.8,8.0246,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,32,2,256,256,gqa,,16,6.068416,2.831,44.256,2.4697,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,32,2,256,256,gqa,,16,239.010468,9.201,143.828,8.0261,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.31328,109.677,113.801,6.3505,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,3.783328,145.31,37.693,2.1034,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,56.027454,156.996,10.181,0.5681,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,4.361152,126.057,17.311,0.966,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,8.762976,125.472,16.274,0.9081,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,8.340768,131.824,10.057,0.5612,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,16.747393,131.305,9.016,0.5031,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,0.084224,25.497,1593.775,88.9383,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,12.098016,22.721,1420.23,79.2539,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,6.01856,0.714,22.306,1.2448,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,83.359741,6.595,206.143,11.5035,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,6.020608,1.07,22.301,1.2445,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,83.436577,9.883,205.979,11.4943,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,6.03216,1.424,22.261,1.2423,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,generation,cudnn,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,83.532867,13.163,205.766,11.4825,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,16,1,256,256,gqa,,16,0.211616,162.368,168.473,9.4014,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,16,1,256,256,gqa,,16,1.686464,325.981,84.559,4.7187,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,16,1,256,256,gqa,,16,24.180767,363.764,23.59,1.3164,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,16,1,256,256,gqa,,16,2.88064,190.845,26.209,1.4625,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,16,1,256,256,gqa,,16,5.704928,192.73,24.997,1.3949,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,16,1,256,256,gqa,,16,5.657376,194.35,14.828,0.8274,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,16,1,256,256,gqa,,16,11.311232,194.411,13.349,0.7449,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,16,1,256,256,gqa,,16,6.450816,0.333,20.809,1.1612,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,16,1,256,256,gqa,,16,174.973724,1.571,98.197,5.4798,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,16,1,256,256,gqa,,16,5.780544,0.743,23.225,1.296,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,16,1,256,256,gqa,,16,94.634018,5.809,181.584,10.1331,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,16,1,256,256,gqa,,16,5.785472,1.114,23.208,1.2951,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,16,1,256,256,gqa,,16,94.653122,8.712,181.57,10.1323,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,16,1,256,256,gqa,,16,5.709568,1.504,23.519,1.3124,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp2,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,16,1,256,256,gqa,,16,94.638657,11.618,181.62,10.135,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.199808,85.982,94.463,5.2713,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,2.032704,135.228,37.141,2.0726,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,28.456512,154.553,10.612,0.5922,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,3.26544,84.178,21.836,1.2185,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,6.670016,82.422,20.751,1.158,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,4.334752,126.825,17.417,0.9719,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,8.716672,126.139,16.36,0.913,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,0.085344,12.581,1572.763,87.7658,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,12.091424,11.367,1420.918,79.2923,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,6.111904,0.351,21.963,1.2256,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,44.562305,6.168,385.572,21.5163,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,6.109888,0.527,21.971,1.2261,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,44.609985,9.243,385.183,21.4946,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,6.118112,0.702,21.943,1.2245,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,generation,cudnn,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,44.616608,12.322,385.149,21.4927,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,8,1,256,256,gqa,,16,0.145952,117.709,129.319,7.2165,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,8,1,256,256,gqa,,16,1.10128,249.599,68.554,3.8256,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,8,1,256,256,gqa,,16,11.818464,372.134,25.552,1.4259,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,8,1,256,256,gqa,,16,2.872224,95.702,24.825,1.3853,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,8,1,256,256,gqa,,16,5.698016,96.482,24.291,1.3555,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,8,1,256,256,gqa,,16,2.874176,191.274,26.268,1.4658,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,8,1,256,256,gqa,,16,5.696288,193.022,25.035,1.397,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,8,1,256,256,gqa,,16,6.461184,0.166,20.774,1.1593,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,8,1,256,256,gqa,,16,86.405373,1.591,198.841,11.096,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,8,1,256,256,gqa,,16,5.788096,0.371,23.191,1.2942,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,8,1,256,256,gqa,,16,51.232574,5.365,335.372,18.715,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,8,1,256,256,gqa,,16,5.785984,0.557,23.201,1.2947,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,8,1,256,256,gqa,,16,51.217728,8.05,335.49,18.7215,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,8,1,256,256,gqa,,16,5.791136,0.742,23.182,1.2936,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp4,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,8,1,256,256,gqa,,16,51.188,10.74,335.705,18.7335,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.149696,57.383,70.047,3.9089,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,1.18224,116.253,35.478,1.9798,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,14.67152,149.884,11.435,0.6381,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,3.261664,42.138,21.218,1.184,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,6.672352,41.197,20.43,1.1401,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,3.263776,84.221,21.847,1.2191,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,6.674272,82.369,20.738,1.1573,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,0.084928,6.321,1580.419,88.193,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,12.139296,5.661,1415.271,78.9772,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,6.120512,0.175,21.931,1.2238,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,23.644705,5.813,726.629,40.5485,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,6.119968,0.263,21.933,1.2239,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,23.598593,8.736,728.071,40.6289,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,6.122848,0.351,21.923,1.2234,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,generation,cudnn,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,23.65424,11.621,726.38,40.5346,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,2048,2048,4,1,256,256,gqa,,16,0.138592,61.98,75.659,4.2221,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,8192,8192,4,1,256,256,gqa,,16,0.630208,218.085,66.554,3.714,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,32768,32768,4,1,256,256,gqa,,16,5.98064,367.69,28.053,1.5654,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,65536,4,1,256,256,gqa,,16,2.912768,47.185,23.76,1.3259,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,512,131072,4,1,256,256,gqa,,16,5.78672,47.502,23.557,1.3145,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,65536,4,1,256,256,gqa,,16,2.913472,94.347,24.474,1.3657,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,context,cudnn_oss,bfloat16,bfloat16,1,1024,131072,4,1,256,256,gqa,,16,5.78112,95.095,23.942,1.3361,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,1,131072,4,1,256,256,gqa,,16,6.423648,0.084,20.895,1.166,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,1,131072,4,1,256,256,gqa,,16,39.759937,1.728,432.103,24.1129,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,2,131072,4,1,256,256,gqa,,16,5.785024,0.186,23.202,1.2948,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,2,131072,4,1,256,256,gqa,,16,16.992352,8.088,1011.097,56.4228,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,3,131072,4,1,256,256,gqa,,16,5.780064,0.279,23.223,1.2959,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,3,131072,4,1,256,256,gqa,,16,16.991903,12.133,1011.155,56.426,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,1,4,131072,4,1,256,256,gqa,,16,5.781824,0.371,23.217,1.2956,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], +qwen35,qwen3.5-tp8,generation,cudnn_oss,bfloat16,bfloat16,128,4,131072,4,1,256,256,gqa,,16,16.993088,16.176,1011.115,56.4238,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=64; tile_n=64; cga=1)], diff --git a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png old mode 100644 new mode 100755 index 67863f18b..f4834ce3b Binary files a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png and b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_context.png differ diff --git a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png old mode 100644 new mode 100755 index cba17487a..4cdef05ba Binary files a/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png and b/benchmark/attention_inference/results/qwen35/rtxpro6000/qwen35_generation.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260818_160317.csv b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260819_053502.csv similarity index 61% rename from benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260818_160317.csv rename to benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260819_053502.csv index f2bb41a5f..3e1459da5 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260818_160317.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_20260819_053502.csv @@ -1,13 +1,13 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.428704,932.446,211.056,2.6382,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.157632,1045.65,136.011,1.7001,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.7648,1096.742,90.133,1.1267,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,5.913824,1081.516,61.199,0.765,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,10.549568,1089.311,45.986,0.5748,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,18.479616,1082.274,34.633,0.4329,20,True,,NVIDIA GB200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.629408,635.11,143.755,1.7969,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.547264,782.334,101.76,1.272,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,3.53024,858.943,70.59,0.8824,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,6.762272,945.821,53.521,0.669,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,11.88512,966.904,40.818,0.5102,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,20.345888,983.0,31.456,0.3932,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.423072,944.859,213.866,2.6733,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.171008,1033.706,134.457,1.6807,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.884992,1051.051,86.378,1.0797,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,6.445888,992.244,56.148,0.7018,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,11.471104,1001.801,42.292,0.5286,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,20.019457,999.028,31.969,0.3996,20,True,,NVIDIA GB200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.62304,641.602,145.224,1.8153,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.544672,783.647,101.931,1.2741,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,3.54368,855.685,70.323,0.879,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,7.065184,905.269,51.226,0.6403,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,12.40432,926.433,39.11,0.4889,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,21.213024,942.817,30.17,0.3771,20,True,,NVIDIA GB200,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png old mode 100644 new mode 100755 index d950090d3..912c49297 Binary files a/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/gb200/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260818_160308.csv b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260819_053407.csv similarity index 58% rename from benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260818_160308.csv rename to benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260819_053407.csv index b7d10b0e0..5fd6f2cb1 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260818_160308.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_20260819_053407.csv @@ -1,13 +1,13 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.347456,1150.487,260.409,3.2551,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,0.972128,1245.183,161.965,2.0246,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.360064,1284.827,105.591,1.3199,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,5.007904,1277.16,72.27,0.9034,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,8.846304,1299.047,54.84,0.6855,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,15.315744,1305.846,41.787,0.5223,20,True,,NVIDIA GB300,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.535968,745.835,168.817,2.1102,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.293184,936.044,121.754,1.5219,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.906112,1043.412,85.751,1.0719,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,5.764288,1109.573,62.787,0.7848,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,10.13984,1133.328,47.844,0.598,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,17.393728,1149.84,36.795,0.4599,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.346912,1152.291,260.817,3.2602,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,0.98592,1227.764,159.699,1.9962,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.338528,1296.659,106.563,1.332,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,4.85376,1317.72,74.565,0.9321,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,8.681728,1323.673,55.879,0.6985,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,14.99008,1334.216,42.695,0.5337,20,True,,NVIDIA GB300,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,0.532832,750.224,169.811,2.1226,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,1.283456,943.139,122.677,1.5335,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,2.890272,1049.131,86.22,1.0778,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,5.786624,1105.29,62.545,0.7818,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,9.943936,1155.656,48.787,0.6098,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,17.176064,1164.411,37.261,0.4658,20,True,,NVIDIA GB300,cudnn_oss plan=sdpa_fwd_prefill_sm100_d128[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=2)], diff --git a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png old mode 100644 new mode 100755 index 57046dfe9..c7b5e4a1d Binary files a/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/gb300/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260818_160230.csv b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260819_072541.csv similarity index 63% rename from benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260818_160230.csv rename to benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260819_072541.csv index 8686a3bbe..63ba3fda6 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260818_160230.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_20260819_072541.csv @@ -1,7 +1,7 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.081792,369.52,83.64,1.7425,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.185856,379.954,49.422,1.0296,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.152448,371.946,30.568,0.6368,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,17.008673,376.037,21.279,0.4433,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,30.198944,380.535,16.064,0.3347,20,True,,NVIDIA H200,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,52.976704,377.524,12.081,0.2517,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.077184,371.1,83.997,1.7499,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.225728,375.257,48.811,1.0169,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.395744,361.168,29.682,0.6184,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,17.649952,362.375,20.506,0.4272,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,40.589153,283.124,11.952,0.249,20,True,,NVIDIA H200,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,79.891106,250.341,8.011,0.1669,20,True,,NVIDIA H200,cudnn 9.26.0, diff --git a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png old mode 100644 new mode 100755 index 4475f9ff0..20e40fd24 Binary files a/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/h200/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260818_160141.csv b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260819_072448.csv similarity index 61% rename from benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260818_160141.csv rename to benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260819_072448.csv index db2eb6766..30ea5d353 100644 --- a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260818_160141.csv +++ b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_20260819_072448.csv @@ -1,13 +1,13 @@ config_name,model_name,phase,backend,data_type,kv_cache_dtype,batch_size,q_tokens,kv_len,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,kind,sliding_window_size,page_size,time_ms,tflops,gbps,sol_pct,num_iterations,success,error_message,gpu_name,backend_detail,cudnn_backend_version -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.64624,242.822,54.962,3.0671,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,5.01424,241.408,31.401,1.7523,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,12.196128,248.626,20.433,1.1402,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,25.696863,248.898,14.084,0.786,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,46.026592,249.677,10.54,0.5882,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,79.768639,250.725,8.023,0.4477,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.177216,339.567,76.86,4.2891,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.375904,358.564,46.639,2.6026,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.264736,366.893,30.152,1.6826,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,17.154305,372.845,21.098,1.1773,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,30.450048,377.397,15.932,0.8891,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], -qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,52.421505,381.523,12.209,0.6813,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.610592,248.197,56.178,3.135,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,5.013408,241.448,31.406,1.7526,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,12.178496,248.986,20.462,1.1419,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,24.917952,256.678,14.525,0.8105,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,44.870079,256.112,10.812,0.6033,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,78.029411,256.314,8.202,0.4577,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn 9.26.0, +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,8836,8836,16,16,80,80,gqa,,64,1.211872,329.856,74.662,4.1664,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,15376,15376,16,16,80,80,gqa,,64,3.341216,362.286,47.124,2.6297,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,24336,24336,16,16,80,80,gqa,,64,8.049024,376.726,30.96,1.7277,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,35344,35344,16,16,80,80,gqa,,64,16.696608,383.066,21.676,1.2096,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,47376,47376,16,16,80,80,gqa,,64,29.824415,385.314,16.266,0.9077,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], +qwen3vl_vit,qwen3vl_vit,context,cudnn_oss,bfloat16,bfloat16,1,62500,62500,16,16,80,80,gqa,,64,51.360226,389.406,12.461,0.6954,20,True,,NVIDIA RTX PRO 6000 Blackwell Server Edition,cudnn_oss plan=sdpa_fwd_prefill_sm120[SdpaFwdKnobs(sched_policy=0; tile_m=128; tile_n=128; cga=1)], diff --git a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png old mode 100644 new mode 100755 index cbfec9b74..976392a53 Binary files a/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png and b/benchmark/attention_inference/results/qwen3vl_vit/rtxpro6000/qwen3vl_vit_context.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260818_124333.csv b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052837.csv similarity index 62% rename from benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260818_124333.csv rename to benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052837.csv index 51d7e318a..ad0aba304 100644 --- a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260818_124333.csv +++ b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_20260819_052837.csv @@ -1,31 +1,31 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.214,1321.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.163,1728.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.166,1706.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.742,381.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.730,387.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.221,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.213,1376.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.162,1808.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.166,1765.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.740,397.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.730,402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,4201.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.220,1334.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.403,1458.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.300,1954.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.303,1934.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.741,792.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.732,802.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.401,1462.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2493.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.756,1553.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.568,2068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,3529.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.588,1996.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,3454.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.802,1463.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.734,1599.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,4184.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.030,1140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.408,1668.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.123,2091.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.162,2021.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.551,1514.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.455,1614.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.990,1180.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.209,1348.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.159,1772.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.162,1744.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.731,386.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.731,386.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.218,1293.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.213,1379.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.159,1849.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.162,1811.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.731,402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.730,402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.216,1359.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.389,1510.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.297,1977.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.298,1968.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.742,791.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.733,801.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.394,1491.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.718,1635.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.564,2080.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.583,2013.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.768,1528.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.734,1600.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.029,1141.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.446,1624.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.118,2101.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.155,2033.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.522,1543.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.453,1616.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,2.003,1172.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png old mode 100644 new mode 100755 index aabaf0aae..4d4a6d513 Binary files a/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260818_124935.csv b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052715.csv similarity index 64% rename from benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260818_124935.csv rename to benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052715.csv index ca6865740..47471669d 100644 --- a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260818_124935.csv +++ b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_20260819_052715.csv @@ -1,31 +1,31 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.156,1814.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.110,2562.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.117,2417.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.602,469.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.155,1819.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.109,2600.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.116,2438.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.596,474.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2540.000 auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.494,572.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.194,1455.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.156,1884.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.110,2672.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.117,2506.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.595,493.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.494,595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.193,1520.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.191,1475.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.157,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.109,2699.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.116,2531.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.593,495.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.493,595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.191,1540.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.295,1991.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.207,2829.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.220,2667.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.602,975.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.496,1185.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.363,1619.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.564,2081.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2316.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.394,2981.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.425,2764.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.631,1860.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2521.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.501,2345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.559,2099.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.114,2108.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.780,3009.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.843,2785.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.247,1884.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.990,2371.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.108,2120.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.205,2863.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.217,2700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.597,983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.494,1189.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,0.357,1645.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.562,2090.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.387,3033.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.418,2806.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.624,1880.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.494,2376.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,0.557,2109.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.115,2107.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.768,3056.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.833,2819.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.240,1894.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,mxfp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,0.980,2397.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,1.105,2124.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png old mode 100644 new mode 100755 index d0f8e02c1..2066a19e0 Binary files a/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260818_133357.csv b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260819_062001.csv similarity index 66% rename from benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260818_133357.csv rename to benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260819_062001.csv index 20d9ff009..898bd6b54 100644 --- a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260818_133357.csv +++ b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_20260819_062001.csv @@ -1,16 +1,16 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.723,390.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.513,551.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.645,438.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.720,408.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.511,575.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.642,457.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.466,400.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.023,574.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.321,444.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.215,530.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.528,768.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.007,585.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.750,626.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,2.545,923.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.409,689.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.716,394.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,0.508,556.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.638,443.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.716,410.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.507,578.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.636,461.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.475,398.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.021,575.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.331,441.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.254,521.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,1.542,761.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.048,573.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.822,614.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,2.591,906.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,3.486,674.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png old mode 100644 new mode 100755 index a082a27fe..572f2ee87 Binary files a/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/h200/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260818_134814.csv b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260819_062304.csv similarity index 68% rename from benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260818_134814.csv rename to benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260819_062304.csv index d67e2f664..5a14ef5d8 100644 --- a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260818_134814.csv +++ b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_20260819_062304.csv @@ -1,21 +1,21 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,1.997,141.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,1.267,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.993,284.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,1.990,142.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.998,147.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.268,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.998,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.987,148.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.844,318.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.277,460.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,2.142,274.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.998,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.683,319.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.541,462.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.538,332.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.991,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.332,320.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,5.031,467.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.041,334.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.942,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,2.000,141.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,985,62208,9,9,128,128,fwd,False,1.268,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,0.990,285.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,985,62208,9,9,128,128,fwd,False,1.987,142.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.999,147.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.269,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,0.996,295.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,1024,62208,9,9,128,128,fwd,False,1.989,148.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.846,318.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,2048,62208,9,9,128,128,fwd,False,1.284,457.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,2.020,291.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,2048,62208,9,9,128,128,fwd,False,2.001,293.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.677,319.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,4096,62208,9,9,128,128,fwd,False,2.538,463.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.536,332.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,4096,62208,9,9,128,128,fwd,False,3.990,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,cudnn,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.314,321.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,cudnn,fp8,no_mask,1,8192,62208,9,9,128,128,fwd,False,5.031,467.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +auto_regressive_dit,auto_regressive_dit,cudnn_oss,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.034,334.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +auto_regressive_dit,auto_regressive_dit,flash_attention_4,bfloat16,no_mask,1,8192,62208,9,9,128,128,fwd,False,7.938,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 diff --git a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png old mode 100644 new mode 100755 index e29a25f61..4e47ef502 Binary files a/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png and b/benchmark/attention_training/results/auto_regressive_dit/rtxpro6000/auto_regressive_dit_no_mask.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260818_123739.csv b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260819_052447.csv similarity index 67% rename from benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260818_123739.csv rename to benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260819_052447.csv index 8d753fb24..6391d6f9f 100644 --- a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260818_123739.csv +++ b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_20260819_052447.csv @@ -1,9 +1,9 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.489,157.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.692,161.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.873,155.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.439,152.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,7.010,156.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.392,161.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.688,160.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.860,155.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.319,165.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.651,165.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.829,163.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.419,159.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,6.695,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.246,168.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.638,165.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.816,163.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png old mode 100644 new mode 100755 index 7223c2488..d46ab6ebf Binary files a/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/gb200/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260818_124332.csv b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260819_052317.csv similarity index 67% rename from benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260818_124332.csv rename to benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260819_052317.csv index 43028c589..57374bc7f 100644 --- a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260818_124332.csv +++ b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_20260819_052317.csv @@ -1,9 +1,9 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.283,167.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.588,172.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.785,172.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.391,170.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,6.579,166.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.177,172.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.559,174.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.773,172.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,3.092,177.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,1.506,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,0.744,182.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,0.368,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,6.200,177.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,3.006,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,1.482,183.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,0.730,183.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png old mode 100644 new mode 100755 index 78f509219..b9c5b9e60 Binary files a/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/gb300/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260818_124629.csv b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260819_053303.csv similarity index 67% rename from benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260818_124629.csv rename to benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260819_053303.csv index f72cf752e..d0b0b675d 100644 --- a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260818_124629.csv +++ b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_20260819_053303.csv @@ -1,9 +1,9 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.944,55.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.903,56.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.393,57.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.188,56.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,19.773,55.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.785,56.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.761,57.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.354,57.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.878,55.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.868,56.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.375,57.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.178,57.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,19.753,55.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.723,56.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.738,57.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.351,57.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png old mode 100644 new mode 100755 index b7296482b..2549b1e0f Binary files a/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/h200/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260818_125724.csv b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260818_125724.csv deleted file mode 100644 index 050c9ee09..000000000 --- a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260818_125724.csv +++ /dev/null @@ -1,9 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.456,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.716,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.351,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.237,54.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,18.831,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.399,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.666,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.317,57.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260819_053909.csv b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260819_053909.csv new file mode 100644 index 000000000..af29c59c3 --- /dev/null +++ b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_20260819_053909.csv @@ -0,0 +1,9 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,16384,16384,64,1,512,512,fwd,False,9.449,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,8192,8192,64,1,512,512,fwd,False,4.730,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,4096,4096,64,1,512,512,fwd,False,2.441,55.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +deepseek_v4,dsv4_flash,cudnn,bfloat16,top_left,2,2048,2048,64,1,512,512,fwd,False,1.216,55.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,16384,16384,128,1,512,512,fwd,False,18.831,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,8192,8192,128,1,512,512,fwd,False,9.387,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,4096,4096,128,1,512,512,fwd,False,4.668,58.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +deepseek_v4,dsv4_pro,cudnn,bfloat16,top_left,2,2048,2048,128,1,512,512,fwd,False,2.402,55.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 diff --git a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png old mode 100644 new mode 100755 index 5858c4fa8..6f2b59eeb Binary files a/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png and b/benchmark/attention_training/results/deepseek_v4/rtxpro6000/deepseek_v4_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_20260818_125849.csv b/benchmark/attention_training/results/dsv3/gb200/dsv3_20260819_054000.csv similarity index 54% rename from benchmark/attention_training/results/dsv3/gb200/dsv3_20260818_125849.csv rename to benchmark/attention_training/results/dsv3/gb200/dsv3_20260819_054000.csv index 22a45012a..a670f9e0f 100644 --- a/benchmark/attention_training/results/dsv3/gb200/dsv3_20260818_125849.csv +++ b/benchmark/attention_training/results/dsv3/gb200/dsv3_20260819_054000.csv @@ -1,131 +1,131 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,50.703,1735.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,214.227,1068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,213.585,1071.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,111.295,1581.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,428.714,1067.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,422.837,1082.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,36.034,2441.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,124.151,1842.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,119.146,1920.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,75.982,2315.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,243.446,1879.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,241.692,1892.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,39.145,2247.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,144.983,1577.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,144.370,1584.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,72.799,2417.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,289.195,1582.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,298.924,1530.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,62.961,1397.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,125.537,1401.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,55.722,1579.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,215.314,1062.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,219.163,1044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,111.654,1576.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,398.471,1148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,406.441,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,12.472,1763.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,50.304,1137.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,53.400,1071.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,24.346,1806.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,104.106,1098.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,97.591,1172.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,8.645,2544.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,30.363,1883.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,28.918,1977.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,17.267,2547.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,54.760,2088.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,60.491,1890.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,9.199,2391.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,33.826,1690.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,35.171,1626.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,17.458,2519.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,67.117,1704.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,66.044,1731.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,14.062,1564.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,29.832,1474.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,12.588,1747.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,52.802,1083.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,54.765,1044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,24.175,1819.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,102.553,1115.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,106.553,1073.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.443,1597.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,12.594,1135.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,11.629,1229.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.006,1831.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,24.755,1155.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,26.375,1084.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.289,2402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.249,1972.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.312,1955.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.223,2604.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.948,2049.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.475,2121.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.415,2277.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,9.282,1540.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,8.701,1643.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.431,2481.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,16.812,1700.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,16.167,1768.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.755,1464.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.937,1585.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.483,1579.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,13.798,1036.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,13.238,1080.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.125,1795.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,24.843,1151.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,25.613,1116.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,1.018,1351.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.347,1068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.267,1094.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.640,1676.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.980,1195.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.967,1198.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.656,2096.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.084,1715.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.087,1713.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.089,2523.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.468,2061.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.480,2054.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.672,2046.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.555,1399.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.533,1411.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.155,2380.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,4.134,1729.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,4.188,1707.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,1.043,1317.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.764,1558.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,1.006,1366.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.709,964.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.840,931.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.607,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,6.119,1168.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.263,1141.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.317,1084.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.047,854.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.997,897.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.467,1473.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.677,1066.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.613,1107.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.205,1677.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.667,1340.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.672,1331.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.313,2199.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.985,1813.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,0.964,1854.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.219,1566.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.843,1060.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.837,1068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.332,2068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.176,1520.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.172,1525.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.336,1022.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.491,1400.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.319,1077.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.192,750.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.237,722.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.464,1480.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.757,1017.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.795,995.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,54.527,1613.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,206.815,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,209.620,1091.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,114.865,1532.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,418.462,1093.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,414.584,1103.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,34.847,2524.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,121.353,1885.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,121.973,1875.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,69.525,2530.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,233.177,1962.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,234.969,1947.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,40.460,2174.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,140.254,1631.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,140.361,1629.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,76.959,2286.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,284.477,1608.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,283.137,1615.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,61.808,1423.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,124.142,1417.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,52.579,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,210.398,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,212.466,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,110.435,1593.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,397.750,1150.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,397.369,1151.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,13.783,1596.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,51.369,1113.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,49.281,1160.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,26.039,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,106.657,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,102.765,1113.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,8.577,2564.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,28.196,2028.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,29.506,1938.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,17.559,2505.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,54.953,2081.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,54.993,2079.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,9.106,2415.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,33.638,1700.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,34.596,1653.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,18.367,2395.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,64.688,1768.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,64.224,1780.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,14.011,1570.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,29.166,1508.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,13.681,1607.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,52.526,1089.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,53.692,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,23.911,1839.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,101.607,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,103.043,1110.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.481,1579.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,13.374,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,12.749,1121.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.301,1745.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,25.578,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,25.108,1139.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.304,2387.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.435,1923.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.300,1958.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.268,2576.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.257,2156.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.310,2148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,2.461,2234.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,8.558,1670.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,9.249,1546.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,4.617,2381.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,15.803,1809.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,15.924,1795.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.827,1437.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.996,1572.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.617,1520.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,13.340,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,13.309,1074.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,6.193,1775.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,24.152,1184.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,23.913,1195.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.998,1377.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.463,1032.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.302,1082.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.699,1618.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.997,1192.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.993,1193.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.666,2064.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.088,1712.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.090,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.141,2410.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.496,2044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.503,2040.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.708,1942.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.557,1398.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.545,1405.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.145,2401.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,4.188,1706.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,4.179,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,1.062,1294.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.764,1559.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,1.013,1357.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.710,964.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.840,931.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.608,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,6.084,1175.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.269,1140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.322,1068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.054,848.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.987,906.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.471,1458.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.667,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.638,1090.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.210,1638.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.676,1322.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.675,1325.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.320,2151.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.992,1802.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.002,1782.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.222,1551.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.848,1054.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.846,1056.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.337,2038.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.175,1521.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.201,1488.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.340,1012.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.496,1385.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.323,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.193,749.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.237,722.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.471,1459.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.755,1018.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.804,990.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,149.000 diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png old mode 100644 new mode 100755 index 27719b760..704fb7a27 Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png old mode 100644 new mode 100755 index f866052c9..2600ba897 Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png old mode 100644 new mode 100755 index c1244be62..af38b376a Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png old mode 100644 new mode 100755 index 427ded130..5554c280b Binary files a/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb200/dsv3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_20260818_130454.csv b/benchmark/attention_training/results/dsv3/gb300/dsv3_20260819_053824.csv similarity index 53% rename from benchmark/attention_training/results/dsv3/gb300/dsv3_20260818_130454.csv rename to benchmark/attention_training/results/dsv3/gb300/dsv3_20260819_053824.csv index d723ae09a..9ec037eb6 100644 --- a/benchmark/attention_training/results/dsv3/gb300/dsv3_20260818_130454.csv +++ b/benchmark/attention_training/results/dsv3/gb300/dsv3_20260819_053824.csv @@ -1,131 +1,131 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,44.119,1994.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,173.829,1316.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,174.704,1309.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,90.481,1944.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,365.232,1252.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,351.969,1300.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,25.525,3446.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,92.824,2464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,93.200,2454.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,53.305,3300.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,185.264,2469.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,184.040,2485.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,28.766,3058.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,114.657,1995.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,115.237,1985.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,58.126,3027.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,226.915,2016.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,228.260,2004.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,48.659,1808.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,101.184,1739.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,47.678,1845.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,184.638,1239.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,191.496,1194.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,86.303,2038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,357.430,1280.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,370.257,1235.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,10.785,2039.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,44.694,1279.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,42.942,1332.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,20.851,2109.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,87.017,1314.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,84.627,1351.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,6.538,3364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,23.489,2434.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,22.705,2518.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,12.675,3470.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,43.480,2630.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,45.032,2539.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,7.309,3009.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,28.880,1980.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,29.196,1958.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,14.053,3130.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,54.570,2095.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,54.289,2106.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,12.371,1778.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,23.689,1857.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,11.487,1915.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,47.113,1214.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,48.081,1189.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,20.766,2118.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,88.892,1286.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,91.822,1245.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,2.881,1908.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,11.045,1294.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,10.626,1345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.195,2117.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,20.785,1375.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,20.523,1393.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.723,3190.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,6.107,2341.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,6.116,2337.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.181,3456.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,11.039,2590.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,11.059,2585.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.936,2839.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.777,1838.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.779,1838.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.544,3102.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.839,2066.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.983,2044.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.209,1713.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.960,1845.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.058,1798.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,12.310,1161.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,12.894,1109.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.310,2071.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,22.008,1299.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,23.045,1240.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.818,1681.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.113,1148.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,2.983,1198.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.357,2026.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.628,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.599,1277.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.485,2834.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,1.784,2004.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,1.788,1999.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.816,3367.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,2.931,2438.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,2.932,2437.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.544,2529.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.308,1548.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.304,1551.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.908,3027.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.673,1946.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.671,1947.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.885,1553.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.523,1805.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.868,1584.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.566,1002.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.737,957.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.380,1991.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.938,1204.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.148,1162.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.256,1341.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,0.978,914.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.930,961.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.364,1888.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.608,1111.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.555,1149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.151,2279.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.589,1518.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.588,1519.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.220,3121.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.844,2116.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,0.844,2116.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.168,2051.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.775,1153.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.777,1151.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.248,2775.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.069,1671.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.070,1670.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.272,1266.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.396,1734.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.275,1252.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.149,778.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.204,742.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.387,1778.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.715,1042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.764,1013.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,42.710,2060.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,171.146,1336.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,172.219,1328.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,88.301,1992.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,360.136,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,347.818,1315.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,25.932,3392.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,False,89.690,2550.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,bwd,True,91.080,2511.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,50.511,3483.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,184.480,2479.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,182.464,2507.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,fwd,False,28.480,3089.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,False,115.705,1977.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,32768,32768,128,128,192,128,bwd,True,115.849,1974.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,56.801,3097.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,False,223.500,2047.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,32768,32768,128,128,192,128,bwd,True,225.941,2024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,48.876,1800.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,103.721,1696.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,44.367,1983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,181.850,1258.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,190.938,1198.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,87.228,2017.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,353.198,1295.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,365.503,1251.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,10.642,2066.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,42.932,1332.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,40.796,1402.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,20.248,2172.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,87.126,1312.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,83.826,1364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,6.456,3407.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,False,23.126,2472.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,bwd,True,22.639,2526.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,12.460,3530.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,44.072,2595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,42.966,2661.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,fwd,False,7.230,3042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,False,28.457,2009.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,16384,16384,128,128,192,128,bwd,True,28.648,1996.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,14.052,3130.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,False,55.100,2075.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,16384,16384,128,128,192,128,bwd,True,53.516,2137.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,12.048,1825.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,23.498,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,11.408,1928.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,46.057,1241.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,47.925,1193.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,20.290,2168.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,89.876,1272.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,91.583,1249.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,2.849,1930.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,10.934,1307.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,10.611,1347.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.117,2149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,20.999,1361.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,20.412,1400.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.706,3224.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,False,6.048,2364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,bwd,True,6.051,2363.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.142,3499.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,11.013,2596.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,10.945,2612.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,fwd,False,1.903,2890.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,False,7.699,1857.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,8192,8192,128,128,192,128,bwd,True,7.701,1856.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,3.503,3139.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,False,13.705,2086.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,8192,8192,128,128,192,128,bwd,True,13.680,2090.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.186,1726.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.885,1868.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,3.041,1808.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,12.228,1169.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,12.843,1113.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,5.163,2129.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,22.006,1299.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,22.709,1259.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.812,1693.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.095,1155.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,2.967,1205.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.346,2042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.576,1282.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,5.545,1289.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.476,2885.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,False,1.763,2027.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,bwd,True,1.762,2029.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.807,3407.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,2.901,2464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,2.905,2460.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,fwd,False,0.529,2597.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,False,2.282,1566.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,4096,4096,128,128,192,128,bwd,True,2.287,1563.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,0.902,3049.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,False,3.638,1965.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,4096,4096,128,128,192,128,bwd,True,3.634,1967.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.887,1550.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.511,1820.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,0.864,1591.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,3.545,1008.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,3.716,962.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,1.363,2016.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,5.900,1211.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,6.119,1168.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.254,1353.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,0.970,922.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,0.922,969.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.370,1855.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.594,1121.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.540,1161.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.145,2369.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.581,1538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.581,1537.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.221,3109.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,0.835,2139.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,0.833,2145.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.161,2130.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,False,0.768,1164.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,top_left,2,2048,2048,128,128,192,128,bwd,True,0.768,1164.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.248,2772.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.052,1698.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn,mxfp8,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.051,1700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,299.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.272,1262.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.404,1702.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.271,1269.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,1.142,783.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,1.198,746.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,0.381,1805.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,1.703,1049.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,1.757,1017.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,149.000 diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png old mode 100644 new mode 100755 index 5c6c0b7e4..8a8cf33d7 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png old mode 100644 new mode 100755 index 3d2ade640..ad936c2c1 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png old mode 100644 new mode 100755 index 566251c11..87ea839b4 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png old mode 100644 new mode 100755 index 01888ea2b..24181b444 Binary files a/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png and b/benchmark/attention_training/results/dsv3/gb300/dsv3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_20260818_124320.csv b/benchmark/attention_training/results/dsv3/h200/dsv3_20260819_052958.csv similarity index 57% rename from benchmark/attention_training/results/dsv3/h200/dsv3_20260818_124320.csv rename to benchmark/attention_training/results/dsv3/h200/dsv3_20260819_052958.csv index 1f39a28e4..cdf067593 100644 --- a/benchmark/attention_training/results/dsv3/h200/dsv3_20260818_124320.csv +++ b/benchmark/attention_training/results/dsv3/h200/dsv3_20260819_052958.csv @@ -1,67 +1,67 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,147.794,595.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,456.609,501.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,286.173,615.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,1425.937,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,141.374,622.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,282.565,623.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,130.691,673.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,465.198,492.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,528.566,433.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,275.044,640.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,931.029,491.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,970.564,471.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,36.796,598.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,115.503,495.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,70.862,621.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,353.356,324.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,35.119,626.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,69.693,631.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,31.722,693.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,120.111,476.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,138.037,414.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,67.628,650.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,230.835,495.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,246.450,464.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,8.613,638.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,28.819,496.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,35.691,401.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,17.642,623.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,89.386,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,91.821,311.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,9.025,609.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,17.306,635.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,7.680,716.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,29.481,485.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,35.152,407.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,16.582,663.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,57.134,500.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,62.915,454.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.261,608.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.306,489.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.145,391.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.084,673.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,22.323,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,22.260,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,2.352,585.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.435,620.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.063,666.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.306,489.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.630,371.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,3.913,703.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,14.222,503.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,16.485,434.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.626,549.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.160,414.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,2.529,353.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.068,643.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,5.936,301.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,5.673,315.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.649,529.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.150,597.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.602,571.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.160,414.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,186.927,471.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,826.320,277.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,632.849,278.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,3228.439,142.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,32768,32768,128,128,192,128,fwd,False,335.371,262.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,32768,32768,128,128,192,128,fwd,False,591.781,297.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,185.498,474.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,1053.460,217.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,True,1450.583,158.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,768.688,229.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,2854.295,160.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,True,2904.269,157.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,90.119,244.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,341.276,168.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,156.006,282.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,834.072,137.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,16384,16384,128,128,192,128,fwd,False,56.326,390.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,16384,16384,128,128,192,128,fwd,False,116.986,376.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,33.634,654.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,180.539,317.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,True,224.157,255.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,149.375,294.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,495.480,231.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,True,605.317,189.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,12.624,436.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,45.975,311.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,56.457,253.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,25.689,428.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,207.832,138.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,159.682,179.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,8192,8192,128,128,192,128,fwd,False,10.948,502.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,8192,8192,128,128,192,128,fwd,False,24.097,456.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,7.907,695.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,31.082,460.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,True,35.761,400.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,16.726,657.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,73.955,387.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,True,83.165,344.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.333,589.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.571,472.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.261,386.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.224,651.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,22.554,317.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,23.094,309.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,4096,4096,128,128,192,128,fwd,False,2.396,574.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.399,625.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,2.108,652.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,7.507,476.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,True,9.626,371.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,4.015,685.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,14.526,492.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,True,16.648,429.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.638,539.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.207,405.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,2.553,350.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.083,635.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,5.898,303.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,5.643,317.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,cudnn,fp8,top_left,2,2048,2048,128,128,192,128,fwd,False,0.642,535.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,cudnn,fp8,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.141,602.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,0.611,562.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,2.193,407.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,True,2.870,311.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.036,663.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,3.738,478.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,4.710,379.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.052,653.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,3.763,475.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,True,4.677,382.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png old mode 100644 new mode 100755 index d54bc0541..77a6d70e7 Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png old mode 100644 new mode 100755 index 40e3a77cd..841f3febc Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png old mode 100644 new mode 100755 index 6c182e88f..871b3472b Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png old mode 100644 new mode 100755 index 0bbbe9912..1e6cfcd56 Binary files a/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png and b/benchmark/attention_training/results/dsv3/h200/dsv3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260818_124554.csv b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260819_052757.csv similarity index 50% rename from benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260818_124554.csv rename to benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260819_052757.csv index 8b9e68ce3..3694a08e7 100644 --- a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260818_124554.csv +++ b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_20260819_052757.csv @@ -1,41 +1,41 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,384.165,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,770.095,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,218.841,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,434.211,405.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,249.481,353.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,1427.766,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,468.624,375.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,2886.154,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,98.372,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,195.169,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,55.222,398.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,108.806,404.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,62.986,349.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,463.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,362.238,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,116.580,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,724.367,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,24.126,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,47.610,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,13.430,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,26.907,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,15.702,350.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,455.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,93.064,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,28.716,383.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,183.320,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,6.236,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,12.051,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,3.650,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,6.613,416.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,4.148,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,24.516,146.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,463.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,7.223,381.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,46.971,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.679,205.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,3.104,221.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.082,318.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.777,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.184,290.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,6.803,131.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.934,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,12.328,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,376.915,233.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,754.787,233.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,213.576,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,423.193,416.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,fwd,False,243.567,361.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,192,128,bwd,False,1424.817,161.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,fwd,False,456.789,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,32768,32768,128,128,192,128,bwd,False,2879.008,159.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,95.672,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,190.083,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,54.363,405.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,106.405,413.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,fwd,False,61.109,360.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,192,128,bwd,False,361.488,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,fwd,False,113.717,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,16384,16384,128,128,192,128,bwd,False,723.043,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,24.139,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,47.639,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,13.218,416.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,26.286,418.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,fwd,False,15.700,350.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,192,128,bwd,False,92.951,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,fwd,False,28.297,389.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,8192,8192,128,128,192,128,bwd,False,182.967,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,6.244,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,12.049,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,3.639,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,6.590,417.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,fwd,False,4.147,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,192,128,bwd,False,24.485,146.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,fwd,False,7.220,381.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,4096,4096,128,128,192,128,bwd,False,46.882,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +dsv3,dsv3,cudnn,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.737,198.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,cudnn,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,3.106,221.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.115,308.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,cudnn_oss,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.851,371.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,fwd,False,1.211,284.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,192,128,bwd,False,7.109,126.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,fwd,False,1.994,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +dsv3,dsv3,flash_attention_4,bfloat16,no_mask,2,2048,2048,128,128,192,128,bwd,False,12.320,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 diff --git a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png old mode 100644 new mode 100755 index 2a59540b3..8a7738311 Binary files a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png and b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_no_mask.png differ diff --git a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png old mode 100644 new mode 100755 index 1fac25ee9..895475a46 Binary files a/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png and b/benchmark/attention_training/results/dsv3/rtxpro6000/dsv3_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260818_123255.csv b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260819_051737.csv similarity index 66% rename from benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260818_123255.csv rename to benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260819_051737.csv index ceec9a59b..04d57def2 100644 --- a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260818_123255.csv +++ b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_20260819_051737.csv @@ -1,66 +1,66 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.642,167.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.647,188.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2223.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.722,145.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.498,183.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,91.604,7.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,91.467,7.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.669,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,107.211,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,107.209,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.423,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.611,170.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.651,188.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2306.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.727,145.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.499,183.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,91.317,8.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,91.340,8.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.559,176.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,106.970,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,106.949,6.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.430,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.818,151.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,4.677,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,28.601,24.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.824,166.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,1.811,189.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,2.373,144.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,28.625,24.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.809,169.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,1.812,189.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,2.375,144.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.751,182.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,23.564,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,23.565,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.835,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,27.681,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,27.660,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,2.213,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.912,150.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.328,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.905,43.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.416,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.900,189.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,23.589,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,23.591,15.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.782,175.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,27.627,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,4221.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,27.612,12.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,2.211,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.913,150.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.321,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.908,43.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.407,167.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2362.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.899,190.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.194,143.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.378,180.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.286,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.282,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.424,161.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,7.356,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,7.357,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.109,61.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.291,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.290,27.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.394,173.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,7.360,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,7.338,23.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.108,62.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.459,149.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.158,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.327,73.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.211,160.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.157,147.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.328,73.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.207,164.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.459,184.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.608,139.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.192,177.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.771,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.769,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.212,159.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,2.067,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,2.065,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.556,61.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.192,176.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.770,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.770,48.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.200,169.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,2.066,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,2.067,41.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.555,61.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.232,146.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.579,146.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.761,111.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.109,153.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.233,179.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.762,111.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.106,156.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.233,178.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.312,133.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.099,168.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.547,76.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.547,76.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.111,150.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.641,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.642,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.281,59.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.119,139.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.293,142.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.324,128.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.549,76.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.104,160.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.640,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.641,65.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.280,59.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.119,140.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.292,142.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.324,129.000,0.000,10,128,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png old mode 100644 new mode 100755 index baf071449..c81060675 Binary files a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png old mode 100644 new mode 100755 index db8597e13..d7c58dbf5 Binary files a/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png and b/benchmark/attention_training/results/gpt_oss/gb200/gpt_oss_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260818_123851.csv b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260819_051623.csv similarity index 65% rename from benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260818_123851.csv rename to benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260819_051623.csv index b2635601d..058e38790 100644 --- a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260818_123851.csv +++ b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_20260819_051623.csv @@ -1,66 +1,66 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.518,181.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.453,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.336,158.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.318,208.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,74.874,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,74.897,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.347,204.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,94.766,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,94.774,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.385,63.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.423,193.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2194.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,3.449,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,4.335,158.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.265,217.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,False,74.869,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,bwd,True,74.899,9.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,fwd,False,1.302,211.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,False,94.871,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,32768,32768,128,128,64,64,bwd,True,94.867,7.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,4.384,63.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,1.681,163.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,4.358,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,27.964,25.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.762,180.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,4.357,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,27.971,25.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.715,192.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,1.720,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,2.177,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.661,207.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,19.392,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,19.392,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.678,202.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,24.468,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,24.448,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.635,216.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,False,19.391,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,bwd,True,19.392,18.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5060.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,fwd,False,0.654,209.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,False,24.459,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,16384,16384,128,128,64,64,bwd,True,24.465,14.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,2.194,62.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.844,162.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.180,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.712,44.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.384,177.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.863,198.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.101,155.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.333,205.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,5.203,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,5.203,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.343,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.518,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.516,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.098,62.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2316.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,0.843,162.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,2.178,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.722,44.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.360,189.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,0.862,198.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.098,155.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.320,213.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,False,5.201,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,bwd,True,5.201,33.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.330,207.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,False,6.522,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,8192,8192,128,128,64,64,bwd,True,6.520,26.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,1.099,62.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.424,161.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.089,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.267,75.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.196,173.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.441,192.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.563,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.170,199.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.489,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.489,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.175,193.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.846,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.846,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.550,62.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.215,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.549,154.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.088,157.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,2.266,75.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.183,185.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.442,192.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.562,151.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.162,209.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.485,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.484,57.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.168,201.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,False,1.843,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,4096,4096,128,128,64,64,bwd,True,1.844,46.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.551,61.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.215,158.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.547,155.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.740,114.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.101,165.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.095,175.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.228,183.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.290,144.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.088,189.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.470,89.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.470,89.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.092,182.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.577,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.577,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.289,144.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.084,198.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.468,89.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.468,89.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.087,191.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,False,0.574,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +gpt_oss,gpt_oss,cudnn,mxfp8,top_left,2,2048,2048,128,128,64,64,bwd,True,0.575,72.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.277,60.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.111,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.278,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.310,134.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.277,150.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.309,135.000,0.000,10,128,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png old mode 100644 new mode 100755 index 377e69d37..02efc321e Binary files a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png old mode 100644 new mode 100755 index 4d7e45c84..5aeeb4b09 Binary files a/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png and b/benchmark/attention_training/results/gpt_oss/gb300/gpt_oss_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260818_122903.csv b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260819_051408.csv similarity index 63% rename from benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260818_122903.csv rename to benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260819_051408.csv index 7240fdd44..823ae5848 100644 --- a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260818_122903.csv +++ b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_20260819_051408.csv @@ -1,34 +1,34 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.047,134.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,7.296,94.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,2.975,92.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.717,101.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,6.029,114.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,6.259,110.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.025,134.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.645,94.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.471,93.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.368,100.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.010,114.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,3.119,110.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.518,132.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.826,93.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,16.733,10.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.737,93.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.685,100.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.504,113.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.557,109.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.264,128.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.917,92.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,4.635,18.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.372,91.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.352,96.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.756,112.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.785,108.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.134,124.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.464,90.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,1.326,31.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.190,88.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.014,136.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,7.232,95.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,2.946,93.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.706,101.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,6.004,114.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,6.222,110.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.011,135.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.616,95.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.448,95.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.358,101.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,3.000,114.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,3.113,110.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.511,133.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.810,94.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,16.559,10.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.721,95.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.682,100.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,1.499,114.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,1.551,110.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.262,129.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.911,93.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,4.616,18.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.367,92.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.351,96.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,0.755,112.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,0.784,108.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.133,125.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.463,90.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,1.319,32.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.187,89.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.178,93.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.386,108.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.387,107.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.403,103.000,0.000,10,128,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png old mode 100644 new mode 100755 index fdf0a63a2..84f13aa36 Binary files a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png old mode 100644 new mode 100755 index f486fb260..6606502ad Binary files a/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png and b/benchmark/attention_training/results/gpt_oss/h200/gpt_oss_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260818_122958.csv b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260818_122958.csv deleted file mode 100644 index a9f8f7ad8..000000000 --- a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260818_122958.csv +++ /dev/null @@ -1,44 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.961,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,14.284,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,3.007,91.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.918,94.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,14.604,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,14.613,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,110.809,2.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,12.967,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.483,92.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,7.102,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.510,91.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.477,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,7.223,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.237,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,27.318,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,463.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,6.437,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.757,90.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.516,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,32.001,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.763,89.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.758,90.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.589,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,3.596,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,7.013,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.190,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.395,86.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.755,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,8.509,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.410,83.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,904.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.400,85.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.800,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,1.810,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,1.859,18.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.583,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.210,79.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.881,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,2.411,17.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.212,78.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,904.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.216,77.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.901,46.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.901,46.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.571,29.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.797,52.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260819_051439.csv b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260819_051439.csv new file mode 100644 index 000000000..cadb97d2b --- /dev/null +++ b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_20260819_051439.csv @@ -0,0 +1,44 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.961,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,14.277,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,128,128,64,64,fwd,False,3.009,91.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,2.918,94.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,14.608,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,True,14.615,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,fwd,False,107.912,3.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,32768,32768,128,128,64,64,bwd,False,12.946,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.485,92.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,7.100,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,128,128,64,64,fwd,False,1.536,89.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,1.476,93.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,7.241,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,True,7.262,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,fwd,False,27.327,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,16384,16384,128,128,64,64,bwd,False,6.432,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.760,90.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.527,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,31.292,5.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,465.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,128,128,64,64,fwd,False,0.786,87.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,0.759,90.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.599,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,True,3.614,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,fwd,False,7.017,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,8192,8192,128,128,64,64,bwd,False,3.195,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.396,86.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.761,48.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,8.342,10.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,128,128,64,64,fwd,False,0.402,84.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,915.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,0.400,85.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.799,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,True,1.816,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,fwd,False,1.939,17.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,4096,4096,128,128,64,64,bwd,False,1.583,53.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.210,79.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.882,47.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,2.415,17.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,128,128,64,64,fwd,False,0.209,80.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,915.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.214,78.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.896,46.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,cudnn_oss,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,True,0.903,46.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,fwd,False,0.557,30.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +gpt_oss,gpt_oss,flash_attention_4,bfloat16,top_left,2,2048,2048,128,128,64,64,bwd,False,0.797,52.000,0.000,10,128,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png old mode 100644 new mode 100755 index a4ed2d860..1c616d4a6 Binary files a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png and b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left.png differ diff --git a/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left_det_overhead.png b/benchmark/attention_training/results/gpt_oss/rtxpro6000/gpt_oss_top_left_det_overhead.png old mode 100644 new mode 100755 diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260818_124112.csv b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260819_052610.csv similarity index 57% rename from benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260818_124112.csv rename to benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260819_052610.csv index c0149272a..33825452c 100644 --- a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260818_124112.csv +++ b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_20260819_052610.csv @@ -1,131 +1,131 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,25.241,1743.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,107.597,1063.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,102.701,1113.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,1335.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,55.030,1598.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,217.583,1051.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,207.979,1100.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,17.497,2514.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,61.052,1873.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,55.221,2071.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,35.971,2445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,119.822,1909.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,115.253,1984.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,18.851,2333.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,73.627,1553.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,70.530,1621.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,37.653,2336.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,137.867,1659.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,137.487,1663.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,31.366,1402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,64.237,1369.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,27.329,1609.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,105.506,1084.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,108.815,1051.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,52.567,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,208.327,1098.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,209.888,1090.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.353,1731.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2194.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,25.558,1119.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,22.761,1256.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,13.438,1636.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,52.532,1088.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,47.138,1213.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.298,2558.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,14.144,2021.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,13.682,2090.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.241,2668.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,28.176,2029.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,26.836,2130.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.526,2429.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,16.719,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,17.625,1622.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.797,2500.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,33.663,1698.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,32.630,1752.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,7.089,1551.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,14.446,1522.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.465,1701.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,26.282,1088.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,26.637,1073.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,12.272,1792.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,50.138,1140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,49.546,1154.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.717,1602.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.963,1199.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.800,1232.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.132,1755.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.036,1295.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.884,1203.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.161,2368.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.640,1963.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.645,1961.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.122,2591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.520,2192.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.549,2183.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.227,2241.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,4.341,1647.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,4.342,1646.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.220,2477.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,8.094,1766.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,7.924,1804.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.896,1450.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.460,1589.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.733,1586.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.462,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.689,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.144,1748.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,12.366,1156.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.789,1212.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.508,1354.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.682,1062.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.661,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.833,1649.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.983,1198.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.980,1199.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.340,2019.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.068,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.064,1680.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.562,2446.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.753,2038.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.749,2043.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.359,1912.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.304,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.311,1364.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.612,2246.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.108,1695.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.090,1710.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.543,1266.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.905,1519.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.516,1331.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.889,946.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.955,914.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.848,1622.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,3.078,1161.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.148,1135.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.168,1024.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.534,837.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.522,857.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.240,1434.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.867,1030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.836,1068.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.111,1543.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.353,1266.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.351,1271.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.167,2053.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.522,1711.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.521,1714.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.118,1453.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.440,1015.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.441,1014.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.175,1966.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.630,1418.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.628,1422.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.178,968.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.256,1340.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.169,1015.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.623,718.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.642,696.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.243,1417.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.909,983.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.928,963.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,26.666,1649.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,101.670,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,96.032,1191.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,1746.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,50.706,1735.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,209.828,1090.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,202.150,1131.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,16.432,2677.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,59.456,1923.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,4762.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,57.824,1978.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,35.239,2496.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,4968.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,112.962,2025.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,114.972,1989.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,17.820,2468.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,66.848,1711.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,68.123,1679.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,36.537,2407.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,138.103,1656.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,138.804,1648.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,25.829,1703.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,57.760,1523.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,27.044,1626.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,104.626,1093.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,106.922,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,49.908,1762.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,198.655,1151.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,203.649,1123.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.126,1795.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,22.533,1269.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,22.475,1272.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,12.412,1772.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,48.551,1178.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,47.074,1215.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.149,2650.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,13.474,2122.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,14.161,2019.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.093,2717.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,26.296,2174.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,26.627,2147.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,4.489,2450.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,15.665,1825.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,15.681,1823.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,8.453,2601.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,31.253,1829.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,30.918,1849.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.693,1643.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,14.461,1521.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.225,1766.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,24.763,1154.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,25.663,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,12.577,1748.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,48.063,1190.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,48.432,1181.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.705,1613.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.751,1243.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.651,1265.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.086,1781.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.519,1241.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.570,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.125,2444.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.584,1994.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.580,1996.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.035,2701.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.342,2254.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.344,2253.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,1.209,2274.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,4.226,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,4.226,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.194,2506.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,7.589,1883.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,7.588,1884.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.800,1528.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.249,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.719,1599.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.451,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.681,1070.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,3.008,1828.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.776,1214.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,12.039,1187.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.498,1381.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.656,1079.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.613,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.814,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.927,1221.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.881,1240.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.327,2101.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.056,1692.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.058,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.562,2447.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.738,2056.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.712,2087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.348,1974.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.277,1400.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.282,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.589,2334.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.032,1759.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.045,1747.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.519,1326.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.864,1592.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.495,1389.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.890,945.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.959,912.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.813,1690.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,3.079,1161.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.163,1130.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.163,1053.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.520,860.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.511,875.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.231,1484.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.856,1044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.836,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.105,1630.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.348,1282.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.350,1277.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.161,2140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.511,1748.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.512,1745.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.113,1515.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.435,1026.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.436,1026.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.169,2031.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.617,1448.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.617,1448.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.171,1004.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.243,1415.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.163,1053.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.619,722.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.643,695.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.232,1479.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.913,979.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.935,955.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png old mode 100644 new mode 100755 index 56748ed2a..1e608c1d7 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png old mode 100644 new mode 100755 index a765f02d2..823b204b9 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png old mode 100644 new mode 100755 index 89a57cf91..d66095745 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png old mode 100644 new mode 100755 index e0c4f4908..5f6437d81 Binary files a/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb200/kimiK26_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260818_124710.csv b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260819_052447.csv similarity index 58% rename from benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260818_124710.csv rename to benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260819_052447.csv index 690f2234a..3c7005eb1 100644 --- a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260818_124710.csv +++ b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_20260819_052447.csv @@ -1,131 +1,131 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,21.051,2089.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,86.031,1329.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,87.310,1310.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,43.052,2043.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,185.917,1230.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,175.926,1300.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,12.784,3440.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,4388.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,45.000,2541.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,44.323,2580.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,24.981,3521.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,91.134,2509.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,91.860,2490.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,14.154,3107.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,58.991,1938.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,56.831,2012.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,28.691,3066.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,112.009,2042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,112.350,2036.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,24.410,1802.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,49.740,1768.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,22.372,1966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,91.498,1250.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,96.370,1187.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,42.210,2084.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,178.694,1280.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,185.574,1232.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.335,2061.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,21.260,1345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,20.962,1364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.126,2172.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,42.801,1336.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,39.594,1444.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.243,3391.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,11.348,2519.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,11.243,2543.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,6.217,3537.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,21.703,2634.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,21.690,2636.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.635,3025.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,14.244,2007.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,14.283,2002.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,7.002,3141.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,27.453,2083.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,27.642,2068.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.062,1814.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,11.976,1836.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.677,1937.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,22.753,1257.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,24.057,1188.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.266,2142.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,44.164,1295.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,44.484,1285.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.423,1932.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.486,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.275,1355.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.589,2123.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,10.393,1375.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,9.993,1430.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.857,3207.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.057,2338.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.061,2335.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.572,3498.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,5.462,2617.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,5.454,2621.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.963,2853.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.899,1833.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.897,1834.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.761,3122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.838,2090.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.837,2091.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.590,1729.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.950,1864.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.526,1802.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.194,1154.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.514,1097.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.581,2130.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,11.037,1295.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.505,1242.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.412,1669.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.555,1149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.507,1186.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.682,2016.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.831,1262.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.821,1267.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.246,2793.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,0.905,1974.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,0.904,1978.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.415,3309.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.474,2424.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.470,2430.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.275,2502.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.174,1523.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.174,1522.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.467,2946.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.850,1932.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.850,1932.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.441,1558.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.759,1810.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.438,1570.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.817,983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.902,940.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.698,1969.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.995,1193.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.095,1155.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.133,1289.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.496,901.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.479,934.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.189,1814.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.837,1067.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.805,1110.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.080,2158.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.305,1464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.305,1465.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.114,3005.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.440,2030.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.441,2026.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.088,1951.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.400,1116.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.401,1113.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.129,2673.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.558,1602.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.561,1592.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,20.703,2124.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2465.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,82.079,1393.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,887.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,86.623,1320.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,42.858,2052.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,187.010,1223.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,179.692,1273.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,12.637,3480.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,False,45.083,2537.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,bwd,True,45.379,2520.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,25.447,3457.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,91.473,2500.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,89.252,2562.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,fwd,False,14.126,3113.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,False,56.391,2028.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,32768,32768,64,64,192,128,bwd,True,57.028,2005.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,28.631,3072.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,False,111.575,2050.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,32768,32768,64,64,192,128,bwd,True,115.225,1985.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,23.464,1874.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,49.784,1767.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,23.050,1908.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,90.084,1269.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,95.219,1201.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,44.091,1995.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,175.537,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,183.027,1250.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.290,2079.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,21.249,1345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,20.429,1399.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.064,2185.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,42.850,1334.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,42.074,1359.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.201,3435.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,False,11.219,2548.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,bwd,True,11.174,2558.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,6.193,3551.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,21.518,2657.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,21.514,2658.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,fwd,False,3.584,3068.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,False,14.394,1986.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,16384,16384,64,64,192,128,bwd,True,14.122,2024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,6.910,3182.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,False,27.157,2105.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,16384,16384,64,64,192,128,bwd,True,27.057,2113.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,6.009,1830.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,11.611,1894.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,5.645,1948.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,22.721,1258.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,23.974,1192.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,10.119,2173.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,43.601,1311.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,45.239,1264.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.410,1949.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,5.430,1316.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,5.216,1370.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.559,2148.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,10.381,1377.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,10.121,1412.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.847,3244.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.024,2364.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.023,2365.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.562,3521.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,5.407,2644.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,5.420,2637.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,fwd,False,0.946,2906.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,False,3.851,1856.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,8192,8192,64,64,192,128,bwd,True,3.854,1855.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,1.742,3156.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,False,6.787,2106.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,8192,8192,64,64,192,128,bwd,True,6.829,2093.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.592,1727.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.947,1866.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,1.516,1814.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,6.162,1160.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,6.488,1102.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,2.556,2151.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,10.971,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,11.455,1248.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.408,1684.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.542,1159.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.495,1196.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.680,2022.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.812,1271.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,2.787,1282.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.240,2870.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,False,0.893,2001.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,bwd,True,0.893,2001.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.413,3324.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.452,2462.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.455,2457.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,fwd,False,0.265,2595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,False,1.161,1540.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,4096,4096,64,64,192,128,bwd,True,1.160,1540.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.464,2965.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,False,1.825,1958.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,4096,4096,64,64,192,128,bwd,True,1.827,1956.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.443,1552.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.762,1803.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,0.434,1584.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,1.807,989.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,1.894,944.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,0.693,1983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,2.979,1199.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,3.082,1159.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.132,1306.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.492,909.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.473,944.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.188,1827.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.830,1076.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.797,1121.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.076,2247.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.301,1485.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.301,1485.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.114,3024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.434,2058.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.434,2058.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.084,2040.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,False,0.397,1126.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,top_left,2,2048,2048,64,64,192,128,bwd,True,0.397,1127.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.128,2684.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.549,1626.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +kimiK26,kimiK26,cudnn,mxfp8,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.549,1626.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.139,1236.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.203,1693.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.142,1207.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.596,749.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.625,715.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.193,1777.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.885,1009.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.914,977.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.202,1700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.141,1221.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,0.592,754.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,0.621,720.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.192,1793.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,0.881,1014.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,0.910,981.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png old mode 100644 new mode 100755 index 8b372c23f..9f35fd735 Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png old mode 100644 new mode 100755 index eb2b521ce..217a7df02 Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png old mode 100644 new mode 100755 index d89005e18..8c78f7b0d Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png old mode 100644 new mode 100755 index e8ae57ba8..feb639604 Binary files a/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png and b/benchmark/attention_training/results/kimiK26/gb300/kimiK26_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260818_132228.csv b/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260819_061004.csv similarity index 58% rename from benchmark/attention_training/results/kimiK26/h200/kimiK26_20260818_132228.csv rename to benchmark/attention_training/results/kimiK26/h200/kimiK26_20260819_061004.csv index 76edecc68..94c73fefb 100644 --- a/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260818_132228.csv +++ b/benchmark/attention_training/results/kimiK26/h200/kimiK26_20260819_061004.csv @@ -1,69 +1,69 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,72.379,608.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,225.108,508.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,141.197,623.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,712.503,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,69.918,629.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,140.808,625.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,64.192,685.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,236.068,484.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,265.510,431.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,136.675,644.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,461.860,495.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,486.338,470.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,17.952,612.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,55.794,512.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,70.227,407.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,35.357,622.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,176.653,324.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,182.094,314.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,17.517,628.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,34.452,638.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,16.288,675.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,58.114,492.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,67.323,425.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,33.379,659.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,116.286,492.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,122.055,468.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,4.320,636.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,14.335,499.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,17.403,411.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.358,658.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,43.925,325.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,44.145,324.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,4.462,616.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.793,625.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,3.817,720.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,14.416,496.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,17.440,410.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,7.628,721.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,28.231,506.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,31.445,455.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.150,598.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.703,483.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.574,391.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.057,668.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,11.279,317.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,10.959,326.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,1.163,591.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.258,609.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.032,666.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.700,483.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.839,369.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,1.963,700.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,6.687,534.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,8.186,437.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.328,525.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,1.116,400.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.311,341.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.546,630.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,3.042,294.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.879,310.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.325,528.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.588,584.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.305,563.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,1.110,402.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.446,309.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.529,650.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,1.914,467.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.326,384.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,83.252,528.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,374.590,305.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,235.033,374.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,1416.289,161.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,32768,32768,64,64,192,128,fwd,False,115.263,382.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,32768,32768,64,64,192,128,fwd,False,273.942,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,81.095,542.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,427.354,268.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,True,547.450,209.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,283.990,310.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,1278.858,179.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,True,1363.572,168.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,29.780,369.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,117.250,244.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,186.909,153.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,67.434,326.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,433.937,132.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,482.690,118.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,16384,16384,64,64,192,128,fwd,False,29.194,377.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,16384,16384,64,64,192,128,fwd,False,57.371,383.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,16.642,661.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,73.725,388.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,True,79.823,358.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,47.088,467.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,200.390,285.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,True,220.629,259.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,4.468,615.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,15.398,464.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,20.768,344.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.493,647.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,57.960,247.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,58.220,246.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,8192,8192,64,64,192,128,fwd,False,4.630,594.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,8192,8192,64,64,192,128,fwd,False,8.672,634.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,3.913,703.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,15.001,476.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,True,17.575,407.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,7.841,701.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,29.736,481.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,True,31.668,451.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.177,584.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.816,468.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.639,385.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.096,656.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,11.281,317.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,11.132,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,4096,4096,64,64,192,128,fwd,False,1.179,583.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,4096,4096,64,64,192,128,fwd,False,2.219,619.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.052,653.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,3.776,473.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,True,4.829,370.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,1.996,689.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,6.861,521.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,True,8.217,435.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.328,524.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,1.133,395.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.318,339.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.553,621.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,3.012,297.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.870,311.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,cudnn,fp8,top_left,2,2048,2048,64,64,192,128,fwd,False,0.322,534.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,cudnn,fp8,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.583,590.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.308,557.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,1.123,398.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,True,1.444,309.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.535,642.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,1.909,468.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,True,2.319,385.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png old mode 100644 new mode 100755 index c4f81fa67..5942fca9c Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png old mode 100644 new mode 100755 index ad21e57f0..0117d9cd7 Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png old mode 100644 new mode 100755 index fb067c094..65bcdb0fa Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png old mode 100644 new mode 100755 index a892fc79c..3a17b1dcc Binary files a/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png and b/benchmark/attention_training/results/kimiK26/h200/kimiK26_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260818_133419.csv b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260818_133419.csv deleted file mode 100644 index ec5a79e7b..000000000 --- a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260818_133419.csv +++ /dev/null @@ -1,41 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,191.584,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,383.331,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,108.950,404.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,216.258,407.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,124.522,353.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,715.385,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,232.797,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,1443.942,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,48.410,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,96.223,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,27.287,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,54.177,406.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,30.964,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,181.398,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,58.112,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,363.536,157.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,12.197,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,23.926,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,6.668,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,13.006,423.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,7.977,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,46.636,153.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,14.168,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,455.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,91.990,155.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,3.204,215.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,6.114,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.868,368.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.334,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,2.148,320.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,12.409,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.636,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,23.587,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,463.000 -kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.909,189.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,1.577,218.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.594,289.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.910,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.666,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,3.464,129.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.992,346.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,6.220,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260819_061126.csv b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260819_061126.csv new file mode 100644 index 000000000..ea846dbfa --- /dev/null +++ b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_20260819_061126.csv @@ -0,0 +1,41 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,188.939,233.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,377.618,233.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,106.745,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,210.920,417.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,fwd,False,122.310,360.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,32768,32768,64,64,192,128,bwd,False,713.673,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,fwd,False,227.313,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,64,192,128,bwd,False,1440.309,159.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,47.665,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,94.712,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,26.447,416.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,52.909,416.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,fwd,False,30.809,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,16384,16384,64,64,192,128,bwd,False,181.095,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,fwd,False,56.250,391.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,64,192,128,bwd,False,362.733,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,12.221,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,23.931,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,6.655,413.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,12.811,429.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,fwd,False,7.975,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,8192,8192,64,64,192,128,bwd,False,46.567,153.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,fwd,False,14.177,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,64,192,128,bwd,False,91.849,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,3.215,214.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,6.109,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,1.932,356.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.323,414.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,fwd,False,2.235,308.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,4096,4096,64,64,192,128,bwd,False,12.365,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,fwd,False,3.637,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,64,192,128,bwd,False,23.553,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,cudnn,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.919,187.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,cudnn,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,1.640,210.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.611,281.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,0.949,362.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,fwd,False,0.652,264.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,top_left,2,2048,2048,64,64,192,128,bwd,False,3.622,123.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,fwd,False,1.018,337.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimiK26,kimiK26,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,64,192,128,bwd,False,6.221,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png old mode 100644 new mode 100755 index 304291b31..6c47a96ec Binary files a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png and b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_no_mask.png differ diff --git a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png old mode 100644 new mode 100755 index c6cae4f92..99826295d Binary files a/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png and b/benchmark/attention_training/results/kimiK26/rtxpro6000/kimiK26_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260818_123442.csv b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260819_052144.csv similarity index 57% rename from benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260818_123442.csv rename to benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260819_052144.csv index 404f094ac..a5e70a854 100644 --- a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260818_123442.csv +++ b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_20260819_052144.csv @@ -1,121 +1,121 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,39.868,1655.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,156.318,1097.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,153.604,1117.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,86.626,1523.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,316.630,1083.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,307.681,1115.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,25.968,2541.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,91.225,1880.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,87.845,1953.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,52.661,2506.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,180.944,1896.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,179.095,1915.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,27.685,2383.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,106.982,1603.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,106.209,1615.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,56.524,2334.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,200.797,1708.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,204.158,1680.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,40.451,1631.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,161.465,1062.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,161.057,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,81.944,1610.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,298.865,1148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,302.017,1136.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.814,1681.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,36.541,1174.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,36.305,1181.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,20.147,1637.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,81.985,1046.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,78.189,1097.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.390,2581.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,21.013,2041.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,21.102,2032.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,13.599,2426.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,42.085,2038.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,41.828,2050.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.764,2439.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,26.245,1634.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,23.561,1820.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,12.914,2554.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,50.927,1684.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,47.612,1801.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.897,1667.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,39.448,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,40.055,1071.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,19.250,1714.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,76.728,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,75.106,1142.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.559,1611.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.346,1147.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.360,1145.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.640,1777.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,18.303,1171.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,17.928,1196.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.721,2397.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.423,1977.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,4333.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.421,1978.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.146,2621.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,10.166,2109.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,10.420,2058.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.848,2232.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,6.445,1663.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,6.437,1666.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.282,2513.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,11.827,1813.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,11.614,1846.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.573,1603.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.673,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,10.023,1070.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.634,1779.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,17.938,1195.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,18.916,1133.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.763,1351.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.477,1082.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.423,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.227,1681.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.441,1207.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.414,1214.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.495,2083.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.582,1695.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.567,1711.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.831,2482.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.584,2074.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.577,2080.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.528,1953.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.900,1411.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.899,1412.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.894,2306.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,3.089,1735.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,3.087,1737.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.741,1391.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.796,959.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.897,925.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.248,1652.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.581,1170.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.739,1131.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.241,1070.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.784,855.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.747,898.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.355,1454.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.298,1032.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.228,1092.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.159,1626.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.514,1304.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.512,1309.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.245,2104.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.742,1805.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.754,1777.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.167,1543.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.639,1049.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.639,1049.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.259,1986.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.909,1474.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.910,1473.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.242,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.905,741.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.936,716.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.365,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.347,995.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.381,970.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,37.207,1773.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,160.134,1071.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,153.937,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,81.276,1623.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,307.124,1117.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,300.337,1142.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,25.680,2569.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,88.571,1937.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,88.423,1940.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,50.614,2607.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,169.029,2030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,173.050,1982.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,27.080,2436.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,101.852,1684.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,102.592,1672.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,54.897,2403.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,203.193,1688.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,202.912,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,39.439,1673.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,154.031,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,156.070,1099.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,79.776,1654.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,288.242,1190.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,299.464,1146.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.460,1743.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,31.963,1342.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,34.377,1247.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,19.022,1734.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,73.686,1164.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,75.515,1136.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.277,2627.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,20.869,2055.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,20.726,2069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,12.625,2613.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,38.113,2250.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,40.744,2105.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,6.837,2412.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,24.864,1725.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,24.019,1785.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,13.296,2481.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,45.680,1877.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,45.848,1871.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,9.373,1760.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,37.467,1145.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,38.421,1116.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,17.558,1879.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,73.865,1161.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,74.679,1148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.554,1615.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,8.683,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,8.579,1250.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.523,1823.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,18.369,1167.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,16.769,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.668,2472.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.380,1993.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.381,1993.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.086,2672.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,9.611,2231.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,9.575,2239.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.793,2300.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,6.349,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,6.350,1689.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.286,2509.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,12.069,1776.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,11.497,1865.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.576,1601.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.645,1112.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.968,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,4.474,1843.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,17.873,1200.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,18.221,1177.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.737,1398.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.479,1081.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.399,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.219,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.332,1237.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.375,1225.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.483,2136.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.566,1712.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.566,1711.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.830,2484.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.547,2104.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.547,2104.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.513,2009.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.897,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.898,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.877,2350.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,3.062,1750.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,3.063,1750.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.735,1403.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.803,956.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.901,924.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.227,1680.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.591,1167.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.763,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.236,1092.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.768,873.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.744,901.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.348,1480.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.275,1051.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.220,1099.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.154,1675.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.511,1312.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.510,1313.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.241,2135.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.745,1799.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.745,1798.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.164,1572.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.635,1055.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.637,1053.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.254,2030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.902,1486.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.903,1483.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,5135.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.237,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.906,740.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.940,713.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.356,1446.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.352,991.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.388,965.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600,2568.000 diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png old mode 100644 new mode 100755 index 3e59f9125..82eccc889 Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png old mode 100644 new mode 100755 index 66553d4a1..eef6e73cc Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png old mode 100644 new mode 100755 index c51581856..7b17b8618 Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png old mode 100644 new mode 100755 index 5ffc426af..9b6c6113a Binary files a/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb200/kimi_k3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260818_124028.csv b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260819_052012.csv similarity index 58% rename from benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260818_124028.csv rename to benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260819_052012.csv index 85ca7e797..84214fe1b 100644 --- a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260818_124028.csv +++ b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_20260819_052012.csv @@ -1,121 +1,121 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,32.293,2043.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,130.780,1312.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,131.587,1304.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,64.415,2048.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,275.080,1247.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,263.949,1300.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,19.186,3439.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,68.617,2500.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,69.002,2486.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,38.719,3408.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,138.501,2477.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,138.941,2469.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,21.458,3074.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,86.370,1986.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,86.781,1977.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,42.949,3072.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,170.203,2016.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,171.801,1997.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,34.314,1923.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,136.573,1256.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,144.869,1184.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,68.311,1931.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,268.970,1275.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,277.985,1234.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.082,2041.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,32.474,1321.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,30.872,1389.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.434,2137.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,66.148,1297.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,62.646,1369.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,4.873,3384.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,17.049,2515.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,16.828,2548.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,9.357,3525.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,32.559,2634.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,32.845,2611.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,5.450,3026.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,21.253,2018.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,21.628,1983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,10.522,3135.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,41.151,2084.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,41.069,2088.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.526,1934.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,34.629,1238.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,36.010,1191.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.947,2068.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,65.555,1308.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,70.066,1224.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.137,1930.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,8.199,1308.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,8.012,1338.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.880,2125.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,15.981,1342.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,15.555,1378.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.284,3211.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,4.565,2349.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,4.561,2351.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.374,3474.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,8.176,2623.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,8.227,2606.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.440,2864.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.805,1847.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.807,1846.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.654,3107.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,10.349,2072.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,10.238,2094.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.282,1807.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.221,1163.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.682,1107.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.925,2101.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,16.640,1289.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,17.134,1251.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.613,1681.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,32.690,2018.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,134.621,1274.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,128.157,1338.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,63.940,2064.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,272.241,1260.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,264.532,1297.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,19.088,3456.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,False,69.538,2467.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,bwd,True,68.788,2494.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,37.854,3486.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,138.369,2479.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,138.286,2481.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,fwd,False,21.512,3067.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,False,85.655,2003.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,32768,32768,96,96,192,128,bwd,True,85.465,2007.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,42.132,3132.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,False,171.514,2000.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,32768,32768,96,96,192,128,bwd,True,171.442,2001.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,34.358,1920.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,137.701,1246.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,143.286,1197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,64.737,2038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,267.039,1285.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,276.013,1243.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.118,2032.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,31.984,1341.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,30.726,1396.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.149,2177.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,65.819,1303.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,60.311,1422.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,4.847,3403.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,False,17.075,2512.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,bwd,True,17.225,2490.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,9.366,3522.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,32.348,2651.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,32.741,2619.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,fwd,False,5.429,3038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,False,21.252,2018.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,16384,16384,96,96,192,128,bwd,True,21.487,1996.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,10.491,3144.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,False,40.387,2123.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,16384,16384,96,96,192,128,bwd,True,40.713,2106.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,8.542,1931.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,34.341,1249.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,35.820,1197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,15.840,2082.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,65.511,1309.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,68.706,1248.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.150,1918.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,8.201,1307.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,7.901,1357.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.891,2119.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,15.939,1345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,15.411,1391.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.281,3219.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,False,4.556,2353.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,bwd,True,4.558,2352.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.381,3464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,8.228,2606.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,8.213,2611.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,fwd,False,1.432,2880.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,False,5.801,1848.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,8192,8192,96,96,192,128,bwd,True,5.803,1848.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,2.661,3099.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,False,10.332,2075.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,8192,8192,96,96,192,128,bwd,True,10.207,2101.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,2.286,1804.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,9.192,1166.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,9.656,1110.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,3.933,2097.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,16.443,1304.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,17.071,1256.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.615,1678.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.325,1153.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.233,1201.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.017,2027.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.225,1269.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.189,1280.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.363,2840.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.338,2003.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.339,2003.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.618,3336.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.195,2442.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.192,2445.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.407,2531.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.733,1547.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.734,1546.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.694,2969.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.762,1941.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.761,1942.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.648,1590.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.688,997.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.813,953.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.052,1961.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.476,1198.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.646,1154.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.195,1320.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.733,914.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.703,954.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.283,1820.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.244,1077.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.169,1146.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.115,2240.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.447,1500.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.447,1501.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.173,2976.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.240,1197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.032,1999.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.227,1268.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.174,1284.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.357,2886.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.337,2005.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.337,2005.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.623,3309.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.198,2439.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.198,2439.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,fwd,False,0.398,2593.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,False,1.727,1552.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,4096,4096,96,96,192,128,bwd,True,1.730,1549.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,0.698,2952.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,False,2.756,1945.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,4096,4096,96,96,192,128,bwd,True,2.758,1943.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,0.652,1580.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,2.675,1002.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,2.803,956.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,1.051,1962.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,4.458,1202.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,4.623,1159.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.194,1331.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.730,919.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.699,959.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.287,1795.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.232,1087.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.163,1152.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.112,2310.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.443,1514.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.443,1513.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.174,2966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.641,2089.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.641,2090.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.127,2023.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.587,1141.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.588,1140.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.195,2644.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.816,1643.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.816,1643.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.208,1242.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.872,769.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.916,732.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.295,1749.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.315,1019.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.358,987.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.641,2091.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.123,2092.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,False,0.584,1148.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,top_left,2,2048,2048,96,96,192,128,bwd,True,0.584,1149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.195,2645.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,False,0.812,1650.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,cudnn,mxfp8,no_mask,2,2048,2048,96,96,192,128,bwd,True,0.811,1652.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,5155.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.206,1251.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,0.866,774.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,0.907,739.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.299,1725.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,1.308,1024.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,1.350,993.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600,2578.000 diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png old mode 100644 new mode 100755 index cf9e6e8bb..828024a30 Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png old mode 100644 new mode 100755 index f178ac351..38cd5e858 Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png old mode 100644 new mode 100755 index 9f19c1cc2..1d281574b Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png old mode 100644 new mode 100755 index 5135beb4a..dc88d376c Binary files a/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/gb300/kimi_k3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260818_130936.csv b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260819_055654.csv similarity index 57% rename from benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260818_130936.csv rename to benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260819_055654.csv index 56d9dc617..5252943e0 100644 --- a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260818_130936.csv +++ b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_20260819_055654.csv @@ -1,67 +1,67 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,111.120,594.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,338.906,506.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,210.655,626.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,1068.200,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,106.484,620.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,211.563,624.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,96.836,681.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,351.070,489.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,396.560,433.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,204.334,646.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,693.496,495.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,727.878,471.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,26.703,618.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,86.602,495.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,53.002,622.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,266.713,322.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,26.218,629.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,52.074,633.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,23.627,698.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,89.028,482.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,102.673,418.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,49.954,660.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,174.275,492.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,184.718,464.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,6.459,638.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,21.627,496.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,26.268,408.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,12.755,647.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,66.684,322.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,66.944,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,6.901,598.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,13.008,634.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,5.735,719.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,21.691,494.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,26.402,406.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,12.026,686.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,42.226,508.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,47.339,453.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.708,604.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,5.502,487.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,6.793,395.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.077,670.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,16.858,318.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,16.262,330.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,1.757,587.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.329,619.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.547,667.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,5.508,487.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,7.235,371.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,2.951,699.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,10.546,508.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,12.382,433.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.476,542.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.636,410.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,1.918,349.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.810,636.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,4.509,297.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,4.292,312.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.488,528.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.869,593.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.451,571.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.639,409.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,2.162,310.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.787,655.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,2.836,473.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,3.496,383.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,135.947,485.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,557.246,308.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,394.930,334.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,2302.826,149.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,32768,32768,96,96,192,128,fwd,False,199.020,331.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,32768,32768,96,96,192,128,fwd,False,438.664,301.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,144.996,455.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,834.939,205.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,True,1072.448,160.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,580.278,227.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,2105.620,163.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,True,2176.448,158.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,43.200,382.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,322.614,133.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,132.978,248.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,651.036,132.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,16384,16384,96,96,192,128,fwd,False,44.585,370.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,16384,16384,96,96,192,128,fwd,False,80.370,410.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,27.122,608.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,160.891,267.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,True,192.011,223.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,82.470,400.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,432.584,198.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,True,499.876,172.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,6.737,612.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,33.508,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,38.482,279.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,17.933,460.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,117.734,182.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,128.668,167.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,8192,8192,96,96,192,128,fwd,False,7.020,587.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,8192,8192,96,96,192,128,fwd,False,13.572,608.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,5.930,695.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,23.044,465.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,True,26.908,398.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,12.730,648.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,54.880,391.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,True,59.874,358.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.759,586.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,5.706,470.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,6.932,387.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.176,649.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,16.979,316.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,16.976,316.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,4096,4096,96,96,192,128,fwd,False,1.791,576.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.301,625.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,1.586,650.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,5.669,473.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,True,7.217,371.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,3.022,682.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,10.699,501.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,True,12.421,432.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.482,535.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.669,402.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,1.937,346.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.824,625.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,4.479,299.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,4.264,314.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,cudnn,fp8,top_left,2,2048,2048,96,96,192,128,fwd,False,0.483,534.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,cudnn,fp8,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.865,596.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.457,564.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,1.662,403.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,True,2.151,312.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,0.799,645.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,2.825,474.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,True,3.493,384.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png old mode 100644 new mode 100755 index 576830e48..c02c825e6 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png old mode 100644 new mode 100755 index 608438e5f..fe0757518 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png old mode 100644 new mode 100755 index 80fae7d89..34ab75351 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png old mode 100644 new mode 100755 index bd58a70ce..b1458a250 Binary files a/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png and b/benchmark/attention_training/results/kimi_k3/h200/kimi_k3_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260818_132057.csv b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260819_055938.csv similarity index 54% rename from benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260818_132057.csv rename to benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260819_055938.csv index 91d886bfa..f7c734f57 100644 --- a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260818_132057.csv +++ b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_20260819_055938.csv @@ -1,31 +1,31 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,289.060,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,578.919,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,187.301,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,1073.056,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,350.471,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,2163.711,159.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,73.937,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,146.412,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,47.474,347.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,271.405,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,87.023,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,544.475,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,18.251,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,35.977,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,11.849,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,69.774,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,21.283,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,138.089,155.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,4.738,218.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,9.137,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,3.155,327.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,18.466,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,5.435,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,35.307,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,1.349,191.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,2.355,219.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.955,270.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,5.131,131.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,1.581,326.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,9.290,144.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,284.695,232.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,570.087,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,fwd,False,183.654,359.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,32768,32768,96,96,192,128,bwd,False,1070.473,160.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,fwd,False,341.868,386.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,32768,32768,96,96,192,128,bwd,False,2157.577,159.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,71.744,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,142.817,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,fwd,False,45.965,359.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,465.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,16384,16384,96,96,192,128,bwd,False,270.710,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,fwd,False,85.146,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,16384,16384,96,96,192,128,bwd,False,543.412,158.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,18.284,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,36.011,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,fwd,False,11.849,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,8192,8192,96,96,192,128,bwd,False,69.670,154.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,fwd,False,21.273,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,8192,8192,96,96,192,128,bwd,False,137.800,156.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,4.744,217.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,9.135,226.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,fwd,False,3.156,327.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,4096,4096,96,96,192,128,bwd,False,18.446,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,fwd,False,5.436,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,4096,4096,96,96,192,128,bwd,False,35.238,152.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +kimi_k3,kimi_k3,cudnn,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,1.331,194.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimi_k3,kimi_k3,cudnn,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,2.444,211.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,fwd,False,0.934,276.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,top_left,2,2048,2048,96,96,192,128,bwd,False,5.346,125.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,fwd,False,1.552,332.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +kimi_k3,kimi_k3,flash_attention_4,bfloat16,no_mask,2,2048,2048,96,96,192,128,bwd,False,9.636,139.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 diff --git a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png old mode 100644 new mode 100755 index 97119d9cd..37602a3d3 Binary files a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png and b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_no_mask.png differ diff --git a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png old mode 100644 new mode 100755 index daa33c486..89e1f0e35 Binary files a/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png and b/benchmark/attention_training/results/kimi_k3/rtxpro6000/kimi_k3_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260818_124202.csv b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260819_052235.csv similarity index 58% rename from benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260818_124202.csv rename to benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260819_052235.csv index f33232c80..3abd5ffb5 100644 --- a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260818_124202.csv +++ b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_20260819_052235.csv @@ -1,141 +1,141 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,22.750,1547.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,73.459,1197.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,85.443,1030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,45.263,1555.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,147.288,1194.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,167.219,1052.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,16.405,2145.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,52.653,1671.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,51.811,1698.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,33.158,2122.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,102.282,1720.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,101.815,1728.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.312,2032.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,63.736,1380.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,65.268,1348.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,34.787,2023.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,128.287,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,124.932,1408.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,23.567,1493.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,49.117,1433.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,21.824,1612.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,41.181,1709.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,23.995,1466.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,70.785,1243.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,74.484,1181.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,59.911,1175.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,129.288,1361.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,134.910,1304.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.425,1622.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,17.415,1263.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,18.947,1161.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,11.110,1583.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,34.328,1281.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,39.649,1109.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.174,2107.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,13.656,1610.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,13.463,1634.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.942,2215.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,25.059,1755.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,24.348,1806.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.266,2062.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,15.548,1414.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,16.000,1374.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.210,2143.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,31.033,1417.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,29.977,1467.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.615,1567.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,11.361,1548.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,5.758,1528.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,10.354,1699.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.329,1651.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,16.430,1338.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,15.852,1387.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,14.214,1238.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.003,1516.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,33.006,1333.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.435,1533.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,4.234,1299.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.830,1138.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.587,1700.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.874,1396.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,8.943,1229.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.101,1998.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.800,1447.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.806,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.008,2190.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.190,1776.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.190,1776.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.123,1959.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,4.351,1264.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,4.355,1263.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.073,2122.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.440,1478.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.406,1485.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.588,1385.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.725,1614.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.594,1380.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.634,1670.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.409,1560.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,4.137,1329.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.365,1260.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,3.501,1256.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.335,1499.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,8.014,1372.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.406,1354.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.201,1144.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.365,1007.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.707,1556.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.073,1326.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.364,1163.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.311,1769.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.174,1171.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.171,1174.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.537,2046.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.744,1576.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.741,1579.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.321,1716.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.334,1031.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.334,1031.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.556,1977.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.059,1335.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.060,1335.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.450,1223.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.740,1486.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.495,1111.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.710,1549.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.414,1329.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.230,1117.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.448,950.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.921,1194.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.040,1348.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.250,1222.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.129,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.394,872.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.437,786.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.198,1391.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.618,1111.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.693,992.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.100,1377.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.417,824.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.417,824.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.149,1847.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.562,1223.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.561,1224.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.103,1334.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.469,733.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.467,736.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.156,1762.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.646,1065.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.646,1064.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.151,913.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.205,1339.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.184,748.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.197,1393.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.132,1045.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.418,823.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,22.112,1591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,69.824,1260.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,85.096,1034.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,45.807,1536.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,140.325,1254.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,167.870,1048.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,16.344,2153.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,53.589,1641.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,53.029,1659.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,33.166,2122.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,102.344,1719.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,101.599,1732.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.077,2060.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,64.253,1369.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,61.526,1430.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,33.630,2092.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,121.763,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,125.481,1402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,24.097,1460.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,48.135,1462.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,21.830,1612.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,41.165,1709.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,22.906,1536.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,67.684,1300.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,72.269,1217.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,59.415,1184.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,127.272,1382.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,135.115,1302.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.587,1575.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,16.041,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,20.035,1098.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,11.056,1591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,30.562,1439.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,38.890,1131.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.095,2148.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,13.654,1611.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,13.468,1633.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.959,2210.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,23.956,1836.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,24.111,1824.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.341,2026.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,15.641,1406.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,15.560,1413.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.221,2140.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.887,1472.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,29.609,1485.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.697,1544.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,11.679,1506.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,5.755,1529.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,10.335,1702.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,5.428,1620.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,16.411,1340.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,16.481,1334.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,14.203,1239.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.723,1480.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,31.213,1409.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.450,1516.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,4.298,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.916,1118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2287.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.640,1666.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,8.118,1354.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,9.380,1172.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.076,2044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.807,1444.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.805,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.013,2184.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.196,1774.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.181,1779.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.137,1935.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,4.359,1261.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,4.369,1259.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.076,2118.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.476,1471.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.540,1458.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.578,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.769,1589.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.597,1377.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.628,1674.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.467,1499.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,4.206,1307.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.419,1244.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,3.506,1254.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.437,1478.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.909,1390.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.420,1310.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.228,1119.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.356,1014.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.724,1519.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.164,1270.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.397,1147.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.308,1784.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.179,1166.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.178,1167.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.541,2033.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.746,1574.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.747,1573.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.325,1691.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.343,1024.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.347,1021.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.563,1951.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.071,1327.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.066,1331.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.456,1206.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.755,1457.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.496,1108.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.710,1548.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.415,1325.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.266,1086.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.447,950.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.923,1191.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,2.036,1350.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.225,1235.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.133,1030.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.397,866.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.441,779.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.198,1388.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.632,1087.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.699,983.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.098,1400.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.418,822.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.419,821.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.151,1824.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.565,1215.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.565,1215.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.104,1323.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.470,731.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.471,730.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.159,1734.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.651,1056.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.650,1057.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.155,887.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.210,1306.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.183,752.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.199,1385.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,5135.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.134,1025.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.423,813.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.547,629.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.252,1093.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.607,1132.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.721,952.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.255,1076.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.617,1114.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.724,949.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png old mode 100644 new mode 100755 index 462690879..917c012f5 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png old mode 100644 new mode 100755 index 5ac605afd..ab0211e42 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png old mode 100644 new mode 100755 index 4bfefd18b..06e33e3a8 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png old mode 100644 new mode 100755 index 546c8b3fb..4aba170a3 Binary files a/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb200/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260818_124759.csv b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260819_052115.csv similarity index 58% rename from benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260818_124759.csv rename to benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260819_052115.csv index 5bc6dec0e..125e18813 100644 --- a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260818_124759.csv +++ b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_20260819_052115.csv @@ -1,141 +1,141 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,16.837,2090.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,61.234,1437.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,69.491,1266.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,34.329,2050.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,124.511,1413.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,136.676,1287.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,11.550,3046.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,42.662,2062.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,42.578,2066.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,23.169,3037.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,80.600,2183.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,80.622,2182.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,12.324,2855.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,55.320,1590.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,54.663,1609.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,24.844,2832.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,105.746,1664.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,106.604,1650.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,18.822,1869.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,36.920,1906.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.323,2031.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,27.847,2527.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,18.095,1945.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,56.576,1555.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,59.139,1487.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,32.603,2158.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,117.803,1493.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,125.734,1399.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.379,2009.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,14.667,1499.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,16.522,1331.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.245,2134.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,29.391,1496.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,32.073,1371.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,2.897,3037.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,10.886,2020.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,10.914,2015.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,5.549,3170.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,20.291,2168.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,20.309,2166.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,3.151,2792.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,14.097,1560.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,14.123,1557.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,5.987,2938.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,26.550,1657.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,25.942,1695.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.733,1858.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.994,1956.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.648,1892.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.033,2501.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.425,1988.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,13.967,1575.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,14.935,1473.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.257,2131.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,28.925,1521.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,29.821,1475.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.133,1941.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.846,1430.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.268,1288.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.089,2105.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.190,1529.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,8.067,1363.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.774,2840.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.116,1764.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.120,1762.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.406,3127.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,5.240,2098.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,5.238,2099.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.840,2619.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.943,1395.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.940,1396.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.517,2899.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.770,1624.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.775,1623.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.307,1682.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.278,1931.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.344,1636.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.816,2421.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.174,1873.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.763,1461.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.181,1315.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.070,2125.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.960,1580.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.440,1478.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.309,1781.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.103,1247.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.218,1128.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.537,2047.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.937,1419.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.206,1246.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.223,2468.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,0.995,1382.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,0.994,1383.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.375,2932.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.509,1822.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.510,1821.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.242,2277.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.214,1132.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.215,1131.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.405,2713.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.896,1450.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.894,1451.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.370,1486.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.590,1865.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.442,1243.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.503,2185.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.338,1627.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.136,1210.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.421,968.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.549,2004.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.911,1438.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.169,1267.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.097,1424.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.360,954.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.392,877.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.147,1868.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.570,1205.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.641,1073.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.072,1904.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.368,935.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.367,936.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.103,2681.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.499,1377.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.499,1377.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.079,1742.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.429,801.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.429,801.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.112,2463.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.598,1149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.598,1149.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.125,1102.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.159,1727.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.175,785.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.148,1861.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.108,1268.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.397,866.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.535,642.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.147,1871.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.580,1184.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.713,963.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,17.480,2013.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,57.269,1536.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,65.595,1341.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,34.799,2022.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,121.379,1449.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,131.096,1342.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,11.293,3116.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,42.105,2089.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,41.894,2100.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,22.113,3182.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,79.534,2212.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,79.503,2213.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,12.176,2890.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,False,55.353,1589.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,32768,32768,64,8,128,128,bwd,True,53.731,1637.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,24.495,2873.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,105.818,1662.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,104.817,1678.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,18.877,1864.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,36.859,1909.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,32768,32768,64,8,128,128,fwd,False,17.294,2035.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,27.729,2538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,17.285,2036.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,56.186,1566.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,58.777,1497.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,34.951,2013.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,115.771,1520.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,124.400,1414.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.413,1993.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,14.640,1502.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,16.659,1320.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.195,2147.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2194.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,28.735,1531.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,32.033,1373.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,2.858,3078.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,10.791,2038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,10.767,2042.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,5.503,3197.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,20.529,2142.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,19.762,2225.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,3.094,2843.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,False,14.044,1566.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,16384,16384,64,8,128,128,bwd,True,14.029,1568.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,4724.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,5.922,2971.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,25.754,1708.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,25.931,1696.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.700,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.977,1960.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2521.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,16384,16384,64,8,128,128,fwd,False,4.636,1897.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,7.002,2512.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,4.405,1997.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,13.786,1595.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,14.842,1482.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,8.075,2179.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,27.611,1593.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,29.476,1492.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.156,1902.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.821,1439.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.243,1296.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.056,2140.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,7.113,1546.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.915,1389.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.757,2904.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.093,1778.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.091,1779.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.389,3167.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,5.194,2117.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,5.192,2118.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,0.822,2677.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,False,3.915,1405.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,8192,8192,64,8,128,128,bwd,True,3.914,1405.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.500,2932.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.724,1635.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,6.725,1635.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.303,1687.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.263,1944.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,8192,8192,64,8,128,128,fwd,False,1.337,1645.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,1.801,2441.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,1.163,1891.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,3.740,1470.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,4.148,1326.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,2.036,2160.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,6.904,1593.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,7.387,1488.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.319,1726.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.101,1249.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.210,1136.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.542,2030.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.918,1433.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.190,1255.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.216,2549.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,0.986,1394.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,0.987,1394.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.370,2971.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.492,1842.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.491,1844.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.234,2345.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,False,1.204,1142.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,4096,4096,64,8,128,128,bwd,True,1.204,1142.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.400,2750.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.878,1464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,1.877,1464.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.366,1504.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.586,1876.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.438,1254.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.497,2211.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.332,1656.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,1.126,1220.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,1.412,974.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,0.541,2032.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,1.891,1454.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,2.155,1275.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.096,1427.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.356,966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.387,889.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.147,1871.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.565,1216.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.632,1088.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.069,1985.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.364,943.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.365,942.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.101,2719.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.494,1392.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.495,1388.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.075,1823.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,False,0.424,811.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,top_left,2,2048,2048,64,8,128,128,bwd,True,0.424,811.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.111,2473.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.595,1154.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn,mxfp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.595,1156.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.123,1122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.158,1743.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.173,793.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,cudnn_oss,mxfp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.145,1892.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,5155.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.106,1292.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.393,875.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.532,646.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.145,1898.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,0.571,1203.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,0.709,969.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png old mode 100644 new mode 100755 index c7e4efbc7..e489063f2 Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png old mode 100644 new mode 100755 index 532394817..ea8d4ec8f Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png old mode 100644 new mode 100755 index 8cba6013b..3b04cfa2f Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png old mode 100644 new mode 100755 index 9ff1373ea..e6166003e Binary files a/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/gb300/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260818_125922.csv b/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260819_054609.csv similarity index 58% rename from benchmark/attention_training/results/llama3.1/h200/llama3.1_20260818_125922.csv rename to benchmark/attention_training/results/llama3.1/h200/llama3.1_20260819_054609.csv index e65486d94..457ef3adc 100644 --- a/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260818_125922.csv +++ b/benchmark/attention_training/results/llama3.1/h200/llama3.1_20260819_054609.csv @@ -1,89 +1,89 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,57.396,613.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,166.565,528.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,113.495,620.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,327.154,538.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,38.188,921.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,242.472,363.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,242.468,363.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,75.109,937.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,243.582,722.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,244.431,720.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,55.630,632.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,145.207,606.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,154.596,569.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,104.517,673.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,284.883,618.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,290.367,606.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,14.042,626.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,41.225,533.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,48.455,454.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,28.238,623.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,82.050,536.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,93.794,469.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,8.892,989.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,61.819,356.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,61.766,356.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,18.763,938.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,61.064,720.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,60.894,722.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,13.390,657.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,36.174,608.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,38.186,576.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,25.866,680.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,70.821,621.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,72.453,607.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.377,651.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,10.245,537.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,12.203,451.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,6.315,696.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,19.575,562.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,22.750,483.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,2.288,961.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,16.055,342.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,16.043,343.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,4.285,1026.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,15.489,710.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,15.486,710.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.302,666.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,8.652,635.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,9.672,568.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,5.865,750.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,16.976,648.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,18.133,606.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.897,613.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.747,500.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,3.151,436.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.611,683.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.727,581.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,5.581,493.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.634,868.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,4.296,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,4.291,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.122,980.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.169,659.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.171,659.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.898,613.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.390,575.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,2.786,493.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.518,724.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.213,652.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.543,605.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.258,533.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.841,409.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.918,375.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.422,652.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.306,526.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.488,462.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.193,712.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,1.211,284.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,1.211,284.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.306,899.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.164,590.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.163,591.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.267,515.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.723,476.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.904,380.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.418,657.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.172,586.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,58.829,598.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,238.178,369.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,174.121,404.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,630.218,279.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,52.511,670.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,False,342.071,257.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,bwd,True,374.448,235.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,159.028,442.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,False,501.249,351.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,bwd,True,479.492,367.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,75.548,466.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,259.112,339.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,311.646,282.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,237.283,297.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,757.164,232.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,827.087,213.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,19.762,445.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,70.122,314.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,75.342,292.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,42.190,417.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,184.714,238.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,324.314,136.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,14.648,601.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,False,62.780,350.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,bwd,True,63.940,344.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,31.246,563.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,False,112.094,392.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,bwd,True,139.083,316.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,13.842,636.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,40.824,539.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,44.219,497.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,27.256,645.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,112.979,389.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,134.025,328.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.491,630.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,10.683,515.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,12.720,432.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,6.580,668.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,22.872,481.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,23.960,459.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,2.331,944.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,False,16.052,343.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,bwd,True,16.048,343.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,4.416,996.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,False,15.728,699.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,bwd,True,15.852,694.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,3.375,652.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,8.883,619.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,9.789,562.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,6.052,727.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,17.961,612.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,18.672,589.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.915,601.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.814,488.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,3.194,430.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.639,671.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.841,568.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,5.654,486.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,0.631,871.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,False,4.288,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,bwd,True,4.293,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.124,978.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.140,664.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.136,665.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,0.906,607.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,2.429,566.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,2.779,495.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.544,712.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,4.286,641.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,4.615,596.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.257,536.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.845,407.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.918,374.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.426,645.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.323,520.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.494,460.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.191,719.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,False,1.212,284.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,bwd,True,1.212,284.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.304,903.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.151,597.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.153,596.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,2141.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.267,516.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,0.732,470.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,0.902,381.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.421,652.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,1.187,579.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,1.354,508.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png old mode 100644 new mode 100755 index 9aa739464..ea5d81ad1 Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png old mode 100644 new mode 100755 index bd9ec4847..feb4697eb Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png old mode 100644 new mode 100755 index 37416d14f..d271df8ea Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png old mode 100644 new mode 100755 index 08bbb418c..83399c487 Binary files a/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/h200/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260818_131244.csv b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260818_131244.csv deleted file mode 100644 index 9f6a180e3..000000000 --- a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260818_131244.csv +++ /dev/null @@ -1,87 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,90.516,389.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,304.567,289.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,181.827,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,602.874,292.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,62.874,560.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,122.796,573.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,87.306,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,251.768,349.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,359.170,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,175.473,401.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,492.434,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,695.662,253.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,104.878,335.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,297.020,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,187.492,375.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,609.145,289.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,22.490,391.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,78.004,282.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,45.130,390.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,152.394,289.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,15.423,570.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,30.166,583.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,21.296,413.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,64.530,341.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,94.242,233.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,43.040,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,123.515,356.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,176.659,249.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,25.867,340.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,73.497,299.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,46.418,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,151.165,291.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.560,396.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,20.185,272.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,21.364,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.755,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,39.377,279.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,42.628,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,4.010,549.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,7.391,595.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.377,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,16.695,329.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,25.057,219.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.362,424.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,31.273,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,45.549,241.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,6.650,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,19.206,286.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,11.358,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,38.173,288.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.512,364.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,5.623,244.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,5.784,238.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.753,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,10.587,260.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,10.778,255.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,1.098,501.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,904.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.903,578.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,904.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.454,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,4.888,281.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,7.488,184.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.671,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,8.244,333.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,12.219,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.772,310.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,5.220,263.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.904,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,9.924,277.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.447,308.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.720,200.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,1.693,203.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.732,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.942,234.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,2.977,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.345,398.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,904.000 -llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.510,539.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,904.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.448,307.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.603,215.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,2.368,145.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.721,381.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.407,286.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,3.602,191.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.534,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.532,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.789,348.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.669,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260819_055202.csv b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260819_055202.csv new file mode 100644 index 000000000..17676d822 --- /dev/null +++ b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_20260819_055202.csv @@ -0,0 +1,87 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,88.731,397.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,329.673,267.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,177.520,396.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,575.443,306.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,32768,32768,64,8,128,128,fwd,False,61.256,574.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,32768,32768,64,8,128,128,fwd,False,119.503,589.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,85.178,413.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,246.036,358.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,True,359.436,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,170.652,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,478.762,367.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,True,696.551,253.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,fwd,False,102.082,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,32768,32768,64,8,128,128,bwd,False,288.548,305.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,fwd,False,182.838,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,32768,32768,64,8,128,128,bwd,False,594.798,296.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,22.025,399.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,83.380,264.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,44.269,397.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,145.707,302.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,16384,16384,64,8,128,128,fwd,False,15.329,574.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,16384,16384,64,8,128,128,fwd,False,29.207,602.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,933.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,20.697,425.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,63.327,347.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,True,94.252,233.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,42.179,417.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,120.787,364.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,True,176.674,249.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,fwd,False,25.879,340.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,16384,16384,64,8,128,128,bwd,False,73.497,299.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,fwd,False,45.052,390.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,16384,16384,64,8,128,128,bwd,False,149.734,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.548,396.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,21.821,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,465.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,20.792,264.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.735,410.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,37.638,292.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,42.352,260.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,465.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,8192,8192,64,8,128,128,fwd,False,4.009,549.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,8192,8192,64,8,128,128,fwd,False,7.386,595.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,918.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,5.363,410.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,16.417,335.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,True,25.073,219.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,10.358,425.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,30.787,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,True,45.571,241.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,fwd,False,6.650,331.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,8192,8192,64,8,128,128,bwd,False,19.210,286.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,fwd,False,11.358,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,8192,8192,64,8,128,128,bwd,False,38.163,288.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.588,346.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,6.059,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,5.640,244.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.749,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,9.988,275.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,10.758,256.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,4096,4096,64,8,128,128,fwd,False,1.142,481.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,915.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,4096,4096,64,8,128,128,fwd,False,1.979,556.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,915.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.506,365.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,4.865,283.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,True,7.499,183.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,2.672,412.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,8.127,338.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,True,12.213,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,fwd,False,1.842,299.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,4096,4096,64,8,128,128,bwd,False,5.223,263.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,fwd,False,3.006,366.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,4096,4096,64,8,128,128,bwd,False,9.922,277.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.464,297.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.886,182.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,1.697,203.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.764,360.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.865,240.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,2.981,231.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +llama3.1,llama3.1,cudnn,fp8,top_left,2,2048,2048,64,8,128,128,fwd,False,0.339,406.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,915.000 +llama3.1,llama3.1,cudnn,fp8,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.531,518.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,915.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.439,313.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.653,208.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,True,2.432,141.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.748,367.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.408,285.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,cudnn_oss,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,True,3.713,185.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,fwd,False,0.524,262.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,top_left,2,2048,2048,64,8,128,128,bwd,False,1.581,217.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,fwd,False,0.814,338.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +llama3.1,llama3.1,flash_attention_4,bfloat16,no_mask,2,2048,2048,64,8,128,128,bwd,False,2.674,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png old mode 100644 new mode 100755 index da36fe7c6..080ac5753 Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png old mode 100644 new mode 100755 index 1a0cfa9cf..50342b443 Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png old mode 100644 new mode 100755 index 4d3f8c565..bd36bb5f6 Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left.png differ diff --git a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png old mode 100644 new mode 100755 index 0f8842353..d82a91f8a Binary files a/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png and b/benchmark/attention_training/results/llama3.1/rtxpro6000/llama3.1_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/gb200/ltx2_20260818_122458.csv b/benchmark/attention_training/results/ltx2/gb200/ltx2_20260819_050934.csv similarity index 59% rename from benchmark/attention_training/results/ltx2/gb200/ltx2_20260818_122458.csv rename to benchmark/attention_training/results/ltx2/gb200/ltx2_20260819_050934.csv index ce52ce1ad..6e3148dca 100644 --- a/benchmark/attention_training/results/ltx2/gb200/ltx2_20260818_122458.csv +++ b/benchmark/attention_training/results/ltx2/gb200/ltx2_20260819_050934.csv @@ -1,36 +1,36 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.445,1388.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.210,1277.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.355,1141.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.470,1316.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.592,1044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.163,1330.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.254,1233.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.820,1611.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,5.270,1391.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,6.489,1129.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.962,1494.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,2.531,1158.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.954,1479.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.284,1387.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.075,1642.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,9.580,1318.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,11.499,1098.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.239,1559.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,4.259,1186.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.274,1526.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.784,1437.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,9.470,1582.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,29.222,1282.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,36.059,1039.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,9.416,1591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,12.723,1178.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,25.084,1493.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,26.255,1427.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,14.178,1637.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,44.687,1298.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,51.614,1124.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,15.607,1487.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2456.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,19.712,1177.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,37.971,1528.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,41.651,1393.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.443,1395.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.209,1279.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.382,1119.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.472,1309.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.588,1051.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.118,1383.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.282,1206.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.878,1561.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,5.267,1391.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,6.432,1139.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.960,1496.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,2.527,1160.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.888,1499.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.211,1406.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.011,1677.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,9.298,1358.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,11.814,1069.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,3.208,1574.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,4.195,1204.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.159,1547.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.761,1441.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,8.789,1705.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,29.537,1268.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,37.417,1001.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,10.004,1498.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,12.476,1201.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,25.162,1489.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,26.277,1425.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,15.355,1511.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,44.523,1303.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,52.690,1101.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,15.332,1513.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,18.692,1241.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,37.123,1563.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,42.311,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png old mode 100644 new mode 100755 index 408dd95e6..170ef47c4 Binary files a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png old mode 100644 new mode 100755 index 74e591f1f..c75244aee Binary files a/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/gb200/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/gb300/ltx2_20260818_123039.csv b/benchmark/attention_training/results/ltx2/gb300/ltx2_20260819_050819.csv similarity index 61% rename from benchmark/attention_training/results/ltx2/gb300/ltx2_20260818_123039.csv rename to benchmark/attention_training/results/ltx2/gb300/ltx2_20260819_050819.csv index d0f380313..2f1c23ff7 100644 --- a/benchmark/attention_training/results/ltx2/gb300/ltx2_20260818_123039.csv +++ b/benchmark/attention_training/results/ltx2/gb300/ltx2_20260819_050819.csv @@ -1,36 +1,36 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.336,1843.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.086,1424.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.209,1279.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.369,1678.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.339,1826.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.039,1488.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.346,1787.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.103,1402.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.217,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.374,1654.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.348,1780.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,1.053,1468.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,1.233,1254.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.443,2031.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.769,1537.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.765,1271.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.605,1826.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.438,2038.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.446,1648.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,4.912,1492.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.380,2122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.107,1557.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,9.727,1298.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.639,1913.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.354,2145.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,7.501,1683.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.430,1498.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.061,2122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,24.475,1530.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,29.684,1262.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.831,1913.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,6.980,2147.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,23.313,1607.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,24.972,1500.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,11.082,2094.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,38.074,1524.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,40.880,1419.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,11.856,1957.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,10.874,2134.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,34.868,1664.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,41.036,1414.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.493,1964.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.836,1515.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,5.821,1259.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.630,1798.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,1.481,1980.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,4.501,1628.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,4.944,1482.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.457,2055.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,8.260,1528.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,9.937,1270.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.679,1885.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,2.430,2078.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,7.604,1660.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,8.572,1473.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.221,2075.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,24.622,1521.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,29.838,1255.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.914,1893.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,7.201,2081.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,23.584,1588.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,25.098,1492.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,11.290,2055.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,38.525,1506.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,42.773,1356.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,12.277,1890.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,10.986,2112.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,35.825,1619.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,41.351,1403.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png old mode 100644 new mode 100755 index cf26332d8..3e50bd353 Binary files a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png old mode 100644 new mode 100755 index 50532c603..c38261771 Binary files a/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/gb300/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/h200/ltx2_20260818_122311.csv b/benchmark/attention_training/results/ltx2/h200/ltx2_20260819_050822.csv similarity index 59% rename from benchmark/attention_training/results/ltx2/h200/ltx2_20260818_122311.csv rename to benchmark/attention_training/results/ltx2/h200/ltx2_20260819_050822.csv index ab6aebaf4..5198f5b76 100644 --- a/benchmark/attention_training/results/ltx2/h200/ltx2_20260818_122311.csv +++ b/benchmark/attention_training/results/ltx2/h200/ltx2_20260819_050822.csv @@ -1,30 +1,30 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.913,678.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,2.625,589.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,3.078,502.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.855,723.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,2.325,665.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,2.514,615.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,4.642,631.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,12.842,571.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,14.952,490.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,3.942,744.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,11.055,663.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,11.235,652.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,8.132,621.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,22.445,562.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,26.212,482.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,6.769,746.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,19.592,644.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,20.146,627.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,25.682,583.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,67.103,558.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,78.556,477.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,21.973,682.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,58.867,636.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,60.171,622.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,37.160,624.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,106.600,544.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,33.847,686.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,92.258,629.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,95.002,611.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.922,671.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,2.652,583.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,3.100,499.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,0.867,714.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,2.351,658.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,2.535,610.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,4.728,620.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,12.709,577.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,15.083,486.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,4.027,728.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,10.973,668.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,11.728,625.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,8.146,620.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,23.068,547.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,26.549,476.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,6.915,730.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,19.896,635.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,20.402,619.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,26.141,573.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,82.465,454.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,99.492,376.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,22.109,678.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,72.287,518.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,84.906,441.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,43.742,530.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,168.265,345.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,35.009,663.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,157.340,369.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,142.267,408.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png old mode 100644 new mode 100755 index dd98fa282..988127ede Binary files a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png old mode 100644 new mode 100755 index 29d9b56cb..9810651ed Binary files a/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/h200/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260818_122308.csv b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260818_122308.csv deleted file mode 100644 index 25e30e87e..000000000 --- a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260818_122308.csv +++ /dev/null @@ -1,39 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.791,345.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,450.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,5.932,261.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,450.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.596,234.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,450.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.732,357.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,4.396,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.719,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.922,322.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,7.230,214.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.161,409.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,25.857,283.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,29.331,250.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,6.904,425.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,19.973,367.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,30.334,242.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.746,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,32.863,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.573,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,51.329,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,50.598,250.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.022,420.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,34.951,361.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,51.537,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,13.516,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,56.259,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,38.258,392.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,150.729,249.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,37.112,404.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,103.500,362.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,151.271,248.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,39.864,376.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,164.785,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,59.811,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,197.424,294.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,57.688,402.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,159.914,363.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,230.769,251.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,62.676,370.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,254.244,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260819_050834.csv b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260819_050834.csv new file mode 100644 index 000000000..11d8c6251 --- /dev/null +++ b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_20260819_050834.csv @@ -0,0 +1,39 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.758,352.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,5.683,272.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.576,235.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.695,365.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,4.350,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,True,6.738,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,fwd,False,1.891,327.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,6144,6144,32,32,128,128,bwd,False,7.312,211.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.145,410.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,24.565,298.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,28.815,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,6.903,425.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,19.391,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,True,30.362,241.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,fwd,False,7.750,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,13376,13376,32,32,128,128,bwd,False,32.756,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.526,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,49.732,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,49.030,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,465.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,12.017,420.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,34.184,369.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,True,51.616,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,fwd,False,13.519,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,17556,17556,32,32,128,128,bwd,False,56.312,224.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,38.112,393.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,146.290,256.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,35.517,422.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,463.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,100.660,372.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,True,151.301,248.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,fwd,False,39.575,379.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,30240,30240,32,32,128,128,bwd,False,164.945,227.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,58.542,396.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +ltx2,ltx2,cudnn,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,189.225,307.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,55.958,415.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,156.615,370.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,cudnn_oss,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,True,231.363,251.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,fwd,False,61.127,380.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +ltx2,ltx2,flash_attention_4,bfloat16,no_mask,1,37632,37632,32,32,128,128,bwd,False,254.523,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png old mode 100644 new mode 100755 index 97ab4066a..28f21fe85 Binary files a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png and b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask.png differ diff --git a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png old mode 100644 new mode 100755 index 170d40969..552d596ff Binary files a/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png and b/benchmark/attention_training/results/ltx2/rtxpro6000/ltx2_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/gb200/qwen35_20260818_122507.csv b/benchmark/attention_training/results/qwen35/gb200/qwen35_20260819_050942.csv similarity index 58% rename from benchmark/attention_training/results/qwen35/gb200/qwen35_20260818_122507.csv rename to benchmark/attention_training/results/qwen35/gb200/qwen35_20260819_050942.csv index 0df7348d8..b8c8d8ccf 100644 --- a/benchmark/attention_training/results/qwen35/gb200/qwen35_20260818_122507.csv +++ b/benchmark/attention_training/results/qwen35/gb200/qwen35_20260819_050942.csv @@ -1,31 +1,31 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.261,1714.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,43.324,1015.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,43.234,1017.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.083,1745.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.849,1622.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,48.441,908.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.428,1812.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,10.706,1027.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,9.778,1125.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.391,1840.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.197,1376.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,12.522,878.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.635,1730.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.703,1017.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.696,1020.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.670,1641.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.128,975.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.220,854.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.188,1463.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.788,873.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.792,868.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.204,1348.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.463,594.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.405,489.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.058,1191.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.270,637.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,1756.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.271,635.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.071,964.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2549.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.221,311.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.667,258.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.980,1959.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,42.795,1028.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,43.239,1017.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,9.048,1944.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,11.829,1487.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,49.625,886.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.312,1902.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,9.792,1123.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,9.939,1106.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.383,1846.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.116,1412.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,11.754,936.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.622,1767.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.565,1072.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.579,1066.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.647,1700.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.127,975.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.263,843.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.186,1477.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.776,886.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.776,886.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.197,1394.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.465,591.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.406,489.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.058,1182.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.264,651.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.265,648.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.069,991.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2026.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.222,310.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.663,259.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png old mode 100644 new mode 100755 index c7a570242..a61bc7215 Binary files a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png old mode 100644 new mode 100755 index 8a59653cc..c9bb2ed81 Binary files a/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png and b/benchmark/attention_training/results/qwen35/gb200/qwen35_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/gb300/qwen35_20260818_123050.csv b/benchmark/attention_training/results/qwen35/gb300/qwen35_20260819_050828.csv similarity index 59% rename from benchmark/attention_training/results/qwen35/gb300/qwen35_20260818_123050.csv rename to benchmark/attention_training/results/qwen35/gb300/qwen35_20260819_050828.csv index 34150d6ba..8715c8c57 100644 --- a/benchmark/attention_training/results/qwen35/gb300/qwen35_20260818_123050.csv +++ b/benchmark/attention_training/results/qwen35/gb300/qwen35_20260819_050828.csv @@ -1,31 +1,31 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.317,2115.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,37.344,1178.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,37.290,1179.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.140,2161.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.365,1697.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,43.523,1011.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.073,2122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,9.019,1219.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,9.059,1214.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.093,2102.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.101,1418.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,11.122,989.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.537,2046.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.425,1134.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.427,1133.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.562,1958.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.119,983.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.106,885.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.149,1840.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.716,960.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.718,957.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.167,1651.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.460,598.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.357,507.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.049,1395.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.248,693.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.247,695.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.059,1175.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2250.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.220,313.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.643,267.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.080,2177.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,38.764,1135.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,38.254,1150.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,8.086,2176.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,10.343,1701.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,42.345,1039.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.051,2144.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,8.963,1227.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,8.980,1224.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,2.078,2117.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,3.087,1425.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,11.005,999.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.537,2049.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,2.405,1143.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,2.406,1143.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,0.559,1968.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.113,988.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,3.080,893.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.151,1823.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,0.712,966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,0.712,966.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.164,1679.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.463,594.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.353,508.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.051,1355.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.246,700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.245,700.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2484.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.059,1166.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.218,315.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.638,269.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png old mode 100644 new mode 100755 index bdfe580f0..fd8a6da49 Binary files a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png old mode 100644 new mode 100755 index 95a7e8836..c9ce7e55c Binary files a/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png and b/benchmark/attention_training/results/qwen35/gb300/qwen35_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/h200/qwen35_20260818_132623.csv b/benchmark/attention_training/results/qwen35/h200/qwen35_20260819_061345.csv similarity index 61% rename from benchmark/attention_training/results/qwen35/h200/qwen35_20260818_132623.csv rename to benchmark/attention_training/results/qwen35/h200/qwen35_20260819_061345.csv index 2a3854ddd..ffb1f224e 100644 --- a/benchmark/attention_training/results/qwen35/h200/qwen35_20260818_132623.csv +++ b/benchmark/attention_training/results/qwen35/h200/qwen35_20260819_061345.csv @@ -1,31 +1,31 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,26.173,672.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,124.951,352.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,137.501,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,23.923,735.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,140.529,313.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,141.746,310.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,6.094,722.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,31.015,355.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,34.371,320.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,5.528,796.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,34.170,322.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,34.284,321.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.607,684.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,7.002,393.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,7.879,349.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.419,775.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,8.839,311.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,9.045,304.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.449,612.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.939,355.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.212,311.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.381,722.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,2.409,285.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.497,275.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,26.695,659.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,159.328,276.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,183.851,239.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,24.330,723.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,False,199.242,221.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,32768,32768,32,2,256,256,bwd,True,204.878,215.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,6.301,698.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,35.957,306.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,38.920,283.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,5.667,776.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,False,34.454,319.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,16384,16384,32,2,256,256,bwd,True,35.004,314.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.646,668.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,7.063,389.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,7.974,345.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,1.444,761.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,False,8.765,314.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,8192,8192,32,2,256,256,bwd,True,8.978,306.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.453,607.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,1.948,353.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.200,312.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.386,713.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,False,2.394,287.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,4096,4096,32,2,256,256,bwd,True,2.488,276.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.142,485.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.611,281.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.668,257.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.111,620.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.704,244.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.775,222.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.607,283.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.664,259.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.112,616.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,False,0.699,246.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +qwen35,qwen35,flash_attention_4,bfloat16,top_left,1,2048,2048,32,2,256,256,bwd,True,0.769,224.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png old mode 100644 new mode 100755 index 493ffa581..05efd5c71 Binary files a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png old mode 100644 new mode 100755 index de79919f4..51b31a3b8 Binary files a/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png and b/benchmark/attention_training/results/qwen35/h200/qwen35_top_left_det_overhead.png differ diff --git a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260818_133907.csv b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260818_133907.csv deleted file mode 100644 index 37f286e33..000000000 --- a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260818_133907.csv +++ /dev/null @@ -1,11 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,107.173,164.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,47.376,371.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,453.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,27.198,162.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,456.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,11.416,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,6.994,157.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,2.955,372.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,1.863,148.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.806,341.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.532,129.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.308,223.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 diff --git a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260819_061603.csv b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260819_061603.csv new file mode 100644 index 000000000..61b56830e --- /dev/null +++ b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_20260819_061603.csv @@ -0,0 +1,11 @@ +config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops +qwen35,qwen35,cudnn,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,107.177,164.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,32768,32768,32,2,256,256,fwd,False,46.527,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,27.181,162.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,16384,16384,32,2,256,256,fwd,False,11.410,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,6.987,157.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,8192,8192,32,2,256,256,fwd,False,2.952,373.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,1.933,142.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,4096,4096,32,2,256,256,fwd,False,0.839,328.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +qwen35,qwen35,cudnn,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.543,127.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +qwen35,qwen35,cudnn_oss,bfloat16,top_left,1,2048,2048,32,2,256,256,fwd,False,0.301,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 diff --git a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png old mode 100644 new mode 100755 index c70bf6db6..b19014e83 Binary files a/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png and b/benchmark/attention_training/results/qwen35/rtxpro6000/qwen35_top_left.png differ diff --git a/benchmark/attention_training/results/wan22/gb200/wan22_20260818_122522.csv b/benchmark/attention_training/results/wan22/gb200/wan22_20260819_050958.csv similarity index 60% rename from benchmark/attention_training/results/wan22/gb200/wan22_20260818_122522.csv rename to benchmark/attention_training/results/wan22/gb200/wan22_20260819_050958.csv index 0d83b827b..139777ab2 100644 --- a/benchmark/attention_training/results/wan22/gb200/wan22_20260818_122522.csv +++ b/benchmark/attention_training/results/wan22/gb200/wan22_20260819_050958.csv @@ -1,36 +1,36 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.793,1570.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.334,1335.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.871,1085.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.899,1386.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.132,1101.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.201,1416.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.367,1316.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.629,1662.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,10.756,1402.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,14.596,1033.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.823,1577.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,4.918,1226.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.786,1541.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,10.523,1433.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,14.417,1525.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,43.048,1276.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,54.241,1013.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,14.486,1517.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,17.895,1228.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,37.074,1482.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,40.940,1342.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,32.344,1481.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,100.324,1194.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,124.065,965.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,32.054,1494.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,40.589,1180.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,86.312,1387.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,94.079,1273.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,83.758,1397.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,243.172,1203.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,308.334,949.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,83.894,1395.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,108.625,1078.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,218.518,1339.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,244.671,1196.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.825,1510.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.273,1371.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.802,1112.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.876,1423.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.133,1100.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.151,1448.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.345,1328.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.501,1723.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,10.697,1409.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,13.931,1082.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.794,1590.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,4.930,1223.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.554,1578.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,10.434,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,13.367,1644.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,42.367,1297.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,52.658,1044.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,13.950,1576.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,18.317,1200.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,33.841,1624.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,36.862,1491.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,29.530,1622.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,95.292,1257.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,116.025,1032.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,31.037,1543.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,38.993,1228.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,82.779,1447.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,92.986,1288.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,74.740,1566.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,236.015,1240.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,293.295,998.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,80.994,1445.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,101.395,1154.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,199.328,1468.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,236.688,1236.000,0.000,10,,True,False,,NVIDIA GB200,1.27.0,92600.000,2568.000 diff --git a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png old mode 100644 new mode 100755 index 170d126e8..a13f9fba1 Binary files a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png old mode 100644 new mode 100755 index 36d413889..3c74d86ff Binary files a/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/gb200/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/wan22/gb300/wan22_20260818_123102.csv b/benchmark/attention_training/results/wan22/gb300/wan22_20260819_050838.csv similarity index 62% rename from benchmark/attention_training/results/wan22/gb300/wan22_20260818_123102.csv rename to benchmark/attention_training/results/wan22/gb300/wan22_20260819_050838.csv index 654d5d496..f5d3b9f14 100644 --- a/benchmark/attention_training/results/wan22/gb300/wan22_20260818_123102.csv +++ b/benchmark/attention_training/results/wan22/gb300/wan22_20260819_050838.csv @@ -1,36 +1,36 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.647,1924.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.168,1437.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.613,1192.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.653,1907.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.153,1447.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.590,1203.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.718,1735.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.659,1892.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.038,1529.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.258,1380.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.929,2059.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.816,1536.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,11.922,1265.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.164,1906.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.892,2085.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.160,1646.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,10.242,1472.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.382,2117.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,38.061,1444.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,43.062,1276.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,11.399,1928.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.324,2129.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,33.153,1657.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,38.062,1444.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,23.796,2013.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,86.162,1390.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,101.644,1178.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,26.153,1831.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,22.415,2137.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,76.643,1562.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,90.343,1325.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,59.146,1979.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,214.344,1365.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,253.620,1154.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,64.538,1814.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,60.805,1925.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,192.207,1522.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,229.827,1273.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,0.650,1917.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,2.026,1538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,2.229,1397.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.883,2092.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.805,1538.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,11.838,1274.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,3.139,1921.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,2.829,2132.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,9.082,1660.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,10.425,1446.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.354,2123.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,37.632,1460.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,44.118,1245.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,11.519,1908.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,10.356,2122.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,34.170,1608.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,37.460,1467.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,24.395,1963.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,85.900,1394.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,101.617,1178.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,25.583,1872.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,23.522,2036.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,77.964,1536.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,89.666,1335.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,60.489,1935.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,211.692,1382.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,250.297,1169.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,63.447,1845.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,58.654,1996.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,192.854,1517.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,226.112,1294.000,0.000,10,,True,False,,NVIDIA GB300,1.27.0,92600.000,2578.000 diff --git a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png old mode 100644 new mode 100755 index 29c15e370..81f9528b6 Binary files a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png old mode 100644 new mode 100755 index c99c4eee8..6f6cc2f9a Binary files a/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/gb300/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/wan22/h200/wan22_20260818_133020.csv b/benchmark/attention_training/results/wan22/h200/wan22_20260819_061815.csv similarity index 58% rename from benchmark/attention_training/results/wan22/h200/wan22_20260818_133020.csv rename to benchmark/attention_training/results/wan22/h200/wan22_20260819_061815.csv index 8b9bd83e3..7d7d7b3a0 100644 --- a/benchmark/attention_training/results/wan22/h200/wan22_20260818_133020.csv +++ b/benchmark/attention_training/results/wan22/h200/wan22_20260819_061815.csv @@ -1,28 +1,28 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.978,630.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,5.221,597.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,6.355,490.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.695,735.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,4.615,675.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,4.901,636.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,9.885,610.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,26.735,564.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,31.295,482.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,8.066,748.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,23.098,653.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,24.022,628.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,38.345,573.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,99.337,553.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,32.309,680.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,88.108,624.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,89.851,612.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,83.526,573.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,218.597,548.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,70.317,681.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,192.201,623.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,195.896,611.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,207.773,563.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,537.143,545.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,174.625,670.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,470.768,622.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,480.075,610.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,2.016,618.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,5.334,584.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,6.421,485.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,1.730,720.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,4.700,663.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,4.965,627.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,9.711,621.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,27.793,542.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,32.318,467.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,8.281,728.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,23.942,630.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,25.137,600.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,39.489,557.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,155.526,353.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,33.356,659.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,119.888,458.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,136.177,404.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,137.766,348.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,478.800,250.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,106.833,448.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,431.124,278.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,542.986,221.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,603.184,194.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1683.726,174.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,480.075,244.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1520.242,192.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,1530.849,191.000,0.000,10,,True,False,,NVIDIA H200,1.27.0,92600.000,1071.000 diff --git a/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png b/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png old mode 100644 new mode 100755 index b9c6dd271..f43bed48a Binary files a/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/h200/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png old mode 100644 new mode 100755 index 867cda6dc..ed1db4d28 Binary files a/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/h200/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260818_134432.csv b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260819_062118.csv similarity index 55% rename from benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260818_134432.csv rename to benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260819_062118.csv index eb2e27b50..0fee70a29 100644 --- a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260818_134432.csv +++ b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_20260819_062118.csv @@ -1,38 +1,38 @@ config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,time_ms,tflops,max_diff,num_iterations,sliding_window_size,success,skipped,error_message,gpu_name,cudnn_version,cudnn_backend_version,peak_mma_tflops -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.087,404.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,12.999,240.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,12.767,244.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,2.991,417.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,8.649,360.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,13.329,234.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.305,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,14.157,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,15.234,396.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,452.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,61.328,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,60.815,248.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,14.367,420.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,42.529,355.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,61.578,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,15.951,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,67.135,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,56.728,387.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,220.024,250.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,54.066,407.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,152.515,360.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,223.111,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,58.690,375.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,240.630,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,125.342,382.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,478.876,250.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,118.755,403.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,334.198,358.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,484.704,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,130.233,368.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,523.068,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,309.743,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1163.473,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,294.883,397.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,826.635,354.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,1161.375,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,322.109,363.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 -wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1274.466,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.239,385.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,12.625,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,12.376,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.113,400.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,8.519,366.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,True,13.323,234.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,fwd,False,3.302,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,7800,7800,40,40,128,128,bwd,False,14.158,220.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,14.811,407.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,458.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,59.399,254.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,58.732,257.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,465.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,14.273,423.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,41.590,363.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,True,61.583,245.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,fwd,False,15.965,378.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,462.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,17160,17160,40,40,128,128,bwd,False,67.136,225.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,55.755,394.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,213.308,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,52.846,416.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,459.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,148.560,370.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,460.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,True,223.222,246.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,fwd,False,57.366,383.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,32760,32760,40,40,128,128,bwd,False,240.559,228.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,122.141,392.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,463.675,258.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,115.926,413.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,325.034,368.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,True,484.609,247.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,fwd,False,127.055,377.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,48360,48360,40,40,128,128,bwd,False,522.111,229.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,301.708,388.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1125.321,260.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,286.705,408.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,801.493,365.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,cudnn_oss,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,True,1161.450,252.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,fwd,False,313.086,374.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 +wan22,wan22_a14b,flash_attention_4,bfloat16,no_mask,1,75600,75600,40,40,128,128,bwd,False,1272.643,230.000,0.000,10,,True,False,,NVIDIA RTX PRO 6000 Blackwell Server Edition,1.27.0,92600.000,466.000 diff --git a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png old mode 100644 new mode 100755 index 9d1b89ba1..e34594fe3 Binary files a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png and b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask.png differ diff --git a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png old mode 100644 new mode 100755 index c5a4ba4c6..efe3c22d2 Binary files a/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png and b/benchmark/attention_training/results/wan22/rtxpro6000/wan22_no_mask_det_overhead.png differ diff --git a/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt b/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt old mode 100644 new mode 100755 index 4da2c80a2..af2cc7607 --- a/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB200/output.txt @@ -8,9 +8,9 @@ nvidia-cutlass-dsl: 4.4.1 summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.548 1.882 1.22x -dswiglu 0.884 1.716 1.94x -srelu 0.797 1.043 1.31x -dsrelu 0.811 1.346 1.66x -geglu 1.523 1.913 1.26x -dgeglu 0.892 1.840 2.06x +swiglu 1.557 1.948 1.25x +dswiglu 0.848 1.738 2.05x +srelu 0.792 1.073 1.35x +dsrelu 0.841 1.406 1.67x +geglu 1.596 1.940 1.22x +dgeglu 0.889 1.839 2.07x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt b/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt old mode 100644 new mode 100755 index f6faa7e26..ec24d3098 --- a/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB200/output_verbose.txt @@ -10,62 +10,62 @@ timing mode: CUDA graph replay activation: swiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.548 ms -unfused TE avg: 1.882 ms -speedup vs TE: 1.22x -fused samples: [1.401, 1.509, 1.625, 1.638, 1.568] -TE samples: [1.968, 1.857, 1.853, 1.863, 1.869] +fused cuDNN avg: 1.557 ms +unfused TE avg: 1.948 ms +speedup vs TE: 1.25x +fused samples: [1.387, 1.386, 1.671, 1.675, 1.666] +TE samples: [2.123, 1.914, 1.902, 1.898, 1.902] activation: dswiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.884 ms -unfused TE avg: 1.716 ms -speedup vs TE: 1.94x -fused samples: [0.801, 0.805, 0.917, 0.935, 0.964] -TE samples: [2.234, 1.651, 1.583, 1.548, 1.566] +fused cuDNN avg: 0.848 ms +unfused TE avg: 1.738 ms +speedup vs TE: 2.05x +fused samples: [0.792, 0.793, 0.8, 0.868, 0.985] +TE samples: [2.305, 1.68, 1.573, 1.568, 1.566] activation: srelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=None, activation output N=4096 -fused cuDNN avg: 0.797 ms -unfused TE avg: 1.043 ms -speedup vs TE: 1.31x -fused samples: [0.757, 0.741, 0.775, 0.852, 0.86] -TE samples: [1.137, 1.09, 1.005, 0.989, 0.992] +fused cuDNN avg: 0.792 ms +unfused TE avg: 1.073 ms +speedup vs TE: 1.35x +fused samples: [0.745, 0.726, 0.77, 0.854, 0.867] +TE samples: [1.196, 1.127, 1.037, 1.036, 0.968] activation: dsrelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=4096, activation output N=4096 -fused cuDNN avg: 0.811 ms -unfused TE avg: 1.346 ms -speedup vs TE: 1.66x -fused samples: [0.752, 0.746, 0.823, 0.855, 0.878] -TE samples: [1.548, 1.347, 1.283, 1.275, 1.278] +fused cuDNN avg: 0.841 ms +unfused TE avg: 1.406 ms +speedup vs TE: 1.67x +fused samples: [0.74, 0.739, 0.876, 0.908, 0.942] +TE samples: [1.661, 1.532, 1.289, 1.268, 1.281] activation: geglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.523 ms -unfused TE avg: 1.913 ms -speedup vs TE: 1.26x -fused samples: [1.415, 1.398, 1.515, 1.626, 1.66] -TE samples: [2.082, 1.878, 1.881, 1.867, 1.858] +fused cuDNN avg: 1.596 ms +unfused TE avg: 1.940 ms +speedup vs TE: 1.22x +fused samples: [1.371, 1.542, 1.681, 1.705, 1.678] +TE samples: [2.133, 1.898, 1.867, 1.902, 1.901] activation: dgeglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.892 ms -unfused TE avg: 1.840 ms -speedup vs TE: 2.06x -fused samples: [0.812, 0.832, 0.9, 0.932, 0.982] -TE samples: [2.319, 1.805, 1.723, 1.691, 1.664] +fused cuDNN avg: 0.889 ms +unfused TE avg: 1.839 ms +speedup vs TE: 2.07x +fused samples: [0.817, 0.805, 0.806, 0.999, 1.018] +TE samples: [2.347, 1.738, 1.666, 1.734, 1.711] summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.548 1.882 1.22x -dswiglu 0.884 1.716 1.94x -srelu 0.797 1.043 1.31x -dsrelu 0.811 1.346 1.66x -geglu 1.523 1.913 1.26x -dgeglu 0.892 1.840 2.06x +swiglu 1.557 1.948 1.25x +dswiglu 0.848 1.738 2.05x +srelu 0.792 1.073 1.35x +dsrelu 0.841 1.406 1.67x +geglu 1.596 1.940 1.22x +dgeglu 0.889 1.839 2.07x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB200/results.png b/benchmark/cutedsl_fusion_kernels/results/GB200/results.png old mode 100644 new mode 100755 index f7d94dfaa..bc96a410e Binary files a/benchmark/cutedsl_fusion_kernels/results/GB200/results.png and b/benchmark/cutedsl_fusion_kernels/results/GB200/results.png differ diff --git a/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt b/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt old mode 100644 new mode 100755 index 2131c998b..6b4adbbc9 --- a/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB300/output.txt @@ -8,9 +8,9 @@ nvidia-cutlass-dsl: 4.4.1 summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.319 1.650 1.25x -dswiglu 0.737 1.551 2.10x -srelu 0.676 0.886 1.31x -dsrelu 0.665 1.143 1.72x -geglu 1.329 1.653 1.24x -dgeglu 0.742 1.637 2.20x +swiglu 1.323 1.627 1.23x +dswiglu 0.745 1.542 2.07x +srelu 0.674 0.881 1.31x +dsrelu 0.691 1.196 1.73x +geglu 1.324 1.641 1.24x +dgeglu 0.743 1.647 2.22x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt b/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt old mode 100644 new mode 100755 index 20de3a7b5..6a5cfa20f --- a/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt +++ b/benchmark/cutedsl_fusion_kernels/results/GB300/output_verbose.txt @@ -10,62 +10,62 @@ timing mode: CUDA graph replay activation: swiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.319 ms -unfused TE avg: 1.650 ms -speedup vs TE: 1.25x -fused samples: [1.208, 1.278, 1.354, 1.385, 1.371] -TE samples: [1.755, 1.632, 1.587, 1.602, 1.674] +fused cuDNN avg: 1.323 ms +unfused TE avg: 1.627 ms +speedup vs TE: 1.23x +fused samples: [1.198, 1.278, 1.321, 1.41, 1.406] +TE samples: [1.768, 1.644, 1.594, 1.53, 1.599] activation: dswiglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.737 ms -unfused TE avg: 1.551 ms -speedup vs TE: 2.10x -fused samples: [0.704, 0.704, 0.741, 0.771, 0.767] -TE samples: [1.961, 1.464, 1.463, 1.437, 1.432] +fused cuDNN avg: 0.745 ms +unfused TE avg: 1.542 ms +speedup vs TE: 2.07x +fused samples: [0.704, 0.708, 0.749, 0.752, 0.812] +TE samples: [1.908, 1.479, 1.448, 1.441, 1.435] activation: srelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=None, activation output N=4096 -fused cuDNN avg: 0.676 ms -unfused TE avg: 0.886 ms +fused cuDNN avg: 0.674 ms +unfused TE avg: 0.881 ms speedup vs TE: 1.31x -fused samples: [0.646, 0.643, 0.656, 0.716, 0.716] -TE samples: [0.943, 0.952, 0.871, 0.832, 0.832] +fused samples: [0.641, 0.642, 0.688, 0.682, 0.718] +TE samples: [0.955, 0.956, 0.853, 0.818, 0.822] activation: dsrelu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=4096, activation output N=4096 -fused cuDNN avg: 0.665 ms -unfused TE avg: 1.143 ms -speedup vs TE: 1.72x -fused samples: [0.65, 0.661, 0.669, 0.66, 0.685] -TE samples: [1.341, 1.132, 1.092, 1.077, 1.074] +fused cuDNN avg: 0.691 ms +unfused TE avg: 1.196 ms +speedup vs TE: 1.73x +fused samples: [0.653, 0.65, 0.662, 0.738, 0.754] +TE samples: [1.379, 1.332, 1.097, 1.088, 1.081] activation: geglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=8192, fused C input N=None, activation output N=4096 -fused cuDNN avg: 1.329 ms -unfused TE avg: 1.653 ms +fused cuDNN avg: 1.324 ms +unfused TE avg: 1.641 ms speedup vs TE: 1.24x -fused samples: [1.21, 1.3, 1.361, 1.395, 1.377] -TE samples: [1.721, 1.662, 1.613, 1.685, 1.582] +fused samples: [1.2, 1.288, 1.322, 1.402, 1.409] +TE samples: [1.771, 1.623, 1.568, 1.633, 1.612] activation: dgeglu shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096 fused GEMM N=4096, fused C input N=8192, activation output N=8192 -fused cuDNN avg: 0.742 ms -unfused TE avg: 1.637 ms -speedup vs TE: 2.20x -fused samples: [0.711, 0.718, 0.745, 0.746, 0.793] -TE samples: [2.005, 1.565, 1.544, 1.538, 1.531] +fused cuDNN avg: 0.743 ms +unfused TE avg: 1.647 ms +speedup vs TE: 2.22x +fused samples: [0.71, 0.711, 0.767, 0.76, 0.765] +TE samples: [2.033, 1.567, 1.553, 1.543, 1.541] summary: activation cudnn_ms te_ms te/cudnn -swiglu 1.319 1.650 1.25x -dswiglu 0.737 1.551 2.10x -srelu 0.676 0.886 1.31x -dsrelu 0.665 1.143 1.72x -geglu 1.329 1.653 1.24x -dgeglu 0.742 1.637 2.20x +swiglu 1.323 1.627 1.23x +dswiglu 0.745 1.542 2.07x +srelu 0.674 0.881 1.31x +dsrelu 0.691 1.196 1.73x +geglu 1.324 1.641 1.24x +dgeglu 0.743 1.647 2.22x diff --git a/benchmark/cutedsl_fusion_kernels/results/GB300/results.png b/benchmark/cutedsl_fusion_kernels/results/GB300/results.png old mode 100644 new mode 100755 index 5796bea40..379029eb9 Binary files a/benchmark/cutedsl_fusion_kernels/results/GB300/results.png and b/benchmark/cutedsl_fusion_kernels/results/GB300/results.png differ