diff --git a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/dsv3_20260227_034744.csv b/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/dsv3_20260227_034744.csv deleted file mode 100644 index 9a8a3a5c3..000000000 --- a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/dsv3_20260227_034744.csv +++ /dev/null @@ -1,55 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,fwd_time_ms,bwd_time_ms,fwd_tflops,bwd_tflops,max_diff,num_iterations,sliding_window_size,success,error_message,gpu_name,cudnn_version,cudnn_backend_version -dsv3,dsv3,cudnn,bfloat16,top_left,1,32768,32768,128,128,192,128,both,True,inf,inf,0.000,0.000,0.000,10,,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 806, in - workspace = torch.empty( - ^^^^^^^^^^^^ -torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 256.02 GiB. GPU 0 has a total capacity of 139.80 GiB of which 129.00 GiB is free. Including non-PyTorch memory, this process has 10.79 GiB memory in use. Of the allocated memory 10.27 GiB is allocated by PyTorch, and 0 bytes is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) - -stdout: ",,, -dsv3,dsv3,cudnn,fp8,top_left,1,32768,32768,128,128,192,128,both,True,inf,inf,0.000,0.000,0.000,10,,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -dsv3,dsv3,cudnn,bfloat16,top_left,1,16384,16384,128,128,192,128,both,True,inf,inf,0.000,0.000,0.000,10,,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 1165, in - workspace = torch.empty( - ^^^^^^^^^^^^ -torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 64.01 GiB. GPU 0 has a total capacity of 139.80 GiB of which 6.38 GiB is free. Including non-PyTorch memory, this process has 133.42 GiB memory in use. Of the allocated memory 69.52 GiB is allocated by PyTorch, and 63.27 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) - -stdout: ",,, -dsv3,dsv3,cudnn,fp8,top_left,1,16384,16384,128,128,192,128,both,True,inf,inf,0.000,0.000,0.000,10,,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -dsv3,dsv3,cudnn,bfloat16,top_left,1,8192,8192,128,128,192,128,both,True,4.278,16.162,643.000,442.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901.000 -dsv3,dsv3,cudnn,fp8,top_left,1,8192,8192,128,128,192,128,both,True,inf,inf,0.000,0.000,0.000,10,,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -dsv3,dsv3,cudnn,bfloat16,top_left,1,4096,4096,128,128,192,128,both,True,1.148,4.424,599.000,404.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901.000 -dsv3,dsv3,cudnn,fp8,top_left,1,4096,4096,128,128,192,128,both,True,inf,inf,0.000,0.000,0.000,10,,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -dsv3,dsv3,cudnn,bfloat16,top_left,1,2048,2048,128,128,192,128,both,True,0.330,1.271,522.000,352.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901.000 -dsv3,dsv3,cudnn,fp8,top_left,1,2048,2048,128,128,192,128,both,True,inf,inf,0.000,0.000,0.000,10,,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, diff --git a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/dsv3_top_left.png b/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/dsv3_top_left.png deleted file mode 100644 index f2f7d7e41..000000000 Binary files a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/dsv3_top_left.png and /dev/null differ diff --git a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/gpt_oss_20260227_034819.csv b/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/gpt_oss_20260227_034819.csv deleted file mode 100644 index aef65694d..000000000 --- a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/gpt_oss_20260227_034819.csv +++ /dev/null @@ -1,41 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,fwd_time_ms,bwd_time_ms,fwd_tflops,bwd_tflops,max_diff,num_iterations,sliding_window_size,success,error_message,gpu_name,cudnn_version,cudnn_backend_version -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,32768,32768,64,8,64,64,both,False,3.021,9.045,358.000,299.000,0.000,10,1024,True,,NVIDIA H200,1.19.0,91901.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,32768,32768,64,8,64,64,both,False,inf,inf,0.000,0.000,0.000,10,1024,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,16384,16384,64,8,64,64,both,False,1.499,4.392,355.000,303.000,0.000,10,1024,True,,NVIDIA H200,1.19.0,91901.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,16384,16384,64,8,64,64,both,False,inf,inf,0.000,0.000,0.000,10,1024,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,8192,8192,64,8,64,64,both,False,0.738,2.149,349.000,300.000,0.000,10,1024,True,,NVIDIA H200,1.19.0,91901.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,8192,8192,64,8,64,64,both,False,inf,inf,0.000,0.000,0.000,10,1024,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,4096,4096,64,8,64,64,both,False,0.358,1.042,336.000,289.000,0.000,10,1024,True,,NVIDIA H200,1.19.0,91901.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,4096,4096,64,8,64,64,both,False,inf,inf,0.000,0.000,0.000,10,1024,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, -gpt_oss,gpt_oss,cudnn,bfloat16,top_left,2,2048,2048,64,8,64,64,both,False,0.166,0.488,310.000,264.000,0.000,10,1024,True,,NVIDIA H200,1.19.0,91901.000 -gpt_oss,gpt_oss,cudnn,fp8,top_left,2,2048,2048,64,8,64,64,both,False,inf,inf,0.000,0.000,0.000,10,1024,False,"Benchmark failed with return code 1. -stderr: Traceback (most recent call last): - File ""/workspace/cudnn_frontend/benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py"", line 731, in - graph_bwd.validate() -cudnn._compiled_module.cudnnGraphNotSupportedError: hidden_dim shoud be equal to 128 and hidden_dim should be multiple of 16 - -stdout: ",,, diff --git a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/gpt_oss_top_left.png b/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/gpt_oss_top_left.png deleted file mode 100644 index 37cc06b06..000000000 Binary files a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/gpt_oss_top_left.png and /dev/null differ diff --git a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_20260227_034703.csv b/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_20260227_034703.csv deleted file mode 100644 index b5b473c56..000000000 --- a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_20260227_034703.csv +++ /dev/null @@ -1,21 +0,0 @@ -config_name,model_name,backend,data_type,attn_mask,batch_size,q_seqlen,kv_seqlen,num_q_heads,num_kv_heads,head_dim_qk,head_dim_vo,profile_pass,deterministic_bwd,fwd_time_ms,bwd_time_ms,fwd_tflops,bwd_tflops,max_diff,num_iterations,sliding_window_size,success,error_message,gpu_name,cudnn_version,cudnn_backend_version -llama3.1,llama3.1,cudnn,bfloat16,top_left,1,32768,32768,64,8,128,128,both,False,25.566,75.688,688.000,581.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,1,32768,32768,64,8,128,128,both,False,50.769,153.841,693.000,572.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,top_left,1,32768,32768,64,8,128,128,both,False,16.884,118.729,1042.000,370.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,no_mask,1,32768,32768,64,8,128,128,both,False,33.120,130.528,1062.000,674.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,top_left,1,16384,16384,64,8,128,128,both,False,6.531,19.002,673.000,579.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,1,16384,16384,64,8,128,128,both,False,12.524,35.947,702.000,612.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,top_left,1,16384,16384,64,8,128,128,both,False,4.357,29.422,1010.000,374.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,no_mask,1,16384,16384,64,8,128,128,both,False,8.335,33.355,1055.000,659.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,top_left,1,8192,8192,64,8,128,128,both,False,1.708,5.083,644.000,541.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,1,8192,8192,64,8,128,128,both,False,3.182,9.262,691.000,594.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,top_left,1,8192,8192,64,8,128,128,both,False,1.141,6.392,964.000,430.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,no_mask,1,8192,8192,64,8,128,128,both,False,2.149,8.637,1023.000,636.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,top_left,1,4096,4096,64,8,128,128,both,False,0.465,1.450,591.000,474.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,1,4096,4096,64,8,128,128,both,False,0.819,2.453,671.000,560.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,top_left,1,4096,4096,64,8,128,128,both,False,0.322,1.459,854.000,471.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,no_mask,1,4096,4096,64,8,128,128,both,False,0.565,2.285,973.000,602.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,top_left,1,2048,2048,64,8,128,128,both,False,0.142,0.464,484.000,370.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,bfloat16,no_mask,1,2048,2048,64,8,128,128,both,False,0.217,0.681,633.000,504.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,top_left,1,2048,2048,64,8,128,128,both,False,0.098,0.437,698.000,393.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 -llama3.1,llama3.1,cudnn,fp8,no_mask,1,2048,2048,64,8,128,128,both,False,0.158,0.647,870.000,531.000,0.000,10,,True,,NVIDIA H200,1.19.0,91901 diff --git a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_no_mask.png b/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_no_mask.png deleted file mode 100644 index ac4c3ecdb..000000000 Binary files a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_no_mask.png and /dev/null differ diff --git a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_top_left.png b/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_top_left.png deleted file mode 100644 index cc27cc778..000000000 Binary files a/benchmark/sdpa_benchmark_training/results/h200_919_only_cudnn/llama3.1_top_left.png and /dev/null differ