[Triton/Gluon] DSR1 GEMM tuning

aiter/configs/model_configs/dsr1_bf16_tuned_gemm.csv

@@ -1,15 +1,14 @@
 gfx,cu_num,M,N,K,bias,dtype,outdtype,scaleAB,bpreshuffle,libtype,solidx,splitK,us,kernelName,err_ratio,tflops,bw
-gfx1250,256,768,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21270,0,26.1096,opus_gemm_gfx1250_4wave_wl_co_128x64x128x128_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,890.6,1705.57
-gfx1250,256,1280,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21148,0,29.1484,opus_gemm_gfx1250_4wave_wl_co_128x96x128x128_4x1_16x16x32_0x0x0_w2x2_c2x2_p3_v1024w1,0.0,1329.59,1853.77
-gfx1250,256,1536,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21153,0,31.3278,opus_gemm_gfx1250_4wave_wl_co_128x96x192x128_4x1_16x16x32_0x0x0_w2x2_c2x2_p3_v1024w1,0.0,1484.51,1876.47
-gfx1250,256,1792,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21108,0,33.0084,opus_gemm_gfx1250_4wave_wl_co_128x128x128x128_4x1_16x16x32_0x0x0_w4x1_c2x2_p3_v1024w1,0.0,1643.75,1924.88
+gfx1250,256,768,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,27.1251,auto,0.0,857.26,1641.72
+gfx1250,256,1280,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,28.9209,auto,0.0,1340.05,1868.35
+gfx1250,256,1792,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,32.9531,auto,0.0,1646.51,1928.11
 gfx1250,256,2560,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21174,0,40.0694,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,1934.41,1941.41
-gfx1250,256,3072,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21177,0,47.6917,opus_gemm_gfx1250_4wave_wl_co_128x128x256x128_4x1_16x16x32_0x0x0_w2x2_c4x1_p3_v1024w1,0.0,1950.3,1830.38
-gfx1250,256,3584,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21177,0,53.4387,opus_gemm_gfx1250_4wave_wl_co_128x128x256x128_4x1_16x16x32_0x0x0_w2x2_c4x1_p3_v1024w1,0.0,2030.65,1811.36
-gfx1250,256,4608,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21158,0,81.2315,opus_gemm_gfx1250_4wave_wl_co_128x96x256x128_4x1_16x16x32_0x0x0_w2x2_c2x2_p3_v1024w1,0.0,1717.55,1425.58
+gfx1250,256,3072,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,51.0414,auto,0.0,1822.3,1710.26
+gfx1250,256,3584,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,54.8421,auto,0.0,1978.68,1765.01
+gfx1250,256,4608,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21317,0,81.796,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w2x2_c4x4_p4_v1024w1,0.0,1705.7,1415.74
 gfx1250,256,5120,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21174,0,86.2983,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,1796.34,1452.0
-gfx1250,256,5632,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21172,0,99.9779,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w2x2_c4x1_p3_v1024w1,0.0,1705.61,1348.37
-gfx1250,256,6144,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21154,0,110.0463,opus_gemm_gfx1250_4wave_wl_co_128x96x192x128_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,1690.43,1311.36
+gfx1250,256,5632,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,100.1144,auto,0.0,1703.29,1346.54
+gfx1250,256,6144,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,99.1964,auto,0.0,1875.33,1454.79
 gfx1250,256,64,24576,1536,False,torch.bfloat16,torch.bfloat16,False,False,opus,21267,0,8.6347,opus_gemm_gfx1250_4wave_wl_co_128x64x128x128_4x1_16x16x32_0x0x0_w2x2_c1x4_p3_v1024w1,0.0,559.58,9130.58
 gfx1250,256,128,24576,1536,False,torch.bfloat16,torch.bfloat16,False,False,opus,21019,0,9.5319,opus_gemm_gfx1250_4wave_co_128x128x128x128_4x1_16x16x32_0x0x0_c1x4_p3_v1024w1,0.0,1013.82,8621.8
 gfx1250,256,256,24576,1536,False,torch.bfloat16,torch.bfloat16,False,False,opus,21039,0,12.1231,opus_gemm_gfx1250_4wave_co_128x128x192x128_4x1_16x16x32_0x0x0_c2x4_p3_v1024w1,0.0,1594.26,7330.37

aiter/configs/model_configs/dsv3_bf16_tuned_gemm.csv

@@ -61,44 +61,44 @@ gfx942,80,256,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,1020
 gfx942,80,64,3072,1536,False,torch.bfloat16,torch.bfloat16,False,False,opus,10003,0,16.2504,opus_gemm_gfx942_p1_bk128_256x64x64x128_2x2_16x16x16_0x0x0,0.0,37.17,617.03
 gfx942,80,80,7168,2048,False,torch.bfloat16,torch.bfloat16,False,False,opus,10000,0,42.8315,opus_gemm_gfx942_512x128x128x64_2x4_16x16x16_0x0x0,0.0,54.84,719.91
 gfx942,80,112,7168,2048,False,torch.bfloat16,torch.bfloat16,False,False,opus,10000,0,49.8753,opus_gemm_gfx942_512x128x128x64_2x4_16x16x16_0x0x0,0.0,65.93,630.06
-gfx1250,256,1,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20129,11,4.9754,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x4_p3w2,0.0,0.74,740.62
-gfx1250,256,2,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20129,13,5.1593,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x4_p3w2,0.0,1.42,717.1
-gfx1250,256,4,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20121,11,5.3272,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x2_p3w2,0.0,2.76,700.07
-gfx1250,256,8,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20121,13,5.3681,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x2_p3w2,0.0,5.47,705.8
-gfx1250,256,16,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20121,10,5.55,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x2_p3w2,0.0,10.58,704.07
-gfx1250,256,32,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20002,6,5.6137,opus_gemm_gfx1250_splitk_cluster_tdm_ws_128x16x32x512_1x2_16x16x32_0x0x0_p3w1,0.0,20.92,738.4
-gfx1250,256,48,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20119,6,5.6916,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c3x1_p3w2,0.0,30.95,770.03
-gfx1250,256,64,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20120,6,5.7774,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c4x1_p3w2,0.0,40.66,799.72
-gfx1250,256,80,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20255,10,5.8587,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c1x4_p3w2,0.0,50.11,829.17
-gfx1250,256,96,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20119,6,5.8792,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c3x1_p3w2,0.0,59.93,866.69
-gfx1250,256,112,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20129,6,6.0314,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x4_p3w2,0.0,68.15,884.2
-gfx1250,256,128,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20124,6,6.1139,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c4x2_p3w2,0.0,76.84,911.13
-gfx1250,256,256,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20258,6,6.6006,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c4x4_p3w2,0.0,142.34,1131.88
-gfx1250,256,512,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20256,3,8.0343,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c2x4_p3w2,0.0,233.88,1403.01
-gfx1250,256,1024,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20286,3,10.2302,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x64x256_2x1_16x16x32_0x0x0_c4x2_p3w2,0.0,367.35,1844.97
-gfx1250,256,2048,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21315,0,13.7767,opus_gemm_gfx1250_4wave_wl_co_128x32x64x256_4x1_16x16x32_0x0x0_w1x4_c4x4_p3_v1024w1,0.0,545.57,2473.65
-gfx1250,256,4096,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21310,0,15.0299,opus_gemm_gfx1250_4wave_wl_co_128x64x64x256_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,1000.17,4290.61
-gfx1250,256,8192,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21148,0,27.6301,opus_gemm_gfx1250_4wave_wl_co_128x96x128x128_4x1_16x16x32_0x0x0_w2x2_c2x2_p3_v1024w1,0.0,1088.12,4535.08
+gfx1250,256,1,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,4.7332,auto,0.0,0.78,778.51
+gfx1250,256,2,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,4.7075,auto,0.0,1.56,785.92
+gfx1250,256,4,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,4.9668,auto,0.0,2.96,750.87
+gfx1250,256,8,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,4.8219,auto,0.0,6.09,785.75
+gfx1250,256,16,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,4.921,auto,0.0,11.93,794.06
+gfx1250,256,32,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.0189,auto,0.0,23.4,825.91
+gfx1250,256,48,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.2334,auto,0.0,33.66,837.45
+gfx1250,256,64,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.0355,auto,0.0,46.65,917.54
+gfx1250,256,80,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.1882,auto,0.0,56.59,936.33
+gfx1250,256,96,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.8073,auto,0.0,60.67,877.42
+gfx1250,256,112,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.5322,auto,0.0,74.3,963.99
+gfx1250,256,128,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.7715,auto,0.0,81.39,965.18
+gfx1250,256,256,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,6.4826,auto,0.0,144.93,1152.49
+gfx1250,256,512,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,7.8909,auto,0.0,238.13,1428.51
+gfx1250,256,1024,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,6.3,auto,0.0,596.52,2995.93
+gfx1250,256,2048,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,8.0,auto,0.0,939.52,4259.84
+gfx1250,256,4096,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,20.193,auto,0.0,744.44,3193.55
+gfx1250,256,8192,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,35.3213,auto,0.0,851.18,3547.57
 gfx1250,256,16384,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21177,0,37.4735,opus_gemm_gfx1250_4wave_wl_co_128x128x256x128_4x1_16x16x32_0x0x0_w2x2_c4x1_p3_v1024w1,0.0,1604.59,6589.71
-gfx1250,256,1,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20129,6,6.3985,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x4_p3w2,0.0,4.73,4734.89
-gfx1250,256,2,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20129,6,6.4956,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x4_p3w2,0.0,9.32,4666.97
-gfx1250,256,4,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20121,6,6.4565,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x2_p3w2,0.0,18.76,4700.98
-gfx1250,256,8,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20129,6,6.5542,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x4_p3w2,0.0,36.96,4642.23
-gfx1250,256,16,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20129,6,6.6767,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x16x32x256_1x2_16x16x32_0x0x0_c1x4_p3w2,0.0,72.56,4579.3
-gfx1250,256,32,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20255,6,7.2364,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c1x4_p3w2,0.0,133.89,4266.15
-gfx1250,256,48,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20256,3,8.1694,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c2x4_p3w2,0.0,177.9,3815.28
-gfx1250,256,64,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20256,3,8.1997,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c2x4_p3w2,0.0,236.32,3837.39
-gfx1250,256,80,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20249,2,9.681,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c3x2_p3w2,0.0,250.2,3280.9
-gfx1250,256,96,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20249,2,9.8315,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x32x32x256_2x1_16x16x32_0x0x0_c3x2_p3w2,0.0,295.65,3260.89
-gfx1250,256,112,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20374,3,10.0861,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x64x32x256_2x1_16x16x32_0x0x0_c2x2_p3w2,0.0,336.21,3208.01
-gfx1250,256,128,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20374,3,10.1724,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x64x32x256_2x1_16x16x32_0x0x0_c2x2_p3w2,0.0,380.99,3209.99
-gfx1250,256,256,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21310,0,14.5492,opus_gemm_gfx1250_4wave_wl_co_128x64x64x256_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,532.75,2407.62
-gfx1250,256,512,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,20518,3,23.8238,opus_gemm_gfx1250_splitk_clusterlaunch_tdm_ws_128x128x64x128_2x1_16x16x32_0x0x0_c2x2_p3w2,0.0,650.7,1669.77
-gfx1250,256,1024,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21133,0,27.6728,opus_gemm_gfx1250_4wave_wl_co_128x64x192x128_4x1_16x16x32_0x0x0_w2x2_c2x2_p3_v1024w1,0.0,1120.39,1780.92
-gfx1250,256,2048,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21170,0,35.43,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w2x2_c1x1_p3_v1024w1,0.0,1750.17,1927.42
-gfx1250,256,4096,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21046,0,72.3822,opus_gemm_gfx1250_4wave_co_128x128x192x128_4x1_16x16x32_0x0x0_c4x3_p3_v1024w1,0.0,1713.37,1468.58
-gfx1250,256,8192,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21230,0,133.3386,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w1x4_c1x1_p3_v1024w1,0.0,1860.18,1367.35
-gfx1250,256,16384,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21177,0,247.3669,opus_gemm_gfx1250_4wave_wl_co_128x128x256x128_4x1_16x16x32_0x0x0_w2x2_c4x1_p3_v1024w1,0.0,2005.4,1351.7
+gfx1250,256,1,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.455,auto,0.0,5.55,5553.84
+gfx1250,256,2,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.4481,auto,0.0,11.11,5564.28
+gfx1250,256,4,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,5.7673,auto,0.0,21.0,5262.75
+gfx1250,256,8,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,6.0975,auto,0.0,39.72,4989.93
+gfx1250,256,16,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,6.6581,auto,0.0,72.76,4592.09
+gfx1250,256,32,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,6.5043,auto,0.0,148.96,4746.33
+gfx1250,256,48,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,7.6463,auto,0.0,190.07,4076.29
+gfx1250,256,64,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,7.7773,auto,0.0,249.16,4045.81
+gfx1250,256,80,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,9.8507,auto,0.0,245.89,3224.38
+gfx1250,256,96,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,10.3088,auto,0.0,281.96,3109.91
+gfx1250,256,112,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,9.7422,auto,0.0,348.08,3321.26
+gfx1250,256,128,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,10.2494,auto,0.0,378.12,3185.88
+gfx1250,256,256,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,13.6402,auto,0.0,568.25,2568.07
+gfx1250,256,512,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,13.7,auto,0.0,1131.54,2903.68
+gfx1250,256,1024,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,21.8,auto,0.0,1422.22,2260.69
+gfx1250,256,2048,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21174,0,41.5109,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,1493.79,1645.07
+gfx1250,256,4096,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21316,0,77.5757,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w4x1_c4x4_p4_v1024w1,0.0,1598.66,1370.27
+gfx1250,256,8192,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21174,0,135.0662,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,1836.39,1349.87
+gfx1250,256,16384,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21317,0,261.1141,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w2x2_c4x4_p4_v1024w1,0.0,1899.82,1280.53
 gfx1250,256,1,3072,1536,False,torch.bfloat16,torch.bfloat16,False,False,opus,21312,0,4.6779,opus_gemm_gfx1250_4wave_wl_co_128x32x64x256_4x1_16x16x32_0x0x0_w1x4_c1x4_p3_v1024w1,0.0,2.02,2019.37
 gfx1250,256,2,3072,1536,False,torch.bfloat16,torch.bfloat16,False,False,opus,21312,0,4.7064,opus_gemm_gfx1250_4wave_wl_co_128x32x64x256_4x1_16x16x32_0x0x0_w1x4_c1x4_p3_v1024w1,0.0,4.01,2009.1
 gfx1250,256,4,3072,1536,False,torch.bfloat16,torch.bfloat16,False,False,opus,21312,0,4.774,opus_gemm_gfx1250_4wave_wl_co_128x32x64x256_4x1_16x16x32_0x0x0_w1x4_c1x4_p3_v1024w1,0.0,7.91,1984.51
@@ -175,3 +175,6 @@ gfx1250,256,2048,129280,7168,False,torch.bfloat16,torch.bfloat16,False,False,opu
 gfx1250,256,4096,129280,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21234,0,4731.0094,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w1x4_c4x4_p3_v1024w1,0.0,1604.6,628.01
 gfx1250,256,8192,129280,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21234,0,9353.0764,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w1x4_c4x4_p3_v1024w1,0.0,1623.29,437.17
 gfx1250,256,16384,129280,7168,False,torch.bfloat16,torch.bfloat16,False,False,opus,21179,0,19801.9643,opus_gemm_gfx1250_4wave_wl_co_128x128x256x128_4x1_16x16x32_0x0x0_w2x2_c4x4_p3_v1024w1,0.0,1533.45,319.39
+gfx1250,256,1536,2112,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,28.6,auto,0.0,1626.10,2055.45
+gfx1250,256,1536,256,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,7.6,auto,0.0,741.73,3483.76
+gfx1250,256,6144,32320,7168,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,1124.4,auto,0.0,2531.80,843.62

aiter/ops/triton/_gluon_kernels/gfx1250/gemm/basic/gemm_a16w16.py

@@ -27,28 +27,44 @@
 )
 
 
+# TDM encodes log2(pad interval in dwords) - 1 in a 3-bit field, so the
+# interval cannot exceed 2^8 = 256 dwords (TDMUtility.cpp createTDMDescriptor
+# asserts log2PadIntervalDwords <= 8).
+_MAX_PAD_INTERVAL_DWORDS = 256
+
+
+def _pad_interval(extent, elem_bits):
+    """Largest encodable pad interval, in elements, not exceeding `extent`."""
+    return min(extent, _MAX_PAD_INTERVAL_DWORDS * 32 // elem_bits)
+
+
 def create_shared_layouts(
     BLOCK_M: gl.constexpr,
     BLOCK_N: gl.constexpr,
     BLOCK_K: gl.constexpr,
     LAYOUT: gl.constexpr,
+    elem_bits: int = 16,
+    elem_bits_b: int | None = None,
 ):
+    """elem_bits sizes A's pad interval; elem_bits_b sizes B's (defaults to elem_bits)."""
+    if elem_bits_b is None:
+        elem_bits_b = elem_bits
     if LAYOUT[0] == "T":
         SHARED_LAYOUT_A: gl.constexpr = gl.PaddedSharedLayout.with_identity_for(
-            [[BLOCK_K, 8]], [BLOCK_M, BLOCK_K], [1, 0]
+            [[_pad_interval(BLOCK_K, elem_bits), 8]], [BLOCK_M, BLOCK_K], [1, 0]
         )
     else:
         SHARED_LAYOUT_A: gl.constexpr = gl.PaddedSharedLayout.with_identity_for(
-            [[BLOCK_M, 8]], [BLOCK_K, BLOCK_M], [1, 0]
+            [[_pad_interval(BLOCK_M, elem_bits), 8]], [BLOCK_K, BLOCK_M], [1, 0]
         )
 
     if LAYOUT[1] == "T":
         SHARED_LAYOUT_B: gl.constexpr = gl.PaddedSharedLayout.with_identity_for(
-            [[BLOCK_N, 16]], [BLOCK_K, BLOCK_N], [1, 0]
+            [[_pad_interval(BLOCK_N, elem_bits_b), 16]], [BLOCK_K, BLOCK_N], [1, 0]
         )
     else:
         SHARED_LAYOUT_B: gl.constexpr = gl.PaddedSharedLayout.with_identity_for(
-            [[BLOCK_K, 8]], [BLOCK_N, BLOCK_K], [1, 0]
+            [[_pad_interval(BLOCK_K, elem_bits_b), 8]], [BLOCK_N, BLOCK_K], [1, 0]
         )
 
     return (SHARED_LAYOUT_A, SHARED_LAYOUT_B)