diff --git a/aiter/configs/a8w8_bpreshuffle_tuned_gemm.csv b/aiter/configs/a8w8_bpreshuffle_tuned_gemm.csv index 64f49e9e286..e749f55f511 100644 --- a/aiter/configs/a8w8_bpreshuffle_tuned_gemm.csv +++ b/aiter/configs/a8w8_bpreshuffle_tuned_gemm.csv @@ -597,3 +597,93 @@ gfx1250,256,4096,32768,512,torch.float8_e4m3fn,flydsl,584,0,74.9556,flydsl_bpres gfx1250,256,8192,32768,512,torch.float8_e4m3fn,flydsl,584,0,137.2238,flydsl_bpreshuffle_wmma_t256x128x128_mw2_nw1_nb3_sk1_cm1_cn1,2003.14,4065.2,0.0 gfx1250,256,16384,32768,512,torch.float8_e4m3fn,flydsl,584,0,260.1697,flydsl_bpreshuffle_wmma_t256x128x128_mw2_nw1_nb3_sk1_cm1_cn1,2113.07,4223.81,0.0 gfx1250,256,32768,32768,512,torch.float8_e4m3fn,flydsl,584,0,505.4037,flydsl_bpreshuffle_wmma_t256x128x128_mw2_nw1_nb3_sk1_cm1_cn1,2175.51,4315.44,0.0 +gfx1250,256,1,2112,7168,torch.float8_e4m3fn,flydsl,40,0,6.0724,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,4.99,2494.93,0.0066 +gfx1250,256,1,3584,7168,torch.float8_e4m3fn,flydsl,40,0,6.2224,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,8.26,4130.95,0.0075 +gfx1250,256,1,7168,1024,torch.float8_e4m3fn,flydsl,20,0,3.6692,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,4.0,2004.63,0.0 +gfx1250,256,1,7168,1792,torch.float8_e4m3fn,flydsl,20,0,4.4383,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,5.79,2897.77,0.0 +gfx1250,256,1,12288,1536,torch.float8_e4m3fn,flydsl,28,0,4.1035,flydsl_bpreshuffle_wmma_t16x64x512_mw1_nw4_nb3_sk1_cm1_cn1,9.2,4605.94,0.0 +gfx1250,256,2,2112,7168,torch.float8_e4m3fn,flydsl,40,0,6.0565,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,10.0,2503.36,0.0083 +gfx1250,256,2,3584,7168,torch.float8_e4m3fn,flydsl,40,0,6.2271,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,16.5,4130.14,0.0098 +gfx1250,256,2,7168,1024,torch.float8_e4m3fn,flydsl,20,0,3.7166,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,7.9,1983.2,0.0 +gfx1250,256,2,7168,1792,torch.float8_e4m3fn,flydsl,20,0,4.513,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,11.38,2853.38,0.0 +gfx1250,256,2,12288,1536,torch.float8_e4m3fn,flydsl,28,0,4.1548,flydsl_bpreshuffle_wmma_t16x64x512_mw1_nw4_nb3_sk1_cm1_cn1,18.17,4555.36,0.0 +gfx1250,256,4,2112,7168,torch.float8_e4m3fn,flydsl,40,0,6.0783,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,19.93,2498.13,0.0086 +gfx1250,256,4,3584,7168,torch.float8_e4m3fn,flydsl,40,0,6.306,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,32.59,4083.01,0.0084 +gfx1250,256,4,7168,1024,torch.float8_e4m3fn,flydsl,20,0,3.7272,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,15.75,1985.8,0.0 +gfx1250,256,4,7168,1792,torch.float8_e4m3fn,flydsl,20,0,4.4972,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,22.85,2870.58,0.0 +gfx1250,256,4,12288,1536,torch.float8_e4m3fn,flydsl,28,0,4.1863,flydsl_bpreshuffle_wmma_t16x64x512_mw1_nw4_nb3_sk1_cm1_cn1,36.07,4533.55,0.0 +gfx1250,256,8,2112,7168,torch.float8_e4m3fn,flydsl,40,0,6.1235,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,39.56,2487.13,0.0098 +gfx1250,256,8,3584,7168,torch.float8_e4m3fn,flydsl,40,0,6.328,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,64.96,4077.88,0.0091 +gfx1250,256,8,7168,1024,torch.float8_e4m3fn,flydsl,20,0,3.7676,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,31.17,1980.81,0.0 +gfx1250,256,8,7168,1792,torch.float8_e4m3fn,flydsl,20,0,4.5666,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,45.01,2841.08,0.0 +gfx1250,256,8,12288,1536,torch.float8_e4m3fn,flydsl,28,0,4.2101,flydsl_bpreshuffle_wmma_t16x64x512_mw1_nw4_nb3_sk1_cm1_cn1,71.73,4532.73,0.0 +gfx1250,256,16,3584,7168,torch.float8_e4m3fn,flydsl,40,0,6.409,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,128.27,4044.23,0.0096 +gfx1250,256,16,7168,1024,torch.float8_e4m3fn,flydsl,64,0,3.8085,flydsl_bpreshuffle_wmma_t16x128x256_mw1_nw4_nb4_sk1_cm1_cn1,61.67,1991.81,0.0 +gfx1250,256,16,7168,1792,torch.float8_e4m3fn,flydsl,20,0,4.6238,flydsl_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,88.9,2833.84,0.0 +gfx1250,256,16,12288,1536,torch.float8_e4m3fn,flydsl,72,0,4.3024,flydsl_bpreshuffle_wmma_t16x128x512_mw1_nw4_nb3_sk1_cm1_cn1,140.38,4484.05,0.0 +gfx1250,256,32,3584,7168,torch.float8_e4m3fn,flydsl,40,0,6.8857,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,238.78,3797.56,0.0096 +gfx1250,256,32,7168,1024,torch.float8_e4m3fn,flydsl,340,0,3.8819,flydsl_bpreshuffle_wmma_t32x128x256_mw1_nw4_nb4_sk1_cm1_cn1,121.01,2017.45,0.0 +gfx1250,256,32,7168,1792,torch.float8_e4m3fn,flydsl,224,0,4.8013,flydsl_bpreshuffle_wmma_t32x64x256_mw2_nw2_nb4_sk1_cm1_cn1,171.22,2782.82,0.0 +gfx1250,256,32,12288,1536,torch.float8_e4m3fn,flydsl,360,0,4.5135,flydsl_bpreshuffle_wmma_t32x128x512_mw1_nw4_nb3_sk1_cm1_cn1,267.63,4366.89,0.0 +gfx1250,256,48,2112,7168,torch.float8_e4m3fn,flydsl,40,0,6.6046,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,220.05,2374.96,0.0099 +gfx1250,256,48,3584,7168,torch.float8_e4m3fn,flydsl,40,0,7.301,flydsl_bpreshuffle_wmma_t16x64x1024_mw1_nw4_nb3_sk1_cm1_cn1,337.8,3612.96,0.0096 +gfx1250,256,48,7168,1024,torch.float8_e4m3fn,flydsl,68,0,3.9784,flydsl_bpreshuffle_wmma_t16x128x512_mw1_nw4_nb2_sk1_cm1_cn1,177.12,2030.29,0.0 +gfx1250,256,48,7168,1792,torch.float8_e4m3fn,flydsl,340,0,4.9676,flydsl_bpreshuffle_wmma_t32x128x256_mw1_nw4_nb4_sk1_cm1_cn1,248.23,2741.61,0.0 +gfx1250,256,48,12288,1536,torch.float8_e4m3fn,flydsl,780,0,4.6805,flydsl_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,387.13,4300.34,0.0 +gfx1250,256,64,3584,7168,torch.float8_e4m3fn,flydsl,264,0,7.662,flydsl_bpreshuffle_wmma_t32x64x1024_mw2_nw2_nb3_sk1_cm1_cn1,429.17,3472.67,0.0097 +gfx1250,256,64,7168,1024,torch.float8_e4m3fn,flydsl,748,0,4.0259,flydsl_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk1_cm1_cn1,233.37,2067.38,0.0 +gfx1250,256,64,7168,1792,torch.float8_e4m3fn,flydsl,340,0,5.0132,flydsl_bpreshuffle_wmma_t32x128x256_mw1_nw4_nb4_sk1_cm1_cn1,327.97,2768.14,0.0 +gfx1250,256,64,12288,1536,torch.float8_e4m3fn,flydsl,780,0,4.7195,flydsl_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,511.9,4353.33,0.0 +gfx1250,256,128,3584,7168,torch.float8_e4m3fn,flydsl,750,0,8.9002,flydsl_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk4_cm1_cn1,738.93,3092.64,0.0098 +gfx1250,256,128,7168,1024,torch.float8_e4m3fn,flydsl,748,0,4.2482,flydsl_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk1_cm1_cn1,442.32,2190.6,0.0 +gfx1250,256,128,7168,1792,torch.float8_e4m3fn,flydsl,748,0,5.3828,flydsl_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk1_cm1_cn1,610.9,2769.83,0.0 +gfx1250,256,128,12288,1536,torch.float8_e4m3fn,flydsl,780,0,5.529,flydsl_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,873.91,4018.21,0.0 +gfx1250,256,256,3584,7168,torch.float8_e4m3fn,flydsl,781,0,11.4073,flydsl_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk2_cm1_cn1,1153.06,2573.8,0.0095 +gfx1250,256,256,7168,1024,torch.float8_e4m3fn,flydsl,1156,0,4.8612,flydsl_bpreshuffle_wmma_t128x128x512_mw2_nw2_nb2_sk1_cm1_cn1,773.08,2318.81,0.0 +gfx1250,256,256,7168,1792,torch.float8_e4m3fn,flydsl,748,0,6.1838,flydsl_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk1_cm1_cn1,1063.53,2744.89,0.0 +gfx1250,256,256,12288,1536,torch.float8_e4m3fn,flydsl,1120,0,6.7635,flydsl_bpreshuffle_wmma_t128x128x256_mw1_nw4_nb3_sk1_cm1_cn1,1428.8,3778.97,0.0 +gfx1250,256,512,3584,7168,torch.float8_e4m3fn,flydsl,780,0,14.1744,flydsl_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,1855.93,2330.27,0.0097 +gfx1250,256,512,7168,1024,torch.float8_e4m3fn,flydsl,1120,0,5.803,flydsl_bpreshuffle_wmma_t128x128x256_mw1_nw4_nb3_sk1_cm1_cn1,1295.23,2620.08,0.0 +gfx1250,256,512,7168,1792,torch.float8_e4m3fn,flydsl,1132,0,7.5924,flydsl_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb3_sk1_cm1_cn1,1732.43,2779.44,0.0 +gfx1250,256,512,12288,1536,torch.float8_e4m3fn,flydsl,1376,0,8.7695,flydsl_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,2203.93,3676.8,0.0 +gfx1250,256,1024,3584,7168,torch.float8_e4m3fn,flydsl,1136,0,20.349,flydsl_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb4_sk1_cm1_cn1,2585.55,1983.89,0.0096 +gfx1250,256,1024,7168,1024,torch.float8_e4m3fn,flydsl,1376,0,7.6294,flydsl_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,1970.32,3023.65,0.0 +gfx1250,256,1024,7168,1792,torch.float8_e4m3fn,flydsl,1232,0,10.0397,flydsl_bpreshuffle_wmma_t128x256x256_mw1_nw4_nb3_sk1_cm1_cn1,2620.27,2924.4,0.0 +gfx1250,256,1024,12288,1536,torch.float8_e4m3fn,flydsl,1252,0,12.7075,flydsl_bpreshuffle_wmma_t128x384x128_mw1_nw4_nb4_sk1_cm1_cn1,3041.88,3589.46,0.0 +gfx1250,256,2048,3584,7168,torch.float8_e4m3fn,flydsl,1240,0,27.195,flydsl_bpreshuffle_wmma_t128x256x256_mw2_nw2_nb3_sk1_cm1_cn1,3869.34,2024.28,0.0096 +gfx1250,256,2048,7168,1024,torch.float8_e4m3fn,flydsl,1412,0,11.1132,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,2705.32,3491.1,0.0 +gfx1250,256,2048,7168,1792,torch.float8_e4m3fn,flydsl,1412,0,15.1148,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3480.92,3035.12,0.0 +gfx1250,256,2048,12288,1536,torch.float8_e4m3fn,flydsl,1252,0,22.9798,flydsl_bpreshuffle_wmma_t128x384x128_mw1_nw4_nb4_sk1_cm1_cn1,3364.23,3148.49,0.0 +gfx1250,256,4096,3584,7168,torch.float8_e4m3fn,flydsl,1412,0,42.656,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4933.73,1978.86,0.0097 +gfx1250,256,4096,7168,1024,torch.float8_e4m3fn,flydsl,1412,0,20.3685,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,2952.08,3449.18,0.0 +gfx1250,256,4096,7168,1792,torch.float8_e4m3fn,flydsl,1412,0,28.292,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3719.31,2788.96,0.0 +gfx1250,256,4096,12288,1536,torch.float8_e4m3fn,flydsl,1416,0,39.7741,flydsl_bpreshuffle_wmma_t256x256x256_mw2_nw2_nb2_sk1_cm1_cn1,3887.42,3163.59,0.0 +gfx1250,256,8192,3584,7168,torch.float8_e4m3fn,flydsl,1412,0,82.2675,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,5116.32,1739.82,0.0097 +gfx1250,256,8192,7168,1024,torch.float8_e4m3fn,flydsl,1412,0,38.1558,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3151.79,3490.14,0.0 +gfx1250,256,8192,7168,1792,torch.float8_e4m3fn,flydsl,1412,0,51.8032,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4062.56,2798.39,0.0 +gfx1250,256,8192,12288,1536,torch.float8_e4m3fn,flydsl,1416,0,73.3097,flydsl_bpreshuffle_wmma_t256x256x256_mw2_nw2_nb2_sk1_cm1_cn1,4218.24,3175.35,0.0 +gfx1250,256,16384,3584,7168,torch.float8_e4m3fn,flydsl,1412,0,158.2093,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,5320.89,1647.0,0.0097 +gfx1250,256,16384,7168,1024,torch.float8_e4m3fn,flydsl,1412,0,66.0897,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3639.27,3918.89,0.0 +gfx1250,256,16384,7168,1792,torch.float8_e4m3fn,flydsl,1412,0,90.8838,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4631.26,3048.8,0.0 +gfx1250,256,16384,12288,1536,torch.float8_e4m3fn,flydsl,1412,0,140.3126,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4407.84,3183.56,0.0 +gfx1250,256,32768,3584,7168,torch.float8_e4m3fn,flydsl,1412,0,307.5011,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,5475.19,1611.22,0.0097 +gfx1250,256,32768,7168,1024,torch.float8_e4m3fn,flydsl,1412,0,126.7929,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3793.87,4027.49,0.0 +gfx1250,256,32768,7168,1792,torch.float8_e4m3fn,flydsl,1412,0,177.297,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4748.04,3053.22,0.0 +gfx1250,256,32768,12288,1536,torch.float8_e4m3fn,flydsl,1412,0,273.1677,flydsl_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4528.17,3201.38,0.0 +gfx1250,256,1,2048,512,torch.float8_e4m3fn,flydsl,8,0,3.3917,flydsl_bpreshuffle_wmma_t16x64x128_mw1_nw4_nb4_sk1_cm1_cn1,0.62,310.52,0.0 +gfx1250,256,2,2048,512,torch.float8_e4m3fn,flydsl,8,0,3.4198,flydsl_bpreshuffle_wmma_t16x64x128_mw1_nw4_nb4_sk1_cm1_cn1,1.23,309.31,0.0 +gfx1250,256,4,2048,512,torch.float8_e4m3fn,flydsl,8,0,3.4313,flydsl_bpreshuffle_wmma_t16x64x128_mw1_nw4_nb4_sk1_cm1_cn1,2.44,310.96,0.0 +gfx1250,256,8,2048,512,torch.float8_e4m3fn,flydsl,8,0,3.4614,flydsl_bpreshuffle_wmma_t16x64x128_mw1_nw4_nb4_sk1_cm1_cn1,4.85,313.58,0.0 +gfx1250,256,16,2048,512,torch.float8_e4m3fn,flydsl,8,0,3.4945,flydsl_bpreshuffle_wmma_t16x64x128_mw1_nw4_nb4_sk1_cm1_cn1,9.6,321.16,0.0 +gfx1250,256,32,2048,512,torch.float8_e4m3fn,flydsl,8,0,3.4612,flydsl_bpreshuffle_wmma_t16x64x128_mw1_nw4_nb4_sk1_cm1_cn1,19.39,345.55,0.0 +gfx1250,256,48,2048,512,torch.float8_e4m3fn,flydsl,52,0,3.4526,flydsl_bpreshuffle_wmma_t16x128x128_mw1_nw4_nb4_sk1_cm1_cn1,29.16,367.77,0.0 +gfx1250,256,64,2048,512,torch.float8_e4m3fn,flydsl,52,0,3.5116,flydsl_bpreshuffle_wmma_t16x128x128_mw1_nw4_nb4_sk1_cm1_cn1,38.22,382.59,0.0 +gfx1250,256,128,2048,512,torch.float8_e4m3fn,flydsl,52,0,3.5095,flydsl_bpreshuffle_wmma_t16x128x128_mw1_nw4_nb4_sk1_cm1_cn1,76.49,466.85,0.0 +gfx1250,256,256,2048,512,torch.float8_e4m3fn,flydsl,332,0,3.5572,flydsl_bpreshuffle_wmma_t32x128x256_mw1_nw4_nb2_sk1_cm1_cn1,150.93,626.4,0.0 +gfx1250,256,512,2048,512,torch.float8_e4m3fn,flydsl,712,0,3.6951,flydsl_bpreshuffle_wmma_t64x128x128_mw1_nw4_nb4_sk1_cm1_cn1,290.59,922.27,0.0 +gfx1250,256,1024,2048,512,torch.float8_e4m3fn,flydsl,712,0,3.957,flydsl_bpreshuffle_wmma_t64x128x128_mw1_nw4_nb4_sk1_cm1_cn1,542.7,1457.46,0.0 +gfx1250,256,2048,2048,512,torch.float8_e4m3fn,flydsl,1100,0,4.4972,flydsl_bpreshuffle_wmma_t128x128x128_mw2_nw2_nb4_sk1_cm1_cn1,955.03,2331.62,0.0 +gfx1250,256,4096,2048,512,torch.float8_e4m3fn,flydsl,1356,0,5.9174,flydsl_bpreshuffle_wmma_t256x128x128_mw2_nw2_nb4_sk1_cm1_cn1,1451.64,3366.84,0.0 +gfx1250,256,8192,2048,512,torch.float8_e4m3fn,flydsl,1416,0,8.4972,flydsl_bpreshuffle_wmma_t256x256x256_mw2_nw2_nb2_sk1_cm1_cn1,2021.83,4565.89,0.0 +gfx1250,256,16384,2048,512,torch.float8_e4m3fn,flydsl,1416,0,15.1011,flydsl_bpreshuffle_wmma_t256x256x256_mw2_nw2_nb2_sk1_cm1_cn1,2275.31,5068.91,0.0 +gfx1250,256,32768,2048,512,torch.float8_e4m3fn,flydsl,1416,0,28.0945,flydsl_bpreshuffle_wmma_t256x256x256_mw2_nw2_nb2_sk1_cm1_cn1,2446.01,5411.86,0.0 diff --git a/aiter/configs/model_configs/a8w8_bpreshuffle_untuned_gemm_kimik3.csv b/aiter/configs/model_configs/a8w8_bpreshuffle_untuned_gemm_kimik3.csv index 4414e0fe186..3b8309ed839 100644 --- a/aiter/configs/model_configs/a8w8_bpreshuffle_untuned_gemm_kimik3.csv +++ b/aiter/configs/model_configs/a8w8_bpreshuffle_untuned_gemm_kimik3.csv @@ -575,3 +575,28 @@ M,N,K,q_dtype_w 2048,6144,7168,torch.float8_e4m3fn 4096,6144,7168,torch.float8_e4m3fn 8192,6144,7168,torch.float8_e4m3fn +256,2112,7168,torch.float8_e4m3fn +512,2112,7168,torch.float8_e4m3fn +1024,2112,7168,torch.float8_e4m3fn +2048,2112,7168,torch.float8_e4m3fn +4096,2112,7168,torch.float8_e4m3fn +8192,2112,7168,torch.float8_e4m3fn +16384,2112,7168,torch.float8_e4m3fn +32768,2112,7168,torch.float8_e4m3fn +1,12288,1536,torch.float8_e4m3fn +2,12288,1536,torch.float8_e4m3fn +4,12288,1536,torch.float8_e4m3fn +8,12288,1536,torch.float8_e4m3fn +16,12288,1536,torch.float8_e4m3fn +32,12288,1536,torch.float8_e4m3fn +48,12288,1536,torch.float8_e4m3fn +64,12288,1536,torch.float8_e4m3fn +128,12288,1536,torch.float8_e4m3fn +256,12288,1536,torch.float8_e4m3fn +512,12288,1536,torch.float8_e4m3fn +1024,12288,1536,torch.float8_e4m3fn +2048,12288,1536,torch.float8_e4m3fn +4096,12288,1536,torch.float8_e4m3fn +8192,12288,1536,torch.float8_e4m3fn +16384,12288,1536,torch.float8_e4m3fn +32768,12288,1536,torch.float8_e4m3fn diff --git a/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_tuned_gemm.csv b/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_tuned_gemm.csv index 1bb5777b75c..5221749b487 100644 --- a/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_tuned_gemm.csv +++ b/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_tuned_gemm.csv @@ -1,4 +1,21 @@ gfx,cu_num,M,N,K,w_scale_block,libtype,kernelId,splitK,us,kernelName,tflops,bw,errRatio +gfx1250,256,1,1536,128,128x128,flydsl,33,1,2.1209,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb2_sk1_cm1_cn1,0.19,94.21,0.0 +gfx1250,256,2,1536,128,128x128,flydsl,33,1,2.1065,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb2_sk1_cm1_cn1,0.37,96.37,0.0 +gfx1250,256,4,1536,128,128x128,flydsl,33,1,2.1306,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb2_sk1_cm1_cn1,0.74,98.29,0.0 +gfx1250,256,8,1536,128,128x128,flydsl,33,1,2.1401,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb2_sk1_cm1_cn1,1.47,103.83,0.0002 +gfx1250,256,16,1536,128,128x128,flydsl,33,1,2.1772,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb2_sk1_cm1_cn1,2.89,113.82,0.0002 +gfx1250,256,32,1536,128,128x128,flydsl,11,1,2.1592,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x128_mw1_nw4_nb2_sk1_cm1_cn1,5.83,138.48,0.0002 +gfx1250,256,48,1536,128,128x128,flydsl,33,1,2.1458,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb2_sk1_cm1_cn1,8.8,163.21,0.0002 +gfx1250,256,64,1536,128,128x128,flydsl,11,1,2.0963,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x128_mw1_nw4_nb2_sk1_cm1_cn1,12.0,191.48,0.0003 +gfx1250,256,128,1536,128,128x128,flydsl,11,1,2.0239,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x128_mw1_nw4_nb2_sk1_cm1_cn1,24.87,299.52,0.0003 +gfx1250,256,256,1536,128,128x128,flydsl,11,1,2.0486,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x128_mw1_nw4_nb2_sk1_cm1_cn1,49.14,495.85,0.0003 +gfx1250,256,512,1536,128,128x128,flydsl,229,1,2.39,flydsl_mxfp8_128_bpreshuffle_wmma_t256x32x128_mw4_nw1_nb2_sk1_cm1_cn1,84.24,767.79,0.0003 +gfx1250,256,1024,1536,128,128x128,flydsl,70,1,2.2599,flydsl_mxfp8_128_bpreshuffle_wmma_t32x128x128_mw2_nw2_nb2_sk1_cm1_cn1,178.17,1536.97,0.0003 +gfx1250,256,2048,1536,128,128x128,flydsl,141,1,2.5967,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x128_mw2_nw2_nb2_sk1_cm1_cn1,310.13,2599.53,0.0003 +gfx1250,256,4096,1536,128,128x128,flydsl,215,1,3.3489,flydsl_mxfp8_128_bpreshuffle_wmma_t128x256x128_mw2_nw2_nb2_sk1_cm1_cn1,480.94,3972.59,0.0004 +gfx1250,256,8192,1536,128,128x128,flydsl,253,1,4.6248,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x128_mw4_nw1_nb2_sk1_cm1_cn1,696.51,5710.74,0.0004 +gfx1250,256,16384,1536,128,128x128,flydsl,260,1,7.4573,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb2_sk1_cm1_cn1,863.91,7056.9,0.0003 +gfx1250,256,32768,1536,128,128x128,flydsl,253,1,12.2369,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x128_mw4_nw1_nb2_sk1_cm1_cn1,1052.95,8585.03,0.0002 gfx1250,256,1,1536,7168,128x128,flydsl,566,4,5.4552,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,4.04,2020.14,0.0 gfx1250,256,2,1536,7168,128x128,flydsl,566,4,5.6413,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,7.81,1955.32,0.0 gfx1250,256,4,1536,7168,128x128,flydsl,566,4,5.7349,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,15.36,1926.97,0.0 @@ -6,21 +23,21 @@ gfx1250,256,8,1536,7168,128x128,flydsl,827,8,5.9103,flydsl_mxfp8_128_bpreshuffle gfx1250,256,16,1536,7168,128x128,flydsl,566,4,5.9174,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,59.54,1888.31,0.0 gfx1250,256,48,1536,7168,128x128,flydsl,584,4,6.6876,flydsl_mxfp8_128_bpreshuffle_wmma_t32x64x256_mw2_nw2_nb4_sk4_cm1_cn1,158.05,1719.83,0.0 gfx1250,256,64,1536,7168,128x128,flydsl,645,4,6.7749,flydsl_mxfp8_128_bpreshuffle_wmma_t64x32x256_mw4_nw1_nb4_sk4_cm1_cn1,208.02,1721.86,0.0 -gfx1250,256,1,2176,7168,128x128,flydsl,827,8,6.5427,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb4_sk8_cm1_cn1,4.77,2385.73,0.0 -gfx1250,256,2,2176,7168,128x128,flydsl,305,2,6.3004,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x512_mw2_nw2_nb4_sk2_cm1_cn1,9.9,2479.3,0.0 -gfx1250,256,4,2176,7168,128x128,flydsl,43,1,6.4436,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x1024_mw2_nw2_nb3_sk1_cm1_cn1,19.37,2427.78,0.0 -gfx1250,256,8,2176,7168,128x128,flydsl,43,1,6.4882,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x1024_mw2_nw2_nb3_sk1_cm1_cn1,38.46,2418.19,0.0 -gfx1250,256,16,2176,7168,128x128,flydsl,900,8,6.944,flydsl_mxfp8_128_bpreshuffle_wmma_t64x32x128_mw2_nw2_nb4_sk8_cm1_cn1,71.88,2272.74,0.0 -gfx1250,256,32,2176,7168,128x128,flydsl,827,8,6.9283,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x128_mw2_nw2_nb4_sk8_cm1_cn1,144.08,2304.49,0.0 -gfx1250,256,64,2176,7168,128x128,flydsl,932,8,7.8716,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x128_mw1_nw4_nb4_sk8_cm1_cn1,253.63,2075.16,0.0 -gfx1250,256,128,2176,7168,128x128,flydsl,674,4,9.1194,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk4_cm1_cn1,437.86,1872.07,0.0 -gfx1250,256,256,2176,7168,128x128,flydsl,737,4,11.4849,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb3_sk4_cm1_cn1,695.34,1614.88,0.0 -gfx1250,256,512,2176,7168,128x128,flydsl,735,4,15.6748,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x128_mw2_nw2_nb4_sk4_cm1_cn1,1018.95,1371.36,0.0 -gfx1250,256,1024,2176,7168,128x128,flydsl,196,1,20.5442,flydsl_mxfp8_128_bpreshuffle_wmma_t128x64x256_mw2_nw2_nb3_sk1_cm1_cn1,1554.88,1333.42,0.0 -gfx1250,256,2048,2176,7168,128x128,flydsl,207,1,31.1436,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x128_mw2_nw2_nb4_sk1_cm1_cn1,2051.39,1258.38,0.0 +gfx1250,256,1,2176,7168,128x128,flydsl,533,4,6.2688,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk4_cm1_cn1,4.98,2489.96,0.0087 +gfx1250,256,2,2176,7168,128x128,flydsl,566,4,6.0115,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,10.38,2598.45,0.0083 +gfx1250,256,4,2176,7168,128x128,flydsl,533,4,6.2532,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk4_cm1_cn1,19.95,2501.7,0.0072 +gfx1250,256,8,2176,7168,128x128,flydsl,566,4,6.3719,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,39.17,2462.33,0.0072 +gfx1250,256,16,2176,7168,128x128,flydsl,533,4,6.4037,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk4_cm1_cn1,77.94,2464.5,0.0072 +gfx1250,256,32,2176,7168,128x128,flydsl,566,4,6.4617,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,154.49,2470.9,0.0072 +gfx1250,256,64,2176,7168,128x128,flydsl,43,1,6.9528,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x1024_mw2_nw2_nb3_sk1_cm1_cn1,287.15,2349.39,0.007 +gfx1250,256,128,2176,7168,128x128,flydsl,674,4,8.0723,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk4_cm1_cn1,494.65,2114.9,0.0072 +gfx1250,256,256,2176,7168,128x128,flydsl,676,4,9.7816,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw2_nw2_nb3_sk4_cm1_cn1,816.43,1896.08,0.0071 +gfx1250,256,512,2176,7168,128x128,flydsl,151,1,12.2613,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,1302.63,1753.14,0.0076 +gfx1250,256,1024,2176,7168,128x128,flydsl,210,1,16.8981,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb4_sk1_cm1_cn1,1890.38,1621.13,0.0074 +gfx1250,256,2048,2176,7168,128x128,flydsl,257,1,25.3194,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,2523.27,1547.85,0.0072 gfx1250,256,4096,2176,7168,128x128,flydsl,206,1,46.2725,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x128_mw2_nw2_nb3_sk1_cm1_cn1,2761.37,1356.82,0.0 -gfx1250,256,8192,2176,7168,128x128,flydsl,259,1,84.9925,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw4_nw1_nb3_sk1_cm1_cn1,3006.74,1293.87,0.0 -gfx1250,256,16384,2176,7168,128x128,flydsl,259,1,144.0529,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw4_nw1_nb3_sk1_cm1_cn1,3548.01,1418.52,0.0 +gfx1250,256,8192,2176,7168,128x128,flydsl,257,1,75.6849,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,3376.51,1452.99,0.0074 +gfx1250,256,16384,2176,7168,128x128,flydsl,257,1,130.4719,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,3917.33,1566.17,0.0073 gfx1250,256,1,2304,1536,128x128,flydsl,40,1,3.716,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x512_mw2_nw2_nb3_sk1_cm1_cn1,1.9,954.01,0.0 gfx1250,256,2,2304,1536,128x128,flydsl,40,1,3.8369,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x512_mw2_nw2_nb3_sk1_cm1_cn1,3.69,925.55,0.0 gfx1250,256,4,2304,1536,128x128,flydsl,5,1,3.9159,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,7.23,910.01,0.0 @@ -213,3 +230,73 @@ gfx1250,256,2048,67584,7168,128x128,flydsl,1056,1,352.4866,flydsl_mxfp8_128_bpre gfx1250,256,4096,67584,7168,128x128,flydsl,1056,1,669.1289,flydsl_mxfp8_128_bpreshuffle_compute_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm2_cn2,5930.92,1595.28,0.0 gfx1250,256,8192,67584,7168,128x128,flydsl,1056,1,1293.7773,flydsl_mxfp8_128_bpreshuffle_compute_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm2_cn2,6134.83,1275.69,0.0 gfx1250,256,16384,67584,7168,128x128,flydsl,1056,1,2590.2535,flydsl_mxfp8_128_bpreshuffle_compute_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm2_cn2,6128.43,1087.34,0.0 +gfx1250,256,48,2176,7168,128x128,flydsl,43,1,6.8748,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x1024_mw2_nw2_nb3_sk1_cm1_cn1,217.81,2349.24,0.0069 +gfx1250,256,32768,2176,7168,128x128,flydsl,257,1,265.2178,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,3854.2,1482.12,0.0072 +gfx1250,256,1,3584,7168,128x128,flydsl,566,4,6.4608,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,7.95,3978.52,0.0078 +gfx1250,256,2,3584,7168,128x128,flydsl,533,4,6.3697,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk4_cm1_cn1,16.13,4037.68,0.0068 +gfx1250,256,4,3584,7168,128x128,flydsl,566,4,6.4155,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk4_cm1_cn1,32.04,4013.32,0.0072 +gfx1250,256,8,3584,7168,128x128,flydsl,533,4,6.4591,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk4_cm1_cn1,63.64,3995.11,0.0076 +gfx1250,256,16,3584,7168,128x128,flydsl,533,4,6.6281,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk4_cm1_cn1,124.03,3910.55,0.0073 +gfx1250,256,32,3584,7168,128x128,flydsl,861,8,7.0786,flydsl_mxfp8_128_bpreshuffle_wmma_t32x128x128_mw1_nw4_nb4_sk8_cm1_cn1,232.27,3694.07,0.0073 +gfx1250,256,48,3584,7168,128x128,flydsl,935,8,7.4591,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x128_mw2_nw2_nb4_sk8_cm1_cn1,330.64,3536.38,0.0073 +gfx1250,256,64,3584,7168,128x128,flydsl,41,1,7.7867,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x512_mw2_nw2_nb4_sk1_cm1_cn1,422.3,3417.06,0.0073 +gfx1250,256,128,3584,7168,128x128,flydsl,674,4,8.9149,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk4_cm1_cn1,737.72,3087.54,0.0073 +gfx1250,256,256,3584,7168,128x128,flydsl,135,1,11.6446,flydsl_mxfp8_128_bpreshuffle_wmma_t64x64x512_mw2_nw2_nb3_sk1_cm1_cn1,1129.57,2521.35,0.0075 +gfx1250,256,512,3584,7168,128x128,flydsl,151,1,14.0339,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,1874.51,2353.6,0.0073 +gfx1250,256,1024,3584,7168,128x128,flydsl,210,1,18.8013,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb4_sk1_cm1_cn1,2798.39,2147.2,0.0074 +gfx1250,256,2048,3584,7168,128x128,flydsl,259,1,29.1062,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw4_nw1_nb3_sk1_cm1_cn1,3615.27,1891.36,0.0073 +gfx1250,256,4096,3584,7168,128x128,flydsl,262,1,45.9867,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4576.4,1835.54,0.0074 +gfx1250,256,8192,3584,7168,128x128,flydsl,262,1,87.8232,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4792.66,1629.76,0.0074 +gfx1250,256,16384,3584,7168,128x128,flydsl,262,1,172.7455,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4873.14,1508.41,0.0074 +gfx1250,256,32768,3584,7168,128x128,flydsl,262,1,346.2345,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4862.68,1430.97,0.0073 +gfx1250,256,1,7168,1024,128x128,flydsl,38,1,3.4076,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk1_cm1_cn1,4.31,2158.53,0.0024 +gfx1250,256,2,7168,1024,128x128,flydsl,38,1,3.3804,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk1_cm1_cn1,8.69,2180.44,0.0028 +gfx1250,256,4,7168,1024,128x128,flydsl,38,1,3.4289,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk1_cm1_cn1,17.13,2158.56,0.003 +gfx1250,256,8,7168,1024,128x128,flydsl,16,1,3.4846,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x256_mw1_nw4_nb4_sk1_cm1_cn1,33.7,2141.68,0.0024 +gfx1250,256,16,7168,1024,128x128,flydsl,16,1,3.52,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x256_mw1_nw4_nb4_sk1_cm1_cn1,66.73,2155.05,0.0022 +gfx1250,256,32,7168,1024,128x128,flydsl,16,1,3.6301,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x256_mw1_nw4_nb4_sk1_cm1_cn1,129.41,2157.39,0.0022 +gfx1250,256,48,7168,1024,128x128,flydsl,16,1,3.727,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x256_mw1_nw4_nb4_sk1_cm1_cn1,189.06,2167.24,0.0023 +gfx1250,256,64,7168,1024,128x128,flydsl,75,1,3.8769,flydsl_mxfp8_128_bpreshuffle_wmma_t32x128x256_mw1_nw4_nb4_sk1_cm1_cn1,242.34,2146.84,0.0022 +gfx1250,256,128,7168,1024,128x128,flydsl,146,1,4.0946,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk1_cm1_cn1,458.91,2272.78,0.0023 +gfx1250,256,256,7168,1024,128x128,flydsl,145,1,4.7632,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb3_sk1_cm1_cn1,788.99,2366.52,0.0023 +gfx1250,256,512,7168,1024,128x128,flydsl,209,1,5.7199,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb3_sk1_cm1_cn1,1314.04,2658.15,0.0023 +gfx1250,256,1024,7168,1024,128x128,flydsl,217,1,7.505,flydsl_mxfp8_128_bpreshuffle_wmma_t128x256x128_mw2_nw2_nb4_sk1_cm1_cn1,2002.98,3073.77,0.0024 +gfx1250,256,2048,7168,1024,128x128,flydsl,262,1,10.5334,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,2854.23,3683.27,0.0024 +gfx1250,256,4096,7168,1024,128x128,flydsl,262,1,18.37,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3273.25,3824.42,0.0023 +gfx1250,256,8192,7168,1024,128x128,flydsl,262,1,36.4041,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3303.45,3658.08,0.0024 +gfx1250,256,16384,7168,1024,128x128,flydsl,261,1,65.253,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb3_sk1_cm1_cn1,3685.93,3969.14,0.0024 +gfx1250,256,32768,7168,1024,128x128,flydsl,262,1,126.4004,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3805.66,4039.99,0.0023 +gfx1250,256,1,7168,1792,128x128,flydsl,38,1,4.2102,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk1_cm1_cn1,6.1,3054.77,0.0049 +gfx1250,256,2,7168,1792,128x128,flydsl,38,1,4.1709,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk1_cm1_cn1,12.32,3087.42,0.005 +gfx1250,256,4,7168,1792,128x128,flydsl,38,1,4.19,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x256_mw2_nw2_nb4_sk1_cm1_cn1,24.53,3081.04,0.0046 +gfx1250,256,8,7168,1792,128x128,flydsl,5,1,4.2433,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,48.43,3057.54,0.0043 +gfx1250,256,16,7168,1792,128x128,flydsl,5,1,4.3205,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,95.14,3032.77,0.0042 +gfx1250,256,32,7168,1792,128x128,flydsl,56,1,4.5345,flydsl_mxfp8_128_bpreshuffle_wmma_t32x64x256_mw2_nw2_nb4_sk1_cm1_cn1,181.3,2946.55,0.004 +gfx1250,256,48,7168,1792,128x128,flydsl,16,1,4.6926,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x256_mw1_nw4_nb4_sk1_cm1_cn1,262.78,2902.27,0.0039 +gfx1250,256,64,7168,1792,128x128,flydsl,78,1,4.9089,flydsl_mxfp8_128_bpreshuffle_wmma_t32x128x256_mw2_nw2_nb4_sk1_cm1_cn1,334.94,2826.96,0.0039 +gfx1250,256,128,7168,1792,128x128,flydsl,146,1,5.2945,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk1_cm1_cn1,621.08,2816.02,0.0039 +gfx1250,256,256,7168,1792,128x128,flydsl,149,1,6.1661,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw2_nw2_nb4_sk1_cm1_cn1,1066.58,2752.76,0.0038 +gfx1250,256,512,7168,1792,128x128,flydsl,209,1,7.4047,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb3_sk1_cm1_cn1,1776.35,2849.89,0.0037 +gfx1250,256,1024,7168,1792,128x128,flydsl,257,1,9.8742,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,2664.18,2973.42,0.0037 +gfx1250,256,2048,7168,1792,128x128,flydsl,262,1,13.7067,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3838.51,3346.92,0.0037 +gfx1250,256,4096,7168,1792,128x128,flydsl,262,1,27.0261,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3893.52,2919.6,0.0037 +gfx1250,256,8192,7168,1792,128x128,flydsl,262,1,52.2042,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4031.35,2776.9,0.0037 +gfx1250,256,16384,7168,1792,128x128,flydsl,262,1,93.3233,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4510.2,2969.1,0.0036 +gfx1250,256,32768,7168,1792,128x128,flydsl,262,1,186.5073,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4513.57,2902.45,0.0037 +gfx1250,256,1,12288,1536,128x128,flydsl,5,1,3.9865,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,9.47,4741.12,0.0037 +gfx1250,256,2,12288,1536,128x128,flydsl,5,1,3.9952,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,18.9,4737.33,0.0041 +gfx1250,256,4,12288,1536,128x128,flydsl,40,1,3.9751,flydsl_mxfp8_128_bpreshuffle_wmma_t32x32x512_mw2_nw2_nb3_sk1_cm1_cn1,37.99,4774.42,0.0038 +gfx1250,256,8,12288,1536,128x128,flydsl,5,1,4.0811,flydsl_mxfp8_128_bpreshuffle_wmma_t16x64x256_mw1_nw4_nb4_sk1_cm1_cn1,74.0,4676.01,0.0034 +gfx1250,256,16,12288,1536,128x128,flydsl,16,1,4.1544,flydsl_mxfp8_128_bpreshuffle_wmma_t16x128x256_mw1_nw4_nb4_sk1_cm1_cn1,145.38,4643.79,0.0035 +gfx1250,256,32,12288,1536,128x128,flydsl,56,1,4.4827,flydsl_mxfp8_128_bpreshuffle_wmma_t32x64x256_mw2_nw2_nb4_sk1_cm1_cn1,269.47,4396.89,0.0034 +gfx1250,256,48,12288,1536,128x128,flydsl,151,1,4.7566,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,380.93,4231.54,0.0033 +gfx1250,256,64,12288,1536,128x128,flydsl,151,1,4.8351,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x512_mw1_nw4_nb3_sk1_cm1_cn1,499.66,4249.25,0.0033 +gfx1250,256,128,12288,1536,128x128,flydsl,146,1,5.4992,flydsl_mxfp8_128_bpreshuffle_wmma_t64x128x256_mw1_nw4_nb4_sk1_cm1_cn1,878.64,4039.99,0.0033 +gfx1250,256,256,12288,1536,128x128,flydsl,209,1,6.5077,flydsl_mxfp8_128_bpreshuffle_wmma_t128x128x256_mw2_nw2_nb3_sk1_cm1_cn1,1484.96,3927.51,0.0033 +gfx1250,256,512,12288,1536,128x128,flydsl,257,1,8.5829,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,2251.84,3756.74,0.0034 +gfx1250,256,1024,12288,1536,128x128,flydsl,262,1,12.4287,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,3110.12,3669.98,0.0034 +gfx1250,256,2048,12288,1536,128x128,flydsl,257,1,24.2859,flydsl_mxfp8_128_bpreshuffle_wmma_t256x128x256_mw2_nw2_nb3_sk1_cm1_cn1,3183.3,2979.17,0.0033 +gfx1250,256,4096,12288,1536,128x128,flydsl,263,1,40.6012,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x256_mw2_nw2_nb2_sk1_cm1_cn1,3808.23,3099.15,0.0033 +gfx1250,256,8192,12288,1536,128x128,flydsl,262,1,74.4852,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4151.67,3125.24,0.0033 +gfx1250,256,16384,12288,1536,128x128,flydsl,262,1,141.7125,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4364.3,3152.11,0.0034 +gfx1250,256,32768,12288,1536,128x128,flydsl,262,1,287.2498,flydsl_mxfp8_128_bpreshuffle_wmma_t256x256x128_mw2_nw2_nb4_sk1_cm1_cn1,4306.18,3044.43,0.0034 diff --git a/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_untuned_gemm.csv b/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_untuned_gemm.csv index b5b97dfba9f..5e5fb9ee8bf 100644 --- a/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_untuned_gemm.csv +++ b/aiter/configs/model_configs/kimik3_a8w8_blockscale_mxscale_bpreshuffle_untuned_gemm.csv @@ -1,4 +1,21 @@ M,N,K,w_scale_block,kernelId +1,1536,128,128x128, +2,1536,128,128x128, +4,1536,128,128x128, +8,1536,128,128x128, +16,1536,128,128x128, +32,1536,128,128x128, +48,1536,128,128x128, +64,1536,128,128x128, +128,1536,128,128x128, +256,1536,128,128x128, +512,1536,128,128x128, +1024,1536,128,128x128, +2048,1536,128,128x128, +4096,1536,128,128x128, +8192,1536,128,128x128, +16384,1536,128,128x128, +32768,1536,128,128x128, 1,1536,7168,128x128, 2,1536,7168,128x128, 4,1536,7168,128x128, @@ -213,3 +230,73 @@ M,N,K,w_scale_block,kernelId 4096,67584,7168,128x128, 8192,67584,7168,128x128, 16384,67584,7168,128x128, +48,2176,7168,128x128, +32768,2176,7168,128x128, +1,12288,1536,128x128, +2,12288,1536,128x128, +4,12288,1536,128x128, +8,12288,1536,128x128, +16,12288,1536,128x128, +32,12288,1536,128x128, +48,12288,1536,128x128, +64,12288,1536,128x128, +128,12288,1536,128x128, +256,12288,1536,128x128, +512,12288,1536,128x128, +1024,12288,1536,128x128, +2048,12288,1536,128x128, +4096,12288,1536,128x128, +8192,12288,1536,128x128, +16384,12288,1536,128x128, +32768,12288,1536,128x128, +1,7168,1024,128x128, +2,7168,1024,128x128, +4,7168,1024,128x128, +8,7168,1024,128x128, +16,7168,1024,128x128, +32,7168,1024,128x128, +48,7168,1024,128x128, +64,7168,1024,128x128, +128,7168,1024,128x128, +256,7168,1024,128x128, +512,7168,1024,128x128, +1024,7168,1024,128x128, +2048,7168,1024,128x128, +4096,7168,1024,128x128, +8192,7168,1024,128x128, +16384,7168,1024,128x128, +32768,7168,1024,128x128, +1,3584,7168,128x128, +2,3584,7168,128x128, +4,3584,7168,128x128, +8,3584,7168,128x128, +16,3584,7168,128x128, +32,3584,7168,128x128, +48,3584,7168,128x128, +64,3584,7168,128x128, +128,3584,7168,128x128, +256,3584,7168,128x128, +512,3584,7168,128x128, +1024,3584,7168,128x128, +2048,3584,7168,128x128, +4096,3584,7168,128x128, +8192,3584,7168,128x128, +16384,3584,7168,128x128, +32768,3584,7168,128x128, +1,7168,1792,128x128, +2,7168,1792,128x128, +4,7168,1792,128x128, +8,7168,1792,128x128, +16,7168,1792,128x128, +32,7168,1792,128x128, +48,7168,1792,128x128, +64,7168,1792,128x128, +128,7168,1792,128x128, +256,7168,1792,128x128, +512,7168,1792,128x128, +1024,7168,1792,128x128, +2048,7168,1792,128x128, +4096,7168,1792,128x128, +8192,7168,1792,128x128, +16384,7168,1792,128x128, +32768,7168,1792,128x128, diff --git a/aiter/configs/model_configs/kimik3_bf16_tuned_gemm.csv b/aiter/configs/model_configs/kimik3_bf16_tuned_gemm.csv index a307e8a19bc..f8bb831a598 100644 --- a/aiter/configs/model_configs/kimik3_bf16_tuned_gemm.csv +++ b/aiter/configs/model_configs/kimik3_bf16_tuned_gemm.csv @@ -766,3 +766,20 @@ gfx950,256,176,768,7168,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,5 gfx950,256,192,768,7168,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,5961,7,9.6994,flydsl_hgemm_abf16_wbf16_bf16_t64x64x64x6_ks7_w4x2x1_bias0_ktail0_gm0_pft_gfx950,0.0202,217.94,1449.31 gfx950,256,208,768,7168,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,5851,8,10.5867,flydsl_hgemm_abf16_wbf16_bf16_t64x96x64x4_ks8_w2x2x1_bias0_ktail0_gm0_pft_gfx950,0.0214,216.32,1351.83 gfx950,256,224,768,7168,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,5759,8,10.6513,flydsl_hgemm_abf16_wbf16_bf16_t64x96x128x3_ks8_w2x2x2_bias0_ktail0_gm0_pft_gfx950,0.0282,231.54,1367.47 +gfx1250,256,1,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,38,1,30.7158,flydsl_hgemm_abf16_wbf16_bf16_t16x16x256x6_ks1_w1x1x1_bias0_ktail0_gm0_pft_gfx1250,0.0,16.73,16730.42 +gfx1250,256,2,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,39,1,30.5718,flydsl_hgemm_abf16_wbf16_bf16_t16x16x256x6_ks1_w1x1x1_bias0_ktail0_gm0_pht_gfx1250,0.0,33.61,16812.04 +gfx1250,256,4,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,39,1,31.0787,flydsl_hgemm_abf16_wbf16_bf16_t16x16x256x6_ks1_w1x1x1_bias0_ktail0_gm0_pht_gfx1250,0.0,66.13,16543.37 +gfx1250,256,8,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,45,1,31.7011,flydsl_hgemm_abf16_wbf16_bf16_t16x16x256x6_ks1_w1x1x1_bias0_ktail0_gm0_pht_gfx1250,0.0,129.66,16229.42 +gfx1250,256,16,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,73,2,32.2031,flydsl_hgemm_abf16_wbf16_bf16_t16x64x256x5_ks2_w1x2x1_bias0_ktail0_gm0_pht_gfx1250,0.0,255.28,15997.79 +gfx1250,256,32,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,38.4912,auto,0.0,427.15,13420.07 +gfx1250,256,48,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,73,2,44.0462,flydsl_hgemm_abf16_wbf16_bf16_t64x64x128x6_ks2_w4x2x1_bias0_ktail0_gm0_pht_gfx1250,0.0,559.92,11758.81 +gfx1250,256,64,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,67,2,46.2621,flydsl_hgemm_abf16_wbf16_bf16_t64x64x128x6_ks2_w4x2x1_bias0_ktail0_gm0_pht_gfx1250,0.0,710.81,11225.33 +gfx1250,256,128,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,triton,0,0,72.8665,auto,0.0,902.56,7202.38 +gfx1250,256,256,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,flydsl,61,1,114.7048,flydsl_hgemm_abf16_wbf16_bf16_t64x128x128x6_ks1_w2x2x1_bias0_ktail0_gm0_pht_gfx1250,0.0,1146.71,4671.32 +gfx1250,256,512,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,opus,591,0,159.7101,opus_gemm_gfx1250_4wave_wl_co_128x128x192x128_4x1_16x16x32_0x0x0_w2x2_c4x1_p3_v1024w1,0.0,1647.15,3492.84 +gfx1250,256,1024,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,opus,596,0,214.3804,opus_gemm_gfx1250_4wave_wl_co_128x128x256x128_4x1_16x16x32_0x0x0_w2x2_c4x1_p3_v1024w1,0.0,2454.21,2807.55 +gfx1250,256,2048,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,opus,716,0,428.2359,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w2x2_c4x4_p4_v1024w1,0.0,2457.21,1611.18 +gfx1250,256,4096,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,opus,716,0,944.4064,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w2x2_c4x4_p4_v1024w1,0.0,2228.42,917.11 +gfx1250,256,8192,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,opus,716,0,1909.6372,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w2x2_c4x4_p4_v1024w1,0.0,2204.12,638.05 +gfx1250,256,16384,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,opus,716,0,3759.4041,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w2x2_c4x4_p4_v1024w1,0.0,2239.22,511.54 +gfx1250,256,32768,7168,35840,False,torch.bfloat16,torch.bfloat16,False,False,opus,716,0,7395.8911,opus_gemm_gfx1250_4wave_wlr_co_128x256x256x64_4x1_16x16x32_0x0x0_w2x2_c4x4_p4_v1024w1,0.0,2276.44,450.57 diff --git a/aiter/ops/flydsl/bpreshuffle_gemm_gfx1250.py b/aiter/ops/flydsl/bpreshuffle_gemm_gfx1250.py index 5b613402a73..4a81946a49e 100644 --- a/aiter/ops/flydsl/bpreshuffle_gemm_gfx1250.py +++ b/aiter/ops/flydsl/bpreshuffle_gemm_gfx1250.py @@ -130,9 +130,12 @@ def run_preshuffle_gemm_a8_gfx1250( f"{split_k}*{tile_k}={split_k * tile_k}" ) num_k_tiles = (K // split_k) // tile_k - if num_k_tiles < nb: + # K=128 with two buffers uses only the prologue and drain. Keep all + # other configurations at their existing minimum. + min_k_tiles = 1 if nb == 2 and K == tile_k == 128 else nb + if num_k_tiles < min_k_tiles: raise RuntimeError( - f"[FlyDSL gfx1250] {nb}-buffer pipeline needs >= {nb} K-tiles per " + f"[FlyDSL gfx1250] {nb}-buffer pipeline needs >= {min_k_tiles} K-tiles per " f"split-k chunk, got {num_k_tiles} (K={K}, split_k={split_k}, tile_k={tile_k})" ) if split_k > 1: diff --git a/aiter/ops/flydsl/gemm_tune/flydsl_gemm_a8w8_bpreshuffle_wmma_common.py b/aiter/ops/flydsl/gemm_tune/flydsl_gemm_a8w8_bpreshuffle_wmma_common.py index c3d7277ac36..4a1aa8c8a8a 100644 --- a/aiter/ops/flydsl/gemm_tune/flydsl_gemm_a8w8_bpreshuffle_wmma_common.py +++ b/aiter/ops/flydsl/gemm_tune/flydsl_gemm_a8w8_bpreshuffle_wmma_common.py @@ -135,7 +135,9 @@ def kernel_fits_shape(ki: WmmaKernelInstance, M: int, N: int, K: int) -> bool: return False if N % ki.tile_n != 0 or K % (ki.split_k * ki.tile_k) != 0: return False - if (K // ki.split_k) // ki.tile_k < ki.num_buffers: + # K=128 with two buffers needs no steady-loop iteration. + min_k_tiles = 1 if ki.num_buffers == 2 and K == ki.tile_k == 128 else ki.num_buffers + if (K // ki.split_k) // ki.tile_k < min_k_tiles: return False # M may be ragged; the kernel handles partial M tiles via OOB clamping, # so tile_m > M is legal (and needed for small-M decode shapes). diff --git a/aiter/ops/flydsl/gemm_tune/flydsl_gemm_mxfp8_128_bpreshuffle_wmma_common.py b/aiter/ops/flydsl/gemm_tune/flydsl_gemm_mxfp8_128_bpreshuffle_wmma_common.py index 184f5cb49c8..e5226c1c1c5 100644 --- a/aiter/ops/flydsl/gemm_tune/flydsl_gemm_mxfp8_128_bpreshuffle_wmma_common.py +++ b/aiter/ops/flydsl/gemm_tune/flydsl_gemm_mxfp8_128_bpreshuffle_wmma_common.py @@ -263,7 +263,9 @@ def kernel_fits_shape(ki: WmmaKernelInstance, M: int, N: int, K: int) -> bool: return False if N % ki.tile_n != 0 or K % (ki.split_k * ki.tile_k) != 0: return False - if (K // ki.split_k) // ki.tile_k < ki.num_buffers: + # Generic K=128 with two buffers needs no steady-loop iteration. + min_k_tiles = 1 if ki.num_buffers == 2 and K == ki.tile_k == 128 else ki.num_buffers + if (K // ki.split_k) // ki.tile_k < min_k_tiles: return False # M may be ragged; the kernel handles partial M tiles via OOB clamping, # so tile_m > M is legal (and needed for small-M decode shapes). diff --git a/aiter/ops/flydsl/mxfp8_bpreshuffle_gemm_gfx1250.py b/aiter/ops/flydsl/mxfp8_bpreshuffle_gemm_gfx1250.py index 5041c1b08b2..355b0aee734 100644 --- a/aiter/ops/flydsl/mxfp8_bpreshuffle_gemm_gfx1250.py +++ b/aiter/ops/flydsl/mxfp8_bpreshuffle_gemm_gfx1250.py @@ -247,9 +247,12 @@ def _run_mxfp8_bpreshuffle_gemm_a8_gfx1250( f"split_k*tile_k={split_k}*{tile_k}={split_k * tile_k}" ) num_k_tiles = (K // split_k) // tile_k - if num_k_tiles < nb: + # Generic K=128 with two buffers uses only the prologue and drain. + # Other configurations, including compute-bound kernels, keep their limits. + min_k_tiles = 1 if not compute_bound and nb == 2 and K == tile_k == 128 else nb + if num_k_tiles < min_k_tiles: raise RuntimeError( - f"[FlyDSL gfx1250 mxfp8] {nb}-buffer pipeline needs >= {nb} " + f"[FlyDSL gfx1250 mxfp8] {nb}-buffer pipeline needs >= {min_k_tiles} " f"K-tiles per split-k chunk, got {num_k_tiles}" ) if compute_bound: diff --git a/csrc/gemm_a8w8_mxfp8_bpreshuffle/gemm_a8w8_mxfp8_bpreshuffle_tune.py b/csrc/gemm_a8w8_mxfp8_bpreshuffle/gemm_a8w8_mxfp8_bpreshuffle_tune.py index 2319be7b7ff..ea57d91b3f8 100644 --- a/csrc/gemm_a8w8_mxfp8_bpreshuffle/gemm_a8w8_mxfp8_bpreshuffle_tune.py +++ b/csrc/gemm_a8w8_mxfp8_bpreshuffle/gemm_a8w8_mxfp8_bpreshuffle_tune.py @@ -24,6 +24,10 @@ --mxscale_table \\ -i \\ -o aiter/configs/model_configs/dsv4_a8w8_blockscale_mxscale_bpreshuffle_tuned_gemm.csv + +On gfx1250 --mxscale_table times the mxfp8_128 kernels the 2-D dispatch runs +(flydsl_gemm_mxfp8_128_bpreshuffle_wmma_common, row-major A) on 128x128 +shapes, and a row's kernelId is the kernel's id in that list. """ from typing import Any, ClassVar @@ -162,6 +166,29 @@ def run_gemm_flydsl(x, weight_shuffle, x_scale, w_scale, out, kernel_name): ) +def gfx1250_candidates(m, n, k): + """(kernelId, kernel) of the gfx1250 mxfp8_128 kernels the 2-D dispatch can + run for the shape (row-major A).""" + from aiter.ops.flydsl.gemm_tune.flydsl_gemm_mxfp8_128_bpreshuffle_wmma_common import ( + kernel_fits_shape, + kernels_list, + ) + + return [ + (kernel_id, ki) + for kernel_id, ki in kernels_list.items() + if not ki.a_preshuffle and kernel_fits_shape(ki, m, n, k) + ] + + +def run_gemm_flydsl_gfx1250(x, weight_shuffle, x_scale, w_scale, out, kernel_name): + from aiter.ops.gemm_op_a8w8 import gemm_a8w8_mxfp8_128_bpreshuffle_flydsl + + return gemm_a8w8_mxfp8_128_bpreshuffle_flydsl( + x, weight_shuffle, x_scale, w_scale, out, {"kernelName": kernel_name} + ) + + class GemmA8W8MXFP8BpreshuffleTuner(GemmCommonTuner): ARG_DEFAULTS: ClassVar[dict[str, Any]] = { **GemmCommonTuner.ARG_DEFAULTS, @@ -287,9 +314,9 @@ def run_config(self, args): def tune(self, untunedf, tunedf, args): gfx = self.get_gfx() - if gfx != "gfx950": + if gfx not in ("gfx950", "gfx1250"): logger.warning( - f"gemm_a8w8_mxfp8_bpreshuffle tuner covers gfx950 only, got {gfx}" + f"gemm_a8w8_mxfp8_bpreshuffle tuner covers gfx950 and gfx1250, got {gfx}" ) return [] cu_num = self.get_cu_num() @@ -309,6 +336,13 @@ def tune(self, untunedf, tunedf, args): f"w_scale block {coarse} is tuned for gemm_a8w8_blockscale_" f"bpreshuffle only: pass --mxscale_table" ) + if gfx == "gfx1250" and ( + not self._mxscale_table or set(untunedf["w_scale_block"]) != {"128x128"} + ): + raise ValueError( + "gfx1250 tunes the mxfp8_128 kernels of gemm_a8w8_blockscale_" + "bpreshuffle's table only: pass --mxscale_table and 128x128 shapes" + ) seeds = self._bmm_seeds() if self._mxscale_table else {} for i in range(len(untunedf)): M = int(untunedf.loc[i, "M"]) @@ -317,17 +351,19 @@ def tune(self, untunedf, tunedf, args): block = str(untunedf.loc[i, "w_scale_block"]) info_keys = (gfx, cu_num, M, N, K, block) # One pass per pipeline of the block, in their order. - cands = [ - (kernel_id, ki) - for pipe in PIPELINES_BY_W_SCALE_BLOCK[block] - for kernel_id, ki in pipe.kernels_list.items() - if pipe.fits(ki, M, N, K) - ] - runs = [ - (kernel_id, ki.split_k, ki.name, run_gemm_flydsl) - for kernel_id, ki in cands - ] - if self._mxscale_table: + cands = ( + gfx1250_candidates(M, N, K) + if gfx == "gfx1250" + else [ + (kernel_id, ki) + for pipe in PIPELINES_BY_W_SCALE_BLOCK[block] + for kernel_id, ki in pipe.kernels_list.items() + if pipe.fits(ki, M, N, K) + ] + ) + run_fn = run_gemm_flydsl_gfx1250 if gfx == "gfx1250" else run_gemm_flydsl + runs = [(kernel_id, ki.split_k, ki.name, run_fn) for kernel_id, ki in cands] + if self._mxscale_table and gfx == "gfx950": near = [ name for mm, name in seeds.get((N, K, block), ()) @@ -390,7 +426,8 @@ def result_to_df(self, results): keys, kernelId, splitK, kernelName, libtype = info if time in (self.INVALID_TIME, self.INF_TIME): kernelName = "None" - elif getattr(self, "_mxscale_table", False): + elif getattr(self, "_mxscale_table", False) and keys[0] == "gfx950": + # gfx1250 rows keep their kernels_list id, as the table's do. kernelId = ( MXSCALE_BMM_KERNEL_ID if kernelId >= KERNEL_ID_BASE_BMM