diff --git a/aiter/configs/model_configs/dsv41_tuned_grouped_fmoe.csv b/aiter/configs/model_configs/dsv41_tuned_grouped_fmoe.csv index 247716ac85d..2b2839ba8b7 100644 --- a/aiter/configs/model_configs/dsv41_tuned_grouped_fmoe.csv +++ b/aiter/configs/model_configs/dsv41_tuned_grouped_fmoe.csv @@ -70,3 +70,55 @@ gfx1250,256,4096,5120,2304,48,-1,ActivationType.Silu,torch.bfloat16,torch.float8 gfx1250,256,8192,5120,2304,48,-1,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,156.2070,0,0 gfx1250,256,16384,5120,2304,48,-1,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,2,4,1,4,3,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,238.3880,0,0 gfx1250,256,32768,5120,2304,48,-1,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,1,4,3,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,398.8590,0,0 +gfx1250,256,1,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,128,1,1,1,2,2,2,2,3,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,8.8037,0,0 +gfx1250,256,8,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,512,16,256,256,1,4,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,61.5840,0,0 +gfx1250,256,16,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,128,128,1,2,1,2,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,108.0769,0,0 +gfx1250,256,64,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,256,512,1,2,1,2,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,289.1468,0,0 +gfx1250,256,128,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,512,16,128,512,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,399.1987,0,0 +gfx1250,256,256,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,512,512,1,2,1,4,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,449.1041,0,0 +gfx1250,256,512,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,512,1,4,1,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,458.3507,0,0 +gfx1250,256,1024,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,384,128,32,256,256,1,2,1,4,4,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,471.8984,0,0 +gfx1250,256,2048,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,512,256,64,512,128,1,4,1,4,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,503.7990,0,0 +gfx1250,256,4096,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,512,128,1,4,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,602.8378,0,0 +gfx1250,256,8192,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,1,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,771.7022,0,0 +gfx1250,256,16384,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,256,256,256,256,2,2,2,2,4,4,,,0,1,4,1,4,1,1,1,0,1,0,0,0,1,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1190.0373,0,0 +gfx1250,256,32768,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,256,256,256,256,2,2,2,2,4,4,,,0,1,4,1,4,1,1,1,0,1,0,0,0,1,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1967.6713,0,0 +gfx1250,256,1,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,256,1,1,1,2,2,2,2,3,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,8.6879,0,0 +gfx1250,256,8,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,128,256,1,4,1,2,3,3,,,0,-1,-1,,,-1,,1,1,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,55.7633,0,0 +gfx1250,256,16,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,256,256,1,2,1,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,109.9071,0,0 +gfx1250,256,32,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,1,2,3,,,0,-1,-1,,,1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,189.5117,0,0 +gfx1250,256,64,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,256,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,291.7681,0,0 +gfx1250,256,128,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,512,16,128,256,1,4,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,393.1344,0,0 +gfx1250,256,256,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,512,512,1,2,1,8,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,457.1645,0,0 +gfx1250,256,512,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,512,1,2,1,2,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,468.4928,0,0 +gfx1250,256,1024,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,512,256,32,512,128,1,4,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,491.8166,0,0 +gfx1250,256,2048,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,512,128,64,512,128,1,4,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,538.5656,0,0 +gfx1250,256,4096,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,128,2,2,2,2,2,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,684.2151,0,0 +gfx1250,256,8192,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,512,256,1,4,1,4,3,3,,,0,-1,-1,,,-1,,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1089.3,0,0 +gfx1250,256,16384,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,512,256,2,4,1,4,3,3,,,0,1,4,,,2,,1,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1857.7,0,0 +gfx1250,256,32768,5120,2304,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,128,256,256,128,2,2,2,2,4,3,,,0,2,2,2,2,2,2,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,2704.0969,0,0 +gfx1250,256,1,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,256,16,128,128,1,1,1,2,2,2,4,,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,7.1019,0,0 +gfx1250,256,8,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,64,128,1,2,1,1,4,3,,,0,-1,-1,,,-1,1,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,11.7210,0,0 +gfx1250,256,16,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,64,128,1,2,1,1,3,2,,,0,-1,-1,,,-1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,27.7585,0,0 +gfx1250,256,32,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,64,128,1,1,1,1,3,2,,,0,-1,-1,,,1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,48.7471,0,0 +gfx1250,256,64,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,79.7599,0,0 +gfx1250,256,128,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,128,128,1,2,1,2,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,105.8920,0,0 +gfx1250,256,256,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,512,128,1,2,1,8,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,121.7150,0,0 +gfx1250,256,512,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,512,128,1,2,1,8,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,125.2232,0,0 +gfx1250,256,1024,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,128,256,32,128,128,1,2,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,134.7501,0,0 +gfx1250,256,8192,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,128,256,128,128,128,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,288.1954,0,0 +gfx1250,256,16384,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,384,256,128,256,128,2,2,2,2,4,4,,,0,-1,-1,-1,-1,-1,-1,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,428.5814,0,0 +gfx1250,256,32768,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,384,256,128,256,128,2,2,2,2,4,4,,,0,-1,-1,-1,-1,-1,-1,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,729.0402,0,0 +gfx1250,256,1,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,256,16,128,128,1,1,1,2,2,2,4,,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,7.0833,0,0 +gfx1250,256,8,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,512,128,1,1,1,4,3,4,,,0,-1,-1,,4,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,11.5417,0,0 +gfx1250,256,16,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,64,128,1,2,1,1,3,3,,,0,-1,-1,,,-1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,28.6250,0,0 +gfx1250,256,32,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,128,1,1,1,1,2,2,,,0,-1,-1,,,1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,51.4501,0,0 +gfx1250,256,64,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,128,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,78.3361,0,0 +gfx1250,256,128,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,104.9947,0,0 +gfx1250,256,256,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,256,128,1,2,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,124.6401,0,0 +gfx1250,256,512,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,512,128,1,2,1,8,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,124.1254,0,0 +gfx1250,256,1024,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,128,256,32,256,128,1,2,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,141.8739,0,0 +gfx1250,256,4096,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,384,256,64,256,128,1,2,1,4,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,215.4001,0,0 +gfx1250,256,8192,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,128,256,128,256,128,1,2,2,2,2,3,,,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,337.4960,0,0 +gfx1250,256,16384,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,384,128,256,256,128,2,2,2,2,3,3,,,0,-1,-1,1,4,-1,1,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,563.3912,0,0 +gfx1250,256,32768,5120,576,384,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,384,128,128,256,128,1,2,2,2,3,4,,,0,-1,-1,-1,-1,-1,-1,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,920.4351,0,0 diff --git a/aiter/configs/model_configs/kimik3_gfx1250_tuned_grouped_fmoe.csv b/aiter/configs/model_configs/kimik3_gfx1250_tuned_grouped_fmoe.csv index 67e700615f2..c0027ec905c 100644 --- a/aiter/configs/model_configs/kimik3_gfx1250_tuned_grouped_fmoe.csv +++ b/aiter/configs/model_configs/kimik3_gfx1250_tuned_grouped_fmoe.csv @@ -26,3 +26,52 @@ gfx1250,256,8192,3584,3072,112,-1,ActivationType.Situv2,torch.bfloat16,torch.flo gfx1250,256,16384,3584,3072,112,-1,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,512,256,1,4,1,8,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,672.2920006723,0,0 gfx1250,256,32768,3584,3072,112,-1,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,512,256,1,4,1,4,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1074.4549999799,0,0 gfx1250,256,131072,3584,3072,112,-1,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,512,256,1,4,2,4,2,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,3699.0629996581,0,0 +gfx1250,256,1,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,256,512,1,1,1,4,2,2,,,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,13.9162,0,0 +gfx1250,256,8,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,512,256,1,2,1,4,3,3,,,0,-1,-1,,,-1,,0,1,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,139.8859,0,0 +gfx1250,256,16,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,256,16,128,128,1,2,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,263.5649,0,0 +gfx1250,256,32,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,512,16,64,256,1,2,1,1,4,3,,,0,-1,-1,,,-1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,443.9185,0,0 +gfx1250,256,64,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,512,128,1,4,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,650.3285,0,0 +gfx1250,256,128,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,256,16,512,512,1,2,1,4,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,876.4503,0,0 +gfx1250,256,256,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,256,16,512,256,1,2,1,4,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,935.8242,0,0 +gfx1250,256,512,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,512,16,256,512,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,992.9061,0,0 +gfx1250,256,1024,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,384,512,32,512,256,1,2,1,4,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1039.8420,0,0 +gfx1250,256,2048,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,512,256,64,512,128,1,4,1,4,2,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1077.0983,0,0 +gfx1250,256,4096,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1278.0329,0,0 +gfx1250,256,8192,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,512,256,256,512,2,2,2,2,2,2,,,0,-1,2,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1974.7926,0,0 +gfx1250,256,16384,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,2,4,1,4,3,3,,,0,-1,-1,4,1,-1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,3618.9073,0,0 +gfx1250,256,1,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,256,256,1,1,1,2,2,2,,,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,14.4445,0,0 +gfx1250,256,8,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,512,128,1,2,1,8,2,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,149.8475,0,0 +gfx1250,256,16,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,512,128,1,2,1,8,2,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,264.7482,0,0 +gfx1250,256,32,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,64,256,1,1,1,1,2,3,,,0,-1,-1,,,1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,439.0845,0,0 +gfx1250,256,64,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,512,128,1,4,1,4,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,662.2305,0,0 +gfx1250,256,128,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,512,16,512,256,1,2,1,4,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,887.8147,0,0 +gfx1250,256,256,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,256,512,1,2,1,2,4,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,981.3504,0,0 +gfx1250,256,512,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,512,16,256,512,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1018.5639,0,0 +gfx1250,256,1024,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,384,256,32,512,256,1,2,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1076.7810,0,0 +gfx1250,256,2048,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,512,128,2,2,1,4,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1188.1515,0,0 +gfx1250,256,4096,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,128,1,4,2,2,3,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1506.5369,0,0 +gfx1250,256,8192,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,512,128,64,512,128,1,4,1,4,3,3,,,0,-1,4,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,2474.7820,0,0 +gfx1250,256,16384,3584,3072,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,2,2,2,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,4091.7701,0,0 +gfx1250,256,1,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,256,1,1,1,2,2,2,,3,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,8.1344,0,0 +gfx1250,256,8,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,128,128,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,34.3186,0,0 +gfx1250,256,16,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,64,256,1,2,1,1,4,2,,,0,-1,-1,,,-1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,67.8593,0,0 +gfx1250,256,32,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,256,16,64,256,1,1,1,1,2,2,,,0,-1,-1,,,1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,116.2319,0,0 +gfx1250,256,64,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,128,128,1,8,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,180.0550,0,0 +gfx1250,256,128,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,3,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,237.5469,0,0 +gfx1250,256,256,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,256,1,2,1,4,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,253.9716,0,0 +gfx1250,256,512,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,256,128,1,2,1,2,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,270.4521,0,0 +gfx1250,256,1024,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,384,256,32,256,128,1,2,1,4,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,273.6726,0,0 +gfx1250,256,4096,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,512,128,1,4,1,4,3,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,398.8703,0,0 +gfx1250,256,8192,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,512,256,256,256,2,4,2,2,2,4,,,0,-1,-1,,4,-1,1,0,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,611.4260,0,0 +gfx1250,256,16384,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,4,,,0,1,4,1,4,1,1,1,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,962.8303,0,0 +gfx1250,256,1,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,256,1,1,1,2,2,2,,,0,-1,-1,,,1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,8.0496,0,0 +gfx1250,256,8,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,256,256,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,34.7242,0,0 +gfx1250,256,16,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,64,256,1,2,1,1,4,2,,,0,-1,-1,,,-1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,73.1383,0,0 +gfx1250,256,32,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,64,256,1,2,1,1,2,2,,,0,-1,-1,,,-1,1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,118.7855,0,0 +gfx1250,256,64,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,128,16,256,256,1,2,1,2,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,178.8851,0,0 +gfx1250,256,128,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,256,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,238.5706,0,0 +gfx1250,256,256,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,256,16,256,128,1,2,1,4,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,261.0032,0,0 +gfx1250,256,512,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,512,16,256,128,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,280.5143,0,0 +gfx1250,256,1024,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,384,256,32,512,256,1,2,1,4,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,290.7473,0,0 +gfx1250,256,4096,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,512,256,1,4,1,4,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,454.8532,0,0 +gfx1250,256,16384,3584,768,896,16,ActivationType.Situv2,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1178.8413,0,0 diff --git a/aiter/configs/model_configs/kimik3tp8_tuned_grouped_fmoe.csv b/aiter/configs/model_configs/kimik3tp8_tuned_grouped_fmoe.csv index f17c83970ed..328c82d7f20 100644 --- a/aiter/configs/model_configs/kimik3tp8_tuned_grouped_fmoe.csv +++ b/aiter/configs/model_configs/kimik3tp8_tuned_grouped_fmoe.csv @@ -1,16 +1,16 @@ -gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,ep_fused,q_type,use_g1u1,doweight_stage1,tile_m,tile_n,tile_k,tile_m2,tile_n2,tile_k2,m_warp,n_warp,m_warp2,n_warp2,num_buffers,num_buffer_stage2,split_k1,split_k2,tdm_as_in_prologue,cluster_m,cluster_n,cluster_m2,cluster_n2,waves_per_tensor_tdm,waves_per_tensor_tdm2,next_stage_prefetch,tdm_b_th,lds_soa_load_interleave,rolled_drain,persistent_workers,persistent_workers2,kernelName1,kernelName2,us,tflops,bw -gfx1250,256,1,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,512,128,1,2,1,4,3,2,1,1,0,-1,-1,-1,-1,,,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,20.2994,0,0 -gfx1250,256,2,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,512,128,1,2,1,4,4,2,1,1,0,-1,-1,-1,-1,,,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,21.2357,0,0 -gfx1250,256,4,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,256,128,1,2,1,4,3,2,1,1,0,-1,-1,-1,-1,,,0,1,0,1,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,25.5425,0,0 -gfx1250,256,8,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,256,128,1,2,1,2,4,2,1,1,0,-1,-1,-1,-1,,,0,1,0,1,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,32.1113,0,0 -gfx1250,256,16,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,128,128,1,2,1,2,4,2,1,1,0,-1,-1,-1,-1,,,0,1,0,1,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,51.0913,0,0 -gfx1250,256,32,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,3,2,1,1,0,-1,-1,-1,-1,,,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,71.84,0,0 -gfx1250,256,64,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,3,2,1,1,0,-1,-1,-1,-1,,,0,1,0,1,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,102.6685,0,0 -gfx1250,256,128,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,2,2,1,1,0,-1,-1,-1,-1,,,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,133.0114,0,0 -gfx1250,256,256,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,2,2,1,1,0,-1,-1,-1,-1,,,0,1,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,146.1798,0,0 -gfx1250,256,512,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,128,128,1,2,1,2,2,2,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,157.1095,0,0 -gfx1250,256,1024,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,128,256,32,256,128,2,2,1,2,2,2,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,174.5391,0,0 -gfx1250,256,2048,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,128,128,2,4,1,2,3,3,1,1,0,-1,-1,-1,-1,,,0,1,0,1,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,221.3278,0,0 -gfx1250,256,4096,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,256,128,2,4,1,4,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,354.3946,0,0 -gfx1250,256,8192,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,256,128,2,4,1,4,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,576.6258,0,0 -gfx1250,256,16384,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,128,512,1,4,1,2,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,963.5116,0,0 +gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,ep_fused,q_type,use_g1u1,doweight_stage1,tile_m,tile_n,tile_k,tile_m2,tile_n2,tile_k2,m_warp,n_warp,m_warp2,n_warp2,num_buffers,num_buffer_stage2,split_k1,split_k2,tdm_as_in_prologue,cluster_m,cluster_n,cluster_m2,cluster_n2,waves_per_tensor_tdm,waves_per_tensor_tdm2,next_stage_prefetch,tdm_b_th,lds_soa_load_interleave,rolled_drain,persistent_workers,persistent_workers2,a_preshuffle,kernelName1,kernelName2,us,tflops,bw +gfx1250,256,1,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,128,16,64,128,1,1,1,1,4,3,7,1,0,-1,-1,,,1,1,0,3,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,6.3065,0,0 +gfx1250,256,2,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,512,128,1,2,1,4,4,2,1,1,0,-1,-1,-1,-1,,,0,1,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,21.2357,0,0 +gfx1250,256,4,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,256,128,1,2,1,4,3,2,1,1,0,-1,-1,-1,-1,,,0,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,25.5425,0,0 +gfx1250,256,8,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,64,128,1,1,1,1,2,2,1,1,0,-1,-1,,,1,1,0,2,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,13.1693,0,0 +gfx1250,256,16,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,128,128,1,2,1,1,4,2,2,1,0,-1,-1,,,2,1,0,4,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,28.6514,0,0 +gfx1250,256,32,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,64,128,1,1,1,1,2,3,1,1,0,-1,-1,,,1,1,0,4,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,51.4909,0,0 +gfx1250,256,64,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,64,128,1,2,1,1,2,3,1,1,0,-1,-1,,,-1,1,0,1,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,77.3317,0,0 +gfx1250,256,128,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,128,128,1,2,1,1,3,2,2,1,0,-1,-1,,,2,1,0,1,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,106.815,0,0 +gfx1250,256,256,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,128,16,128,128,1,2,1,1,2,2,1,1,0,-1,-1,,,-1,1,0,4,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,122.2313,0,0 +gfx1250,256,512,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,128,16,128,128,1,1,1,1,2,2,1,1,0,-1,-1,,,1,1,0,6,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,126.2082,0,0 +gfx1250,256,1024,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,128,256,32,256,128,2,2,1,2,2,2,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,174.5391,0,0 +gfx1250,256,2048,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,128,128,2,4,1,2,3,3,1,1,0,-1,-1,-1,-1,,,0,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,221.3278,0,0 +gfx1250,256,4096,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,256,128,2,4,1,4,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,354.3946,0,0 +gfx1250,256,8192,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,256,128,2,4,1,4,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,576.6258,0,0 +gfx1250,256,16384,3584,384,896,16,ActivationType.Situv2,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,128,512,1,4,1,2,3,3,1,1,0,-1,-1,-1,-1,,,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,963.5116,0,0 diff --git a/aiter/configs/model_configs/minimax_m3_gfx1250_tuned_grouped_fmoe.csv b/aiter/configs/model_configs/minimax_m3_gfx1250_tuned_grouped_fmoe.csv index 6f4fe452802..8226917a9b8 100644 --- a/aiter/configs/model_configs/minimax_m3_gfx1250_tuned_grouped_fmoe.csv +++ b/aiter/configs/model_configs/minimax_m3_gfx1250_tuned_grouped_fmoe.csv @@ -1,17 +1,58 @@ -gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,ep_fused,q_type,use_g1u1,doweight_stage1,tile_m,tile_n,tile_k,tile_m2,tile_n2,tile_k2,m_warp,n_warp,m_warp2,n_warp2,num_buffers,num_buffer_stage2,split_k1,split_k2,tdm_as_in_prologue,cluster_m,cluster_n,cluster_m2,cluster_n2,waves_per_tensor_tdm,waves_per_tensor_tdm2,next_stage_prefetch,tdm_b_th,lds_soa_load_interleave,rolled_drain,persistent_workers,persistent_workers2,kernelName1,kernelName2,us -gfx1250,256,1,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,2,3,3,1,1,0,-1,-1,-1,-1,1,-1,0,0,0,0,1024,2048,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,21.85 -gfx1250,256,2,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,2,3,3,1,1,0,-1,-1,-1,-1,1,-1,0,0,0,0,1024,2048,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,22.07 -gfx1250,256,4,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,2,3,3,1,1,0,-1,-1,-1,-1,1,-1,0,0,0,0,1024,2048,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,25.28 -gfx1250,256,8,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,2,3,3,1,1,0,-1,-1,-1,-1,1,-1,0,0,0,0,1024,4096,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,26.75 -gfx1250,256,16,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,2,3,3,1,1,0,-1,-1,-1,-1,1,-1,0,0,0,0,2048,4096,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,28.85 -gfx1250,256,32,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,1024,16,64,128,1,2,1,1,2,3,1,1,0,-1,-1,-1,-1,-1,1,0,0,0,0,1024,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,53.38 -gfx1250,256,64,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,128,1,1,1,1,2,3,1,1,0,-1,-1,-1,-1,1,1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,66.56 -gfx1250,256,128,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,256,256,1,2,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,1024,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,80.25 -gfx1250,256,256,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,128,256,32,256,256,1,2,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,86.28 -gfx1250,256,512,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,128,256,32,128,256,1,2,1,2,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,93.25 -gfx1250,256,1024,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,162.6272 -gfx1250,256,2048,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,3,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,124.27 -gfx1250,256,4096,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,3,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,188.93 -gfx1250,256,8192,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,3,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,321.82 -gfx1250,256,16384,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,3,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,584.85 -gfx1250,256,32768,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,3,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1097.21 +gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,ep_fused,q_type,use_g1u1,doweight_stage1,tile_m,tile_n,tile_k,tile_m2,tile_n2,tile_k2,m_warp,n_warp,m_warp2,n_warp2,num_buffers,num_buffer_stage2,split_k1,split_k2,tdm_as_in_prologue,cluster_m,cluster_n,cluster_m2,cluster_n2,waves_per_tensor_tdm,waves_per_tensor_tdm2,next_stage_prefetch,tdm_b_th,lds_soa_load_interleave,rolled_drain,persistent_workers,persistent_workers2,kernelName1,kernelName2,us,a_preshuffle,tflops,bw +gfx1250,256,2,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,2,3,3,1,1,0,-1,-1,-1,-1,1,-1,0,0,0,0,1024,2048,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,22.07,,, +gfx1250,256,4,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,64,256,1,1,1,2,3,3,1,1,0,-1,-1,-1,-1,1,-1,0,0,0,0,1024,2048,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,25.28,,, +gfx1250,256,1,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,512,1,1,1,2,2,2,2,2,0,-1,-1,,,1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,9.3754,0,0,0 +gfx1250,256,8,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,512,128,1,4,1,8,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,77.9298,0,0,0 +gfx1250,256,16,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,512,256,1,2,1,4,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,119.4024,0,0,0 +gfx1250,256,32,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,128,512,1,1,1,2,3,2,,,0,-1,-1,,,1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,186.9773,0,0,0 +gfx1250,256,64,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,384,128,1,4,1,2,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,229.5677,0,0,0 +gfx1250,256,128,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,512,16,128,512,1,2,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,241.4983,0,0,0 +gfx1250,256,256,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,512,16,384,128,1,4,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,258.5324,0,0,0 +gfx1250,256,512,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,384,256,32,256,512,1,2,1,4,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,269.6804,0,0,0 +gfx1250,256,1024,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,128,128,64,128,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,286.9140,0,0,0 +gfx1250,256,2048,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,128,128,256,256,2,4,2,4,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,335.9373,0,0,0 +gfx1250,256,4096,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,256,256,256,512,2,2,2,2,4,2,,,0,1,4,,,1,,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,467.0129,1,0,0 +gfx1250,256,8192,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,3,,,0,-1,4,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,761.5430,0,0,0 +gfx1250,256,16384,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,256,256,256,256,2,2,2,2,3,3,,,0,1,4,2,2,1,1,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1181.0994,1,0,0 +gfx1250,256,32768,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,256,256,256,256,2,2,2,2,3,3,,,0,4,1,4,4,1,1,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,2166.4935,1,0,0 +gfx1250,256,1,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,512,1,1,1,2,2,2,2,2,0,-1,-1,,,1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,9.5492,0,0,0 +gfx1250,256,8,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,512,16,128,256,1,4,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,82.4577,0,0,0 +gfx1250,256,16,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,512,512,1,4,1,4,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,116.1466,0,0,0 +gfx1250,256,32,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,128,16,256,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,178.2946,0,0,0 +gfx1250,256,64,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,384,128,1,4,1,2,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,231.1327,0,0,0 +gfx1250,256,128,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,384,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,248.0446,0,0,0 +gfx1250,256,256,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,384,128,1,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,256.3651,0,0,0 +gfx1250,256,512,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,512,128,32,384,256,1,4,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,279.9470,0,0,0 +gfx1250,256,2048,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,378.8039,0,0,0 +gfx1250,256,4096,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,652.9629,0,0,0 +gfx1250,256,8192,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,987.9044,0,0,0 +gfx1250,256,16384,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,128,256,256,256,2,2,2,2,4,2,,,0,2,4,,,2,,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,1769.0063,0,0,0 +gfx1250,256,32768,6144,3072,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,128,256,256,128,2,2,2,2,4,4,,,0,2,4,4,4,2,2,1,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,2881.3809,0,0,0 +gfx1250,256,1,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,256,1,1,1,2,2,2,4,3,0,-1,-1,,,1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,6.9235,0,0,0 +gfx1250,256,8,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,512,256,1,2,1,8,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,14.3371,0,0,0 +gfx1250,256,16,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,384,128,16,128,128,1,2,1,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,31.2030,0,0,0 +gfx1250,256,32,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,1024,16,256,128,1,1,1,2,2,3,,,0,-1,-1,,,1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,45.4792,0,0,0 +gfx1250,256,64,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,128,16,512,128,1,8,1,8,4,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,61.6371,0,0,0 +gfx1250,256,128,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,512,16,256,256,1,2,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,1024,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,80.25,,, +gfx1250,256,256,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,128,128,1,4,1,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,69.6598,0,0,0 +gfx1250,256,512,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,256,512,32,256,128,1,4,1,4,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,73.7413,0,0,0 +gfx1250,256,1024,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,256,128,64,128,128,2,2,1,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,83.2841,0,0,0 +gfx1250,256,2048,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,128,128,256,128,2,4,1,4,4,4,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,106.4725,0,0,0 +gfx1250,256,4096,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,3,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,188.93,,, +gfx1250,256,8192,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,1,4,1,4,3,3,,,0,4,4,4,1,1,1,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,241.4516,1,0,0 +gfx1250,256,16384,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,256,256,256,256,2,2,2,2,3,4,,,0,2,2,4,4,1,1,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,406.0694,1,0,0 +gfx1250,256,32768,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,256,256,256,256,256,256,2,2,2,2,3,3,,,0,2,2,4,4,1,1,1,0,1,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,707.0587,0,0,0 +gfx1250,256,1,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,64,512,16,128,256,1,1,1,2,2,2,4,,0,-1,-1,,,1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,7.0223,0,0,0 +gfx1250,256,8,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,512,16,512,256,1,4,1,4,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,14.3670,0,0,0 +gfx1250,256,16,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,256,128,1,2,1,2,4,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,29.3450,0,0,0 +gfx1250,256,32,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,128,256,16,256,128,1,2,1,2,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,47.5227,0,0,0 +gfx1250,256,64,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,128,16,512,128,1,4,1,8,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,62.2897,0,0,0 +gfx1250,256,128,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,128,128,1,8,1,2,4,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,67.3709,0,0,0 +gfx1250,256,256,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,256,16,128,256,1,4,1,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,70.5307,0,0,0 +gfx1250,256,512,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,32,512,128,32,384,128,1,4,1,2,2,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,76.9218,0,0,0 +gfx1250,256,1024,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,64,512,128,64,256,128,1,4,2,2,3,3,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,88.0554,0,0,0 +gfx1250,256,2048,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,119.5323,0,0,0 +gfx1250,256,4096,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,256,256,128,256,256,2,2,2,2,2,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,199.5641,0,0,0 +gfx1250,256,8192,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,512,256,1,4,2,4,3,4,,,0,-1,-1,,,-1,,1,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,301.4199,0,0,0 +gfx1250,256,16384,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,530.9981,0,0,0 +gfx1250,256,32768,6144,768,129,5,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,128,512,256,128,256,256,1,4,2,2,3,2,,,0,-1,-1,,,-1,,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,934.3685,0,0,0 diff --git a/aiter/configs/tuned_grouped_fmoe.csv b/aiter/configs/tuned_grouped_fmoe.csv index 563b5c2e2c3..2463aafea3c 100644 --- a/aiter/configs/tuned_grouped_fmoe.csv +++ b/aiter/configs/tuned_grouped_fmoe.csv @@ -3,7 +3,7 @@ gfx1250,256,64,4096,2048,256,6,ActivationType.Swiglu,torch.bfloat16,torch.float4 gfx1250,256,64,4096,2048,256,6,ActivationType.Swiglu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,128,16,512,128,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,468.0528,0,0 gfx1250,256,64,4096,2048,256,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,128,16,512,128,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,468.0528,0,0 gfx1250,256,64,4096,2048,256,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,512,128,16,512,128,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,468.0528,0,0 -gfx1250,256,1,4096,2048,256,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,256,1,4,1,4,2,2,4,2,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,339.3273,0,0 +gfx1250,256,1,4096,2048,256,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,256,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,339.3273,0,0 gfx1250,256,2,4096,2048,256,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,256,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,347.7829,0,0 gfx1250,256,4,4096,2048,256,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,256,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,365.3146,0,0 gfx1250,256,8,4096,2048,256,6,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,,QuantType.per_1x32,1,0,16,256,256,16,256,256,1,4,1,4,2,2,1,1,0,-1,-1,-1,-1,-1,-1,0,0,0,0,0,0,0,grouped_gemm1_fp4_gugu,grouped_gemm2_fp4,348.3547,0,0 diff --git a/aiter/ops/flydsl/grouped_gemm_mxfp4.py b/aiter/ops/flydsl/grouped_gemm_mxfp4.py index 6c1724ed89c..91766ea5a33 100644 --- a/aiter/ops/flydsl/grouped_gemm_mxfp4.py +++ b/aiter/ops/flydsl/grouped_gemm_mxfp4.py @@ -147,6 +147,7 @@ def flydsl_grouped_gemm_a8w4_masked( persistent_workers=0, rolled_drain=0, m_align=0, + k_waves=1, ): """Launch a contiguous-M grouped GEMM, optionally with persistent workers. @@ -185,7 +186,20 @@ def flydsl_grouped_gemm_a8w4_masked( if float(situ_linear_beta) <= 0.0: raise ValueError(f"situ_linear_beta must be > 0, got {situ_linear_beta!r}") k_tiles = (K + tile_k - 1) // tile_k - num_buffers = min(num_buffers, max(1, k_tiles)) + k_waves = int(k_waves) + if k_waves < 1: + raise ValueError(f"k_waves (CSV split_k1/split_k2) must be >= 1, got {k_waves}") + if k_waves > 1: + # Each wave group runs the full pipeline on its own K slice, so the + # slices must be whole and equal; an EP scatter epilogue cannot be split. + if K % tile_k or k_tiles % k_waves: + raise ValueError( + f"k_waves={k_waves} must evenly split the K tiles " + f"(K={K}, tile_k={tile_k}, {k_tiles} tiles)" + ) + if stage2_scatter is not None: + raise ValueError(f"k_waves={k_waves} does not support the EP scatter") + num_buffers = min(num_buffers, max(1, k_tiles // k_waves)) if bias is None: has_bias = 0 elif bias.dtype in _BIAS_MODE: @@ -221,6 +235,16 @@ def flydsl_grouped_gemm_a8w4_masked( persistent_workers = ( (persistent_workers + cluster_n - 1) // cluster_n * cluster_n ) + # k_waves owns the grid: no clusters, no persistent workers, no As prologue. + # Reject a config that asks for them rather than dropping them. + if k_waves > 1 and ( + cluster_n > 1 or cluster_m > 1 or persistent_workers or tdm_as_in_prologue + ): + raise ValueError( + f"k_waves={k_waves} needs a plain grid, got cluster_n={cluster_n} " + f"cluster_m={cluster_m} persistent_workers={persistent_workers} " + f"tdm_as_in_prologue={tdm_as_in_prologue}" + ) waves_per_tensor_tdm = _select_num_waves_per_tensor_tdm(waves_per_tensor_tdm) rolled_drain = int(rolled_drain) if rolled_drain not in (0, 1): @@ -283,6 +307,7 @@ def flydsl_grouped_gemm_a8w4_masked( rolled_drain, int(a_preshuffled), int(m_align), + k_waves, ) _run_compiled( launch_gemm_a8w4_tdm, diff --git a/aiter/ops/flydsl/grouped_moe_gfx1250.py b/aiter/ops/flydsl/grouped_moe_gfx1250.py index 71c7eb24aab..01c6ccd3988 100644 --- a/aiter/ops/flydsl/grouped_moe_gfx1250.py +++ b/aiter/ops/flydsl/grouped_moe_gfx1250.py @@ -609,6 +609,23 @@ def get_wmma_m_rep( return tile_m // m_warp // wmma_m +@functools.cache +def _log_tile_fit_once(msg: str) -> None: + logger.info("[grouped-moe] default tile narrowed to fit the shape: %s", msg) + + +def _fit_tile_n(n: int, tile_n: int, n_warp: int) -> tuple[int, int]: + """Halve ``tile_n`` (and ``n_warp`` while its wave tile stays >= 64) until + it divides ``n``.""" + while n % tile_n and tile_n > 64: + tile_n //= 2 + if tile_n // n_warp < 64 and n_warp > 1: + n_warp //= 2 + if n % tile_n: + raise ValueError(f"[grouped-moe] no N tile divides N={n}") + return tile_n, n_warp + + def _grouped_a8w4_tdm_moe( hidden_states, w1, @@ -653,6 +670,9 @@ def _grouped_a8w4_tdm_moe( a_preshuffle=0, persistent_workers=0, persistent_workers2=0, + k_waves1=1, + k_waves2=1, + tiles_configured=False, data_format="a8w4", expert_mask=None, num_local_tokens=None, @@ -730,6 +750,31 @@ def _grouped_a8w4_tdm_moe( f"[grouped-moe compact] tiles {tile_m}/{tile_m2} do not divide " f"the compact plan's row alignment {_plan_align}" ) + # The B loads do not clamp N, so a partial last N tile would read the next + # expert's rows (and fault past the last one), and a half-k128 K tail + # (K % 128 == 64) only runs with a k128 tile. A tile the caller configured + # (CSV row or AITER_TDM_* override) must already fit; only the built-in + # default, e.g. 256 wide at inter_dim=576 (N=1152), is narrowed, and the + # tile that actually launches is logged. + fit_n, fit_nw = _fit_tile_n(2 * inter_dim, tile_n, n_warp) + fit_n2, fit_nw2 = _fit_tile_n(model_dim, tile_n2, n_warp2) + fit_k = 128 if model_dim % 128 else tile_k + fit_k2 = 128 if inter_dim % 128 else tile_k2 + asked = (tile_n, n_warp, tile_k, tile_n2, n_warp2, tile_k2) + fitted = (fit_n, fit_nw, fit_k, fit_n2, fit_nw2, fit_k2) + if fitted != asked: + _fit_msg = ( + f"gemm1 (N={2 * inter_dim}, K={model_dim}) tile_n/n_warp/tile_k " + f"{tile_n}/{n_warp}/{tile_k} -> {fit_n}/{fit_nw}/{fit_k}; " + f"gemm2 (N={model_dim}, K={inter_dim}) tile_n2/n_warp2/tile_k2 " + f"{tile_n2}/{n_warp2}/{tile_k2} -> {fit_n2}/{fit_nw2}/{fit_k2}" + ) + if tiles_configured: + raise ValueError( + f"[grouped-moe] configured tile does not fit the shape: {_fit_msg}" + ) + _log_tile_fit_once(_fit_msg) + tile_n, n_warp, tile_k, tile_n2, n_warp2, tile_k2 = fitted wmma_rep = get_wmma_m_rep(tile_m, tile_n, m_warp, n_warp, "gemm1") wmma_rep2 = get_wmma_m_rep(tile_m2, tile_n2, m_warp2, n_warp2, "gemm2") _align_m = max(tile_m, tile_m2) @@ -742,8 +787,18 @@ def _grouped_a8w4_tdm_moe( f"divide their max ({_align_m}); otherwise a GEMM tile straddles " "two experts and silently computes against the wrong weights" ) + # Empty experts take no rows, so at most min(E, routes) experts pad their + # tail: a decode batch launches a few M tiles instead of one per expert. + _routes = token_num * topk contiguous_m = max( - _align_m, _tdm_align_up(token_num * topk + E * _align_m - topk, _align_m) + _align_m, + _tdm_align_up( + min( + _routes + E * _align_m - topk, + _routes + min(E, _routes) * (_align_m - 1), + ), + _align_m, + ), ) max_m = max(_align_m, _tdm_align_up(token_num * topk, _align_m)) if _compact: @@ -1157,7 +1212,10 @@ def _grouped_a8w4_tdm_moe( # Pre-allocate MX payload + preshuffled e8m0 scale for gemm1 output. # These are written directly by the kernel's fused quant epilogue. payload_bytes = inter_dim // 2 if _is_fp4 else inter_dim - scale_bytes = inter_dim // 32 # one e8m0 byte per 32-element MX block + # One e8m0 byte per 32-element MX block, laid out per whole k128 (four + # blocks), so a half-k128 tail (inter_dim % 128 == 64) keeps a padded + # dword that gemm1 fills with a finite scale. + scale_bytes = (inter_dim + 127) // 128 * 4 a2_payload = torch.empty( (1, contiguous_m, payload_bytes), dtype=torch.uint8, device=device ) @@ -1208,6 +1266,7 @@ def _grouped_a8w4_tdm_moe( tdm_b_th=tdm_b_th, lds_soa_load_interleave=lds_soa_load_interleave, rolled_drain=rolled_drain, + k_waves=k_waves1, row_major_ascale=int(_row_major_ascale), a_row_stride_bytes=_a1_wire_stride, a_scale_row_stride_bytes=_a1_wire_stride, @@ -1248,6 +1307,7 @@ def _grouped_a8w4_tdm_moe( tdm_b_th=tdm_b_th, lds_soa_load_interleave=lds_soa_load_interleave, rolled_drain=rolled_drain, + k_waves=k_waves1, row_major_ascale=int(_row_major_ascale), a_row_stride_bytes=_a1_wire_stride, a_scale_row_stride_bytes=_a1_wire_stride, @@ -1298,6 +1358,7 @@ def _grouped_a8w4_tdm_moe( tdm_b_th=tdm_b_th, lds_soa_load_interleave=lds_soa_load_interleave, rolled_drain=rolled_drain, + k_waves=k_waves2, **_ep_gemm2_kwargs, ) @@ -1385,6 +1446,7 @@ def _grouped_a8w4_tdm_moe( lds_soa_load_interleave=lds_soa_load_interleave, row_major_ascale=int(_row_major_ascale), rolled_drain=rolled_drain, + k_waves=k_waves1, **_situ_kw, ), ) @@ -1427,6 +1489,7 @@ def _grouped_a8w4_tdm_moe( lds_soa_load_interleave=lds_soa_load_interleave, row_major_ascale=int(_row_major_ascale), rolled_drain=rolled_drain, + k_waves=k_waves1, **_situ_kw, ), ) @@ -1466,6 +1529,7 @@ def _grouped_a8w4_tdm_moe( tdm_b_th=tdm_b_th, lds_soa_load_interleave=lds_soa_load_interleave, rolled_drain=rolled_drain, + k_waves=k_waves2, ), ) ) @@ -1774,6 +1838,12 @@ def _fmt(v): cfg_row.get("lds_soa_load_interleave"), 0 ) _tdm_kw["rolled_drain"] = _as_int(cfg_row.get("rolled_drain"), 0) + # split_k1/split_k2 split each GEMM's K across that many wave groups + # inside one workgroup (k_waves); the grid stays the same. Blank or 0 + # (the model_configs/ merge fills a column an older table lacks with + # 0) means unset, i.e. no split. + _tdm_kw["k_waves1"] = max(1, _as_int(cfg_row.get("split_k1"), 1)) + _tdm_kw["k_waves2"] = max(1, _as_int(cfg_row.get("split_k2"), 1)) # Follow the matched CSV bucket, including padded token counts # (for example, 6144 -> 8192), just like the other kernel knobs. _tdm_kw["a_preshuffle"] = _as_int(cfg_row.get("a_preshuffle"), 0) @@ -1844,6 +1914,12 @@ def _tdm_env(name): _tdm_kw["n_warp"] = _base_nw _tdm_kw["m_warp2"] = _ov_mw2 if _ov_mw2 is not None else _base_mw _tdm_kw["n_warp2"] = _ov_nw2 if _ov_nw2 is not None else _base_nw + # A tile from the CSV row or an AITER_TDM_* override must fit as given; + # only the built-in default may be narrowed (see _grouped_a8w4_tdm_moe). + _tdm_kw["tiles_configured"] = any( + k in _tdm_kw + for k in ("tile_n", "tile_k", "n_warp", "tile_n2", "tile_k2", "n_warp2") + ) return _grouped_a8w4_tdm_moe( hidden_states, w1, diff --git a/aiter/ops/flydsl/kernels/mxfp4_preshuffle_gfx1250_tdm.py b/aiter/ops/flydsl/kernels/mxfp4_preshuffle_gfx1250_tdm.py index aae08a205c8..065a4d698cd 100644 --- a/aiter/ops/flydsl/kernels/mxfp4_preshuffle_gfx1250_tdm.py +++ b/aiter/ops/flydsl/kernels/mxfp4_preshuffle_gfx1250_tdm.py @@ -3,7 +3,6 @@ """Grouped contiguous-M A8W4 preshuffle MoE GEMM for gfx1250 (TDM pipeline).""" -import math import os from collections import namedtuple @@ -182,6 +181,7 @@ def build_gemm_a8w4_tdm_task( raw_drain=0, defer_c_wait=0, tdm_snop=0, + k_waves=1, ): """Build the grouped contiguous-M a8w4 TDM GEMM tile body (``gemm_task``). @@ -271,14 +271,41 @@ def build_gemm_a8w4_tdm_task( WAVE = 32 PACK_TK = tile_k // 2 KWS = tile_k // WMMA_K - if K % WMMA_K or tile_k % WMMA_K: - raise ValueError(f"K={K} and tile_k={tile_k} must be multiples of {WMMA_K}") + # K may end on half a k128 (K % 128 == 64, e.g. 576 = 4 x 128 + 64): the + # weights stay unpadded and only the scales are laid out per whole k128. + K_HALF_STEP = WMMA_K // 2 + if K % K_HALF_STEP or tile_k % WMMA_K: + raise ValueError( + f"K={K} must be a multiple of {K_HALF_STEP} and tile_k={tile_k} " + f"a multiple of {WMMA_K}" + ) + K128 = ceildiv(K, WMMA_K) + if K % WMMA_K and (a_preshuffled or row_major_ascale): + raise ValueError(f"K={K} with a half k128 tail needs the M16K4 A layout") + if K % WMMA_K and tile_k != WMMA_K: + # Only the single-k128 tile is validated on the half k128 tail; a wider + # tile has faulted there, so reject it rather than risk a GPU hang. + raise ValueError(f"K={K} with a half k128 tail needs tile_k={WMMA_K}") # A K that tile_k does not divide ends on a ragged tile of KWS_TAIL k128 # steps. Its TDM loads clamp the inner extent there, so the bytes past the - # row end are zero-filled instead of read, and its compute stops early. + # row end are zero-filled instead of read, and its compute stops early. A + # half k128 tail computes its last k128 on that zero fill. K_TILES = ceildiv(K, tile_k) - KWS_TAIL = (K - (K_TILES - 1) * tile_k) // WMMA_K - HAS_K_TAIL = KWS_TAIL != KWS + K_TAIL = K - (K_TILES - 1) * tile_k + KWS_TAIL = ceildiv(K_TAIL, WMMA_K) + HAS_K_TAIL = K_TAIL != tile_k + # Intra-workgroup split-K: k_waves wave groups of m_warp x n_warp waves each + # run the whole pipeline on their own contiguous K slice and LDS arena, then + # reduce their accumulators through LDS. It adds waves, not workgroups, so + # a decode tile's serial K loop shortens without a global round trip. + if k_waves > 1: + if K_TILES % k_waves or K % tile_k: + raise ValueError(f"k_waves={k_waves} must divide K tiles of K={K}/{tile_k}") + if persistent_workers or cluster_m > 1 or cluster_n > 1: + raise ValueError("k_waves needs a plain (non-persistent, unclustered) grid") + if enable_ep_scatter or tdm_as_in_prologue: + raise ValueError("k_waves does not support EP scatter or the As prologue") + K_TILES = K_TILES // k_waves # Double buffering is sufficient: the carry reads the other LDS buffer # before the post-compute barrier permits reusing the current buffer. next_stage_on = 1 if (next_stage_prefetch and num_buffers >= 2) else 0 @@ -335,6 +362,7 @@ def build_gemm_a8w4_tdm_task( ep_quant_bits, persistent_workers, rolled_drain, + k_waves, m_align, ) _ = cache_tag @@ -368,7 +396,7 @@ def build_gemm_a8w4_tdm_task( n_acc = wmma_m_rep * wmma_n_rep output_n_rep = warp_tile_n // 16 num_waves = m_warp * n_warp - block = num_waves * WAVE + block = num_waves * k_waves * WAVE SCATTER_GROUP_ROWS = 8 SCATTER_PASSES_PER_VECTOR = WAVE // SCATTER_GROUP_ROWS scatter_groups = (tile_m + SCATTER_GROUP_ROWS - 1) // SCATTER_GROUP_ROWS @@ -396,8 +424,8 @@ def build_gemm_a8w4_tdm_task( AS_KSTEPS = tile_k // 128 AS_INNER = AS_KSTEPS * wmma_m_rep * 16 AS_SUPERS = m_warp - AS_FULL_KDW = K // 128 - AS_FULL_INNER = (K // 128) * wmma_m_rep * 16 + AS_FULL_KDW = K128 + AS_FULL_INNER = K128 * wmma_m_rep * 16 # A-scale LDS: one outer row is one wave's M tile, and its inner # (k128, wm, lane16) layout gives each WMMA scale operand a contiguous # 16-dword block -- which is why the global buffer has to arrive already @@ -462,6 +490,17 @@ def _lds_pitch(nbytes): AS_FULL_OFF = INPUT_LDS_B AS_FULL_B = ceildiv(AS_SUPERS * AS_FULL_INNER * 4, 128) * 128 ARENA_B = max(AS_FULL_OFF + (AS_FULL_B if tdm_as_in_prologue else 0), C_STORE_B) + # Each k_waves group gets its own arena, which also holds its partial + # accumulators (one f32 dword4 per lane per chunk) for the final reduction. + K_PARTIAL_B = ( + ((tile_m // m_warp) // 16) + * ((tile_n // n_warp) // (32 if a_is_fp4 else 16)) + * ((32 if a_is_fp4 else 16) // 2) + * 4 + * num_waves + * WAVE + ) + GROUP_ARENA_B = ceildiv(max(ARENA_B, K_PARTIAL_B), 512) * 512 # Stages the mainloop issues before its first fence (mirrors run_mainloop). _mid_compute = not (tile_m <= 32 or num_buffers <= 2) _pro_stages = ( @@ -517,6 +556,7 @@ def _lds_pitch(nbytes): _epq = f"_epq{ep_quant_bits}" if ep_quant_bits else "" _persistent = f"_pw{persistent_workers}" if persistent_workers else "" _drain = "_rolleddrain" if rolled_drain else "" + _kwaves = f"_kw{k_waves}" if k_waves > 1 else "" _malign = f"_ma{m_align}" if m_align else "" _kname = ( f"mxfp4_tdm_{_afp}{_a_layout}" @@ -527,7 +567,7 @@ def _lds_pitch(nbytes): f"{_b_tdm_th}{_waves_per_tensor}{_ep}{_epq}" f"{_mma_group}{_ds_first}" f"{_explicit_vgpr_partition}{_planar_lds}" - f"{_interleaved_lds_load}{_persistent}{_drain}{_malign}" + f"{_interleaved_lds_load}{_persistent}{_drain}{_malign}{_kwaves}" ) # Compile the static path directly, preserving its launch ABI and the @@ -561,11 +601,18 @@ def gemm_task( ): A_KROW = K // A_PACK Kp16 = (K // 2) * 16 - K4 = K // 4 + # Scale dwords per 32-row B super-row: one per k128, padded K included. + K4 = K128 * 32 tid = fx.thread_idx.x bid_x = task_id if persistent_workers else fx.block_idx.x wave = rocdl.readfirstlane(T.i32, tid // WAVE) + k_group = 0 + if const_expr(k_waves > 1): + # Group 0 holds hardware waves 0..num_waves-1, so the group-0 epilogue + # sees the same wave ids as an unsplit workgroup. + k_group = wave // num_waves + wave = wave % num_waves lane = tid % WAVE lane16 = lane % 16 kgrp = lane // 16 @@ -614,7 +661,7 @@ def gemm_task( # The host launches a CUDAGraph-safe worst-case M grid. The final psum # entry is the end of the last expert's valid rows, so reject the padded - # tail before paying for the dependent expert bisect. blk_m is shared + # tail before paying for the expert lookup. blk_m is shared # by every workgroup in a cluster, making this predicate cluster-uniform. i32_ptr = fx.PointerType.get( elem_ty=fx.Int32.ir_type, address_space=fx.AddressSpace.Global, alignment=4 @@ -626,24 +673,26 @@ def gemm_task( has_work = True expert = fx.Int32(tile_meta[0][0]) else: - has_work = blk_m < tile_map[n_experts - 1] - - # In-kernel bisect: find the expert owning this valid M-tile. Seed - # the result with the padding sentinel so it remains defined - # outside the dynamic branch and the existing main-body guard - # stays valid. - expert = blk_m * 0 + n_experts - if has_work: - lo, hi = blk_m * 0, blk_m * 0 + n_experts - for _ in range_constexpr( - max(1, math.ceil(math.log2(max(2, n_experts))) + 1) - ): - mid = (lo + hi) >> 1 - mid_clamped = (mid < n_experts - 1).select(mid, n_experts - 1) - go_right = tile_map[mid_clamped] <= blk_m - lo = go_right.select(mid + 1, lo) - hi = go_right.select(hi, mid) - expert = lo + # psum is sorted, so the owning expert is the count of experts that + # end at or before blk_m. Every lane loads a strided slice in one + # round trip instead of a bisect's log2(E) dependent loads (the bulk + # of a decode tile's prologue latency). + psum_hits = fx.Int32(0) + for i in range_constexpr(ceildiv(n_experts, WAVE)): + e_idx = lane + i * WAVE + if const_expr((i + 1) * WAVE <= n_experts): + end_e = tile_map[e_idx] + psum_hits = psum_hits + (end_e <= blk_m).select(1, 0) + else: + e_ok = e_idx < n_experts + end_e = tile_map[e_ok.select(e_idx, n_experts - 1)] + psum_hits = psum_hits + (e_ok & (end_e <= blk_m)).select(1, 0) + for sh in range_constexpr(int.bit_length(WAVE) - 1): + psum_hits = psum_hits + psum_hits.shuffle_xor( + fx.Int32(1 << sh), fx.Int32(WAVE) + ) + expert = rocdl.readfirstlane(T.i32, psum_hits) + has_work = expert < n_experts if const_expr(m_align): # tile_map holds unpadded expert ends while experts start on # m_align boundaries; a tile narrower than m_align can start in the @@ -661,9 +710,7 @@ def gemm_task( # row stride is not K bytes and the scale row stride is not K/128 dwords. A_GROW = a_row_stride_bytes if a_row_stride_bytes else A_KROW # row-major: scale dwords per M row - SA_GROW = ( - a_scale_row_stride_bytes // 4 if a_scale_row_stride_bytes else K // 128 - ) + SA_GROW = a_scale_row_stride_bytes // 4 if a_scale_row_stride_bytes else K128 c_outer_off, c_inner_off, c_stride = blk_m64, blk_n64, i32_n SB_OUTER_STRIDE = K4 @@ -717,14 +764,26 @@ def cluster_sync(): base_ptr = lds_base[0] else: # static=False (one dyn-shared base) only where a second region is - # needed, so the non-scatter path keeps its per-leaf static - # allocation. + # needed, so the non-scatter path keeps its per-leaf static allocation. _smem = ( fx.SharedAllocator(static=False) if const_expr(enable_ep_scatter) else fx.SharedAllocator(static=True) ) - base_ptr = _smem.allocate(ARENA_B)._ptr + base_ptr = _smem.allocate( + GROUP_ARENA_B * k_waves if k_waves > 1 else ARENA_B + )._ptr + if const_expr(k_waves > 1): + # Rebuild the pointer so the group offset keeps its 16-B alignment; + # plain pointer arithmetic drops it, which the bf16 epilogue rejects. + base_ptr = fx.inttoptr( + fx.PointerType.get( + elem_ty=fx.Int8.ir_type, + address_space=fx.AddressSpace.Shared, + alignment=16, + ), + fx.Int32(fx.ptrtoint(base_ptr)) + k_group * GROUP_ARENA_B, + ) def ptr_to_idx(p): return fx.Index(fx.ptrtoint(p)) @@ -873,6 +932,11 @@ def add_tdm_loads( target_jobs=None, cache_modifier=0, ): + if const_expr(k_waves > 1 and k_adv): + # This group's first k-tile, in the view's element units. + g_off = g_off + fx.Int64(k_group * K_TILES) * fx.Int64( + k_adv // 4 if on_i32 else k_adv + ) split_i = split_inner and len(wv) > 1 if const_expr(len(wv) > 1): if const_expr(split_i): @@ -911,11 +975,14 @@ def add_tdm_loads( # global-load path; nonzero masks select CLUSTER_LOAD_ASYNC. atom = fx.atom_set_value(atom, "workgroup_mask", fx.Int32(wg_mask)) # Every K-advancing load lays a tile out k128-major, so the ragged - # last tile keeps the first KWS_TAIL/KWS of its inner extent. The - # extent is relative to this wave's base, which imm_offset bumps. + # last tile keeps the first K_TAIL/tile_k of its data extent; the + # scales hold whole k128s, KWS_TAIL/KWS of theirs. The extent is + # relative to this wave's base, which imm_offset bumps. tail_ext = None if const_expr(HAS_K_TAIL and k_adv): - tail_ext = inner * KWS_TAIL // KWS - wave_inner_off + tail_ext = ( + inner * KWS_TAIL // KWS if on_i32 else inner * K_TAIL // tile_k + ) - wave_inner_off (jobs if target_jobs is None else target_jobs).append( Job( atom, @@ -1885,6 +1952,57 @@ def compute_drain_tile(j, has_next, kws=KWS): # mainloop body and no wave-parity branch. run_mainloop(bool(interleaved_lds_load_on)) + if const_expr(k_waves > 1): + # Every group is past its last LDS input read once this fence + # retires, so each parks its partials in its own arena. Group 0 + # adds them in group order (deterministic) and alone keeps the + # tile; the other groups blank their epilogue via mn_oob. + pipeline_fence(outstanding=0) + KW_CHUNKS = WMMA_VECTOR_DWORDS // 4 + KW_LANES = num_waves * WAVE + kw_tid = tid % KW_LANES + + def kw_off(idx, q): + # Lane-contiguous dword4s: conflict-free b128 LDS traffic. + return ((idx * KW_CHUNKS + q) * KW_LANES + kw_tid) * 16 + + if k_group > 0: + for idx in range_constexpr(n_acc): + v = Vec(c_frags[idx].load()) + for q in range_constexpr(KW_CHUNKS): + lds_store_b128( + stC_idx, + kw_off(idx, q), + Vec.from_elements( + [v[q * 4 + i] for i in range_constexpr(4)], + fx.Float32, + ) + .bitcast(fx.Int32) + .ir_value(), + ) + workgroup_barrier() + if k_group == 0: + for idx in range_constexpr(n_acc): + own = Vec(c_frags[idx].load()) + tot = [] + for q in range_constexpr(KW_CHUNKS): + acc4 = Vec.from_elements( + [own[q * 4 + i] for i in range_constexpr(4)], + fx.Float32, + ) + for g in range_constexpr(1, k_waves): + part = Vec( + lds_load_b128( + stC_idx, g * GROUP_ARENA_B + kw_off(idx, q) + ) + ).bitcast(fx.Float32) + acc4 = acc4 + part + tot += [acc4[i] for i in range_constexpr(4)] + c_frags[idx].store( + Vec.from_elements(tot, fx.Float32).ir_value() + ) + mn_oob = (k_group == 0).select(mn_oob, mn_oob * 0) + accs = [] output_fragments_per_acc = WMMA_N // 16 for idx in range_constexpr(n_acc): @@ -1959,7 +2077,12 @@ def _biased_acc(wm, wn, col_rel): is_kgrp0 = fx.Int32(kgrp) == 0 # i32_n is the pre-activation gate+up width; the quantized # output has half as many columns and one scale dword per K128. - q_dst_scale_dwpr = i32_n // 256 + q_dst_scale_dwpr = (i32_n + 255) // 256 + # A half-k128 output tail leaves the last scale dword's upper two + # bytes without an MX block; GEMM2 reads them against zero data, + # so they get a finite e8m0 (a stale 0xFF is NaN). + q_last_mx_blk = (i32_n >> 6) - 1 + q_half_tail = (i32_n & 255) != 0 scale_rsrc = create_buffer_resource_from_addr( fx.ptrtoint(scale_ptr), num_records_bytes=( @@ -2127,6 +2250,23 @@ def quant_byte_offset(row, col): dst_byte, mask=is_kgrp0, ) + # Half-k128 tail (inter_dim % 128 == 64): one scale + # dword carries the e8m0 of four 32-column MX blocks, + # i.e. one k128 of GEMM2's K, but the last dword only + # has two real blocks. GEMM2 still reads all four + # bytes (against zero A data in the padded half), so + # the lane that writes the last real block also fills + # bytes 2..3 with 0x7F (2^0): a stale 0xFF there is + # NaN, and 0 * NaN poisons the whole output row. + buffer_store( + fx.Int16(0x7F7F), + scale_rsrc, + dst_byte - byte_in_dw + 2, + mask=is_kgrp0 + & q_half_tail + & (mx_blk_is[mx_blk] == q_last_mx_blk), + offset_is_bytes=True, + ) else: # bf16/f16 activation (or passthrough) -> stage to LDS. if const_expr(enable_ep_scatter): @@ -2439,6 +2579,15 @@ def issue_scatter_group(group, scatter_vector, vector_pass): issue_scatter_group(group, scatter_vector, vector_pass) tdm_ops.tensor_wait(0) else: + + def store_tile(gt, tile, lds_ptr, lds_row): + # A k_waves workgroup's tile belongs to group 0 alone. + if const_expr(k_waves > 1): + if k_group == 0: + tiled_store(gt, tile, lds_ptr, lds_row) + else: + tiled_store(gt, tile, lds_ptr, lds_row) + if const_expr(stage1_act): out_divisor = 4 if is_fp4_quant else 2 out_stride = i32_n // out_divisor @@ -2461,7 +2610,7 @@ def issue_scatter_group(group, scatter_vector, vector_pass): ((mn_oob + 15) // 16, STORE_N * 16), (out_stride * 16, 1), ) - tiled_store( + store_tile( gtC, (tile_m // 16, STORE_N * 16), fx.recast_iter(oc_store, c_q_base_ptr), @@ -2477,7 +2626,7 @@ def issue_scatter_group(group, scatter_vector, vector_pass): # c_q_base_ptr is base_ptr unless defer_c_wait, and that # asserts stage1_act and not enable_ep_scatter -- the only # case it differs already has STORE_PAD == 0 (PITCH == N). - tiled_store( + store_tile( gtC, (tile_m, STORE_PITCH), fx.recast_iter(oc_store, c_q_base_ptr), @@ -2570,6 +2719,7 @@ def launch_gemm_a8w4_tdm( rolled_drain: Constexpr[int] = 0, a_preshuffled: Constexpr[int] = 0, m_align: Constexpr[int] = 0, + k_waves: Constexpr[int] = 1, ): """Launch the grouped contiguous-M a8w4 MoE GEMM for gfx1250. @@ -2633,6 +2783,7 @@ def launch_gemm_a8w4_tdm( rolled_drain=rolled_drain, a_preshuffled=a_preshuffled, m_align=m_align, + k_waves=k_waves, ) @flyc.kernel(name=_kname, known_block_size=[block, 1, 1]) diff --git a/aiter/ops/shuffle.py b/aiter/ops/shuffle.py index 62fda92e779..d252bb2ccdc 100644 --- a/aiter/ops/shuffle.py +++ b/aiter/ops/shuffle.py @@ -248,7 +248,8 @@ def shuffle_scale_n32k4( """Shuffle a raw per-expert e8m0 weight (B) scale into the n32k4 layout. Input: ``(E, N, K//32)`` (3D) or ``(E*N, K//32)`` (2D, needs ``experts_cnt``). - Output: ``(E, N//32, (K//32)*32)`` uint8. + Output: ``(E, N//32, ceil(K/128)*128)`` uint8 (a K % 128 == 64 tail pads + its last k128 with e8m0 1.0). Within a 32-row super-row the column is ``remain_k*128 + row32*4 + r`` so each lane reads its full WMMA scaleB operand (4 e8m0 of one WMMA-K=128 step) with @@ -284,10 +285,14 @@ def shuffle_scale_n32k4( s = s.view(E, 2, N // 2, k_scale).permute(0, 2, 1, 3).reshape(E, N, k_scale) if N % 32 != 0: raise ValueError(f"B-scale rows must be divisible by 32, got {N}") + if k_scale % 2 != 0: + raise ValueError(f"B-scale K//32 must be even (K%64==0), got {k_scale}") if k_scale % 4 != 0: - raise ValueError( - f"B-scale K//32 must be divisible by 4 (K%128==0), got {k_scale}" - ) + # K % 128 == 64: the weights keep their half-k128 tail unpadded, and only + # the scales round up to a whole k128 with a finite e8m0 (1.0) that the + # GEMM multiplies against zero-filled operand bytes. + s = F.pad(s, (0, 4 - k_scale % 4), value=0x7F) + k_scale = s.shape[-1] g = s.view(E, N // 32, 32, k_scale // 4, 4).permute(0, 1, 3, 2, 4).contiguous() return g.reshape(E, N // 32, k_scale * 32) diff --git a/op_tests/flydsl_tests/test_flydsl_grouped_gemm.py b/op_tests/flydsl_tests/test_flydsl_grouped_gemm.py index ae55acf60ce..253e1d2da03 100644 --- a/op_tests/flydsl_tests/test_flydsl_grouped_gemm.py +++ b/op_tests/flydsl_tests/test_flydsl_grouped_gemm.py @@ -1605,13 +1605,12 @@ def main() -> None: run_csv_scenario(args) return - # The >=512 floor is a FlyDSL grouped-kernel constraint (tile_k=256 needs two - # K tiles). - if args.model_dim < 512 or args.inter_dim < 512: + # The >=256 floor is a FlyDSL grouped-kernel constraint (two K tiles at + # tile_k=128; e.g. Kimi-K3 TP8 runs inter_dim=384). + if args.model_dim < 256 or args.inter_dim < 256: raise SystemExit( f"model_dim ({args.model_dim}) and inter_dim ({args.inter_dim}) must be " - "at least 512 for the grouped GEMM kernels (tile_k=256 requires at " - "least two K tiles)." + "at least 256 for the grouped GEMM kernels (two K tiles at tile_k=128)." ) set_data_format(args.data_format)