[CK] Add dsv41 some MoE tuning configs

aiter/configs/model_configs/dsv41_fp4_tuned_fmoe.csv

@@ -111,3 +111,26 @@ gfx950,256,4096,5120,1152,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_
 gfx950,256,8192,5120,1152,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,1178.4115,flydsl_mxmoe_g1_a4w4_64x256x256_xcd2,1.2%,0,flydsl_moe2_layout_afp4_wfp4_bf16_t64x256x128_atomic_sbm64,1.2%,1178.4115,0,0,0,1476.11,5872.82
 gfx950,256,16384,5120,1152,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,1847.0189,flydsl_mxmoe_g1_a4w4_128x256x256_xcd4,1.2%,0,flydsl_moe2_layout_afp4_wfp4_bf16_t128x128x128_atomic_sbm128,1.2%,1847.0189,0,0,0,1883.53,3815.03
 gfx950,256,32768,5120,1152,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,3253.4182,flydsl_mxmoe_g1_a4w4_128x256x256_xcd2,1.2%,0,flydsl_moe2_layout_afp4_wfp4_bf16_t128x128x128_atomic_sbm128,1.2%,3253.4182,0,0,0,2138.63,2243.21
+gfx950,256,1,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,12.5991,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w2_gui_kw4_fp8,0.0%,9.4207,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,22.0198,0,0,0,11.25,309379.86
+gfx950,256,2,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,17.834,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w4_gui_kw2_fp8,0.0%,14.2994,flydsl_moe2_layout_afp8_wfp4_bf16_t32x128x128_reduce_sbm32,0.0%,32.1334,0,0,0,15.42,212006.76
+gfx950,256,4,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,29.1457,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w3_gui_fp8,0.0%,21.3727,flydsl_moe2_layout_afp8_wfp4_bf16_t32x128x128_reduce_nt_sbm32,0.0%,50.5184,0,0,0,19.61,134852.42
+gfx950,256,8,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,56.9439,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w2_gui_fp8,0.0%,34.5609,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,91.5048,0,0,0,21.66,74450.63
+gfx950,256,16,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,111.4429,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w2_gui_fp8,0.0%,59.2071,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,170.65,0,0,0,23.23,39922.14
+gfx950,256,32,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,212.9176,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_gui_kw2_fp8,0.0%,110.7525,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,323.6701,0,0,0,24.49,21049.08
+gfx950,256,64,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,357.1852,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_gui_kw2_fp8,0.0%,183.8611,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,541.0463,0,0,0,29.3,12593.1
+gfx950,256,128,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,359.6521,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_gui_kw2_fp8,0.0%,193.2278,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,552.8799,0,0,0,57.35,12325.34
+gfx950,256,256,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,363.7643,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w3_gui_fp8,0.0%,201.7408,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,565.5051,0,0,0,112.14,12053.65
+gfx950,256,512,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,367.7689,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w3_gui_fp8,0.0%,191.9912,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,559.7601,0,0,0,226.59,12184.38
+gfx950,256,1024,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,378.2528,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w4_gui_fp8,0.0%,225.2484,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,603.5012,0,0,0,420.33,11314.3
+gfx950,256,2048,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,433.2996,flydsl_moe1_afp8_wfp4_bf16_t64x256x256_w3_gui_fp8,0.0%,238.7783,flydsl_moe2_layout_afp8_wfp4_bf16_t64x256x128_atomic_nt_sbm64,0.0%,672.0779,0,0,0,754.89,10183.23
+gfx950,256,4096,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,733.2861,flydsl_moe1_afp8_wfp4_bf16_t128x256x256_gui_fp8,0.0%,386.4571,flydsl_moe2_layout_afp8_wfp4_bf16_t128x256x128_atomic_nt_sbm128,0.0%,1119.7432,0,0,0,906.18,6140.14
+gfx950,256,1,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,11.357,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w4_gui_kw4_fp8,0.0%,9.3492,flydsl_moe2_layout_afp8_wfp4_bf16_t32x128x128_atomic_nt_sbm32,0.0%,20.7062,0,0,0,6.84,110239.17
+gfx950,256,2,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,15.0687,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_bnt0_gui_kw2_fp8,0.0%,11.688,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_reduce_sbm32,0.0%,26.7567,0,0,0,10.58,85311.33
+gfx950,256,4,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,22.8031,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_bnt0_gui_fp8,0.0%,14.9931,flydsl_moe2_layout_afp8_wfp4_bf16_t32x128x128_reduce_sbm32,0.0%,37.7962,0,0,0,14.98,60394.44
+gfx950,256,8,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,37.8476,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w3_gui_fp8,0.0%,23.1824,flydsl_moe2_layout_afp8_wfp4_bf16_t32x128x128_reduce_sbm32,0.0%,61.03,0,0,0,18.56,37403.6
+gfx950,256,16,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,66.8009,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w4_gui_kw2_fp8,0.0%,38.5687,flydsl_moe2_layout_afp8_wfp4_bf16_t32x128x128_atomic_nt_sbm32,0.0%,105.3696,0,0,0,21.5,21665.31
+gfx950,256,32,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,125.0418,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w4_gui_fp8,0.0%,68.6861,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,193.7279,0,0,0,23.38,11785.14
+gfx950,256,64,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,130.5678,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_gui_kw2_fp8,0.0%,70.573,flydsl_moe2_layout_afp8_wfp4_bf16_t32x128x128_reduce_nt_sbm32,0.0%,201.1408,0,0,0,45.04,11353.25
+gfx950,256,128,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,131.4061,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_gui_kw2_fp8,0.0%,72.6615,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,204.0676,0,0,0,88.79,11195.24
+gfx950,256,256,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,133.9661,flydsl_moe1_afp8_wfp4_bf16_t32x64x256_w3_gui_kw2_fp8,0.0%,73.4762,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,207.4423,0,0,0,174.69,11022.59
+gfx950,256,512,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,135.6471,flydsl_moe1_afp8_wfp4_bf16_t32x128x256_w2_gui_fp8,0.0%,76.8347,flydsl_moe2_layout_afp8_wfp4_bf16_t32x256x128_atomic_nt_sbm32,0.0%,212.4818,0,0,0,341.1,10779.67

aiter/configs/model_configs/dsv41_fp4_untuned_fmoe.csv

@@ -15,3 +15,26 @@ token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,
 8192,5120,640,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
 16384,5120,640,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
 32768,5120,640,384,6,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+1,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+2,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+4,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+8,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+16,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+32,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+64,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+128,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+256,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+512,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+1024,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+2048,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+4096,5120,1152,385,7,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+1,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+2,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+4,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+8,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+16,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+32,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+64,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+128,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+256,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0
+512,5120,1152,129,4,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0

csrc/ck_gemm_moe_2stages_codegen/gemm_moe_tune.py

@@ -966,19 +966,22 @@ def generate_v2_stage1_data(
         #   a=a1_qt, w1=w1_qt_shuf, w1_scale=w1_scale_shuf, a1_scale=a1_scale_sort,
         #   v2_output_layout=True, ...). Recipe mirrors
         # mxfp4_v2_tune_utils.populate_baseline_v2_intermediate.
-        d = _v2_gen(
-            token,
-            model_dim,
-            inter_dim,
-            expert,
-            topk,
-            blockM,
-            adtype=adtype,
-            b_dtype=b_dtype,
-            activation=act_type,
-            situ_beta=DEFAULT_SITUV2_BETA,
-            situ_linear_beta=DEFAULT_SITUV2_LINEAR_BETA,
-        )
+        # Generate on the GPU: the CPU fp4 packing of E x 2I x H weights takes
+        # tens of minutes per shape at E=385.
+        with torch.device(device):
+            d = _v2_gen(
+                token,
+                model_dim,
+                inter_dim,
+                expert,
+                topk,
+                blockM,
+                adtype=adtype,
+                b_dtype=b_dtype,
+                activation=act_type,
+                situ_beta=DEFAULT_SITUV2_BETA,
+                situ_linear_beta=DEFAULT_SITUV2_LINEAR_BETA,
+            )
         v = _v2_build_inputs(d, token, model_dim, inter_dim, expert, topk, blockM)
         # Precompute the sorted A-scale here so it is NOT timed in the run func.
         a1_scale_sort = moe_mxfp4_sort(