[ASM] [HIP] 1x32 mxfp4 asm kernel

aiter/configs/model_configs/qwen3_8_2400b_fp4_tuned_fmoe.csv

@@ -1,7 +1,7 @@
 cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,use_g1u1,doweight_stage1,block_m,ksplit,us1,kernelName1,err1,us2,kernelName2,err2,us,run_1stage,xbf16,flat,tflops,bw,_tag
-256,1,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,11.711,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w4_kw4_fp4,1.0%,6.9428,flydsl_moe2_layout_afp4_wfp4_bf16_t32x128x256_atomic_nt_sbm32,1.2%,18.6538,0,0,0,6.75,172686.0,
-256,2,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,13.4714,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_kw4_fp4,1.0%,9.2709,flydsl_moe2_layout_afp4_wfp4_bf16_t32x128x128_atomic_persist_sbm32,1.2%,22.7423,0,0,0,11.07,141642.43,
-256,4,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,43.5997,_ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x128E,0.0%,0.0,,0.0%,43.5997,1,1,1,0,0,
+256,1,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,20.87,_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32E,0.0%,0.0,,0.0%,20.87,1,1,1,0,0,
+256,2,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,23.95,_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32E,0.0%,0.0,,0.0%,23.95,1,1,1,0,0,
+256,4,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,44.71,_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32E,0.0%,0.0,,0.0%,44.71,1,1,1,0,0,
 256,8,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,51.4795,_ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x128E,0.0%,0.0,,0.0%,51.4795,1,1,1,0,0,
 256,16,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,56.2774,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_kw2_fp4,0.1%,34.408,flydsl_moe2_afp4_wfp4_bf16_t16x256x128_atomic_persist_sbm32,2.1%,90.6854,0,0,0,22.2,35525.22,
 256,32,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,87.0351,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w4_fp4,0.2%,53.7459,flydsl_moe2_afp4_wfp4_bf16_t16x256x128_atomic_sbm32,2.1%,140.781,0,0,0,28.6,22886.7,
@@ -15,9 +15,9 @@ cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,
 256,8192,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,349.9716,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w4_bnt0_xcd4_fp4,1.0%,654.1716,flydsl_moe2_layout_afp4_wfp4_bf16_t64x256x128_reduce_sbm64,1.2%,1004.1432,0,0,0,1026.54,3408.43,
 256,16384,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,538.8996,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w3_bnt0_xcd4_fp4,1.0%,1230.5056,flydsl_moe2_layout_afp4_wfp4_bf16_t64x256x128_reduce_sbm64,1.2%,1769.4052,0,0,0,1165.13,2048.08,
 256,32768,8192,256,512,10,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,954.6287,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w4_bnt0_xcd4_fp4,0.1%,2458.5324,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_xcd4,0.1%,3413.1611,0,0,0,1208.02,1179.71,
-256,1,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,11.8696,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_kw4_fp4,1.0%,7.354,flydsl_moe2_layout_afp4_wfp4_bf16_t32x128x128_atomic_nt_sbm32,1.2%,19.2236,0,0,0,7.2,167894.75,
-256,2,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,13.7007,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w2_kw4_fp4,1.0%,9.6647,flydsl_moe2_layout_afp4_wfp4_bf16_t32x128x256_atomic_sbm32,1.2%,23.3654,0,0,0,11.85,138134.42,
-256,4,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,44.4998,_ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x128E,0.0%,0.0,,0.0%,44.4998,1,1,1,0,0,
+256,1,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,21.24,_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32E,0.0%,0.0,,0.0%,21.24,1,1,1,0,0,
+256,2,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,25.42,_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32E,0.0%,0.0,,0.0%,25.42,1,1,1,0,0,
+256,4,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,46.16,_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32E,0.0%,0.0,,0.0%,46.16,1,1,1,0,0,
 256,8,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,16,0,54.7090,_ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x128E,0.0%,0.0,,0.0%,54.7090,1,1,1,0,0,
 256,16,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,57.9574,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_kw2_fp4,0.1%,36.0351,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce,0.1%,93.9925,0,0,0,23.56,34342.21,
 256,32,8192,256,513,11,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,92.3109,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_kw2_fp4,0.1%,55.7202,flydsl_moe2_afp4_wfp4_bf16_t16x256x128_atomic_sbm32,2.2%,148.0311,0,0,0,29.92,21808.28,

csrc/py_itfs_cu/asm_fmoe.cu

@@ -557,6 +557,7 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID(
     FMoeKernel* impl_ptr = nullptr;
     CFG* config_map      = nullptr;
     int smf              = 0;
+    bool is_mxfp4        = false;
     int model_dim        = down->size(1);
     int inter_dim        = down->size(2);
     inter_dim *= model_dim / gate->size(2);
@@ -609,6 +610,7 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID(
             AITER_CHECK(false, __func__, " Not find proper cfg in pertokenMXfp4_g1u1. ");
         impl_ptr = get_heuristic_kernel(inter_dim, sub_X_cnt, config_map, smf, kernel_name_str);
         impl_ptr->set_4bit(true);
+        is_mxfp4 = true;
     }
     else if((input->dtype() == AITER_DTYPE_bf16 || input->dtype() == AITER_DTYPE_fp16) &&
             gate->dtype() == AITER_DTYPE_fp4x2) // bf16/fp16 X + MXFP4 weights (in-kernel X quant)
@@ -628,6 +630,7 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID(
             AITER_CHECK(false, __func__, " Not find proper cfg in pertokenMXfp4_g1u1 (bf16 X). ");
         impl_ptr = get_heuristic_kernel(inter_dim, sub_X_cnt, config_map, smf, kernel_name_str);
         impl_ptr->set_4bit(true);
+        is_mxfp4 = true;
     }
     else if(input->dtype() == AITER_DTYPE_i8 || input->dtype() == AITER_DTYPE_u8) // int8
     {
@@ -666,6 +669,22 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID(
         AITER_CHECK(false, __func__, ": unsupport current input type:", AiterDtype_to_str(input->dtype()));
     }
 
+    // The small-tile asm MXFP4 kernels drain O through a tile schedule derived
+    // from model_dim/1024, with no handling for a short trailing tile. Below
+    // 1024 no tiles are produced at all and the output buffer is left
+    // untouched; a non-multiple leaves the remainder columns unwritten. Both
+    // return a silently wrong result instead of failing, so require an exact
+    // multiple. A multiple of 256 is not enough: model_dim=2304 drops 256
+    // columns. Larger tiles use a different epilogue and are not constrained.
+    if(is_mxfp4 && impl_ptr->get_sub_GU() <= 128)
+    {
+        AITER_CHECK(model_dim >= 1024 && (model_dim % 1024) == 0,
+                    __func__,
+                    " asm MXFP4 kernels with sub_GU <= 128 require model_dim to be a positive "
+                    "multiple of 1024; got model_dim=" +
+                        std::to_string(model_dim));
+    }
+
     impl_ptr->launch_kernel<1, 2>(out,
                                   input,
                                   gate,

hsa/gfx950/fmoe/gelu/fmoe_bf16_pertokenMXfp4_g1u1_gelu.csv

@@ -9,3 +9,4 @@ _ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_vs_gelu_2tg_ps_32x256E,fmoe_bf16_pertoke
 _ZN5aiter52fmoe_bf16_pertokenMXfp4_g1u1_novs_gelu_2tg_ps_32x256E,fmoe_bf16_pertokenMXfp4_g1u1_novs_gelu_2tg_ps_32x256.co,0,0,0,2,1,32,256,0
 _ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_gelu_16x128E,fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_gelu_16x128.co,0,0,0,2,0,16,128,1
 _ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_gelu_16x256E,fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_gelu_16x256.co,0,0,0,2,0,16,256,1
+_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_gelu_16x32E,fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_gelu_16x32.co,0,1,0,2,0,16,32,1

hsa/gfx950/fmoe/silu/fmoe_bf16_pertokenMXfp4_g1u1_silu.csv

@@ -9,3 +9,4 @@ _ZN5aiter52fmoe_bf16_pertokenMXfp4_g1u1_novs_silu_1tg_ps_32x512E,fmoe_bf16_perto
 _ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_vs_silu_2tg_ps_32x256E,fmoe_bf16_pertokenMXfp4_g1u1_vs_silu_2tg_ps_32x256.co,0,1,0,2,1,32,256,0
 _ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x128E,fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x128.co,0,0,0,2,0,16,128,1
 _ZN5aiter50fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x256E,fmoe_bf16_pertokenMXfp4_g1u1_flat_novs_silu_16x256.co,0,0,0,2,0,16,256,1
+_ZN5aiter47fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32E,fmoe_bf16_pertokenMXfp4_g1u1_flat_vs_silu_16x32.co,0,1,0,2,0,16,32,1