Skip to content

tune token 16 on MI355 #1461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
coderfeli merged 10 commits into from
Nov 27, 2025
Merged

tune token 16 on MI355 #1461

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c2c97c0
tune token 16 on MI355
zhiding512 Nov 21, 2025
3ac8e1a
tuned a8w8 m1~16
zhiding512 Nov 21, 2025
9f3a113
tune fp8 m32-1024
zhiding512 Nov 24, 2025
2a86236
tune.py bugfix
xudoyuan Nov 24, 2025
ae092db
rm garbage code
xudoyuan Nov 25, 2025
9051b66
Merge branch 'main' into fp8_tuned
xudoyuan Nov 25, 2025
8faa4ae
logits_diff warning
xudoyuan Nov 25, 2025
865ea8a
black op_tests/test_moe_2stage.py
xudoyuan Nov 25, 2025
34640c6
Merge branch 'main' into fp8_tuned
xudoyuan Nov 25, 2025
80b4323
Merge branch 'main' into fp8_tuned
xudoyuan Nov 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions aiter/configs/tuned_fmoe.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,3 +786,8 @@ cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,
80,256,7168,2048,33,10,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_Token,1,0,32,0,1021.9423,_ZN5aiter45fmoe_bf16_pertokenFp8_g1u1_vs_silu_1tg_32x512E,5.4%,0.0,Null,0.0%,1021.9423,1,220.64,1427.51
80,512,7168,2048,33,10,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_Token,1,0,32,0,1749.1923,_ZN5aiter45fmoe_bf16_pertokenFp8_g1u1_vs_silu_1tg_32x512E,5.4%,0.0,Null,0.0%,1749.1923,1,257.82,837.15
80,1024,7168,2048,33,10,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_Token,1,0,32,0,3226.5114,_ZN5aiter45fmoe_bf16_pertokenFp8_g1u1_vs_silu_1tg_32x512E,5.3%,0.0,Null,0.0%,3226.5114,1,279.54,457.26
256,1,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0,16,0,45.285,moe_ck2stages_gemm1_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight0_silu_F8_F8_B16,0.1%,9.0945,moe_ck2stages_gemm2_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight1_F8_F8_B16,15.7%,54.3795,0,1.62,25916.16
256,2,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0,16,0,46.5232,moe_ck2stages_gemm1_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight0_silu_F8_F8_B16,0.1%,11.8082,moe_ck2stages_gemm2_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight1_F8_F8_B16,15.3%,58.3314,0,3.02,24160.73
256,4,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0,16,0,48.2418,moe_ck2stages_gemm1_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight0_silu_F8_F8_B16,0.1%,17.8498,moe_ck2stages_gemm2_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight1_F8_F8_B16,15.4%,66.0916,0,5.33,21324.53
256,8,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0,16,0,53.6435,moe_ck2stages_gemm1_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight0_silu_F8_F8_B16,0.1%,25.7951,moe_ck2stages_gemm2_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight1_F8_F8_B16,15.7%,79.4386,0,8.87,17742.74
256,16,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0,16,0,71.1678,moe_ck2stages_gemm1_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight0_silu_F8_F8_B16,0.1%,41.5098,moe_ck2stages_gemm2_256x16x128x256_1x4_MulABScaleExpertWeightA8W8blkscale_v1_Nswizzle0_Quant4_MulRoutedWeight1_F8_F8_B16,15.6%,112.6776,0,12.51,12510.3
11 changes: 5 additions & 6 deletions aiter/configs/untuned_fmoe.csv
View file
Open in desktop
Comment thread
xudoyuan marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,use_g1u1,doweight_stage1
512,6144,4096,8,2,ActivationType.Silu,torch.bfloat16,torch.bfloat16,torch.bfloat16,QuantType.No,1,0
512,6144,4096,8,2,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_Token,1,0
512,6144,4096,8,2,ActivationType.Silu,torch.bfloat16,torch.int8,torch.int8,QuantType.per_Tensor,1,0
4,2304,1536,8,2,ActivationType.Gelu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_Token,1,0
4,2304,1536,8,2,ActivationType.Gelu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_Token,1,1
56,6144,4096,8,2,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fnuz,torch.float8_e4m3fnuz,QuantType.per_1x128,1,0
1,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0
2,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0
4,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0
8,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0
16,7168,256,256,8,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x128,1,0
10 changes: 5 additions & 5 deletions aiter/test_common.py
View file
Open in desktop
Comment thread
xudoyuan marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@

pd.set_option("display.max_rows", 200)
## debug ##
# pd.set_option("display.max_rows", None)
# pd.set_option("display.max_columns", None)
# pd.set_option("display.width", None)
# pd.set_option("display.max_colwidth", None)
# pd.set_option("display.expand_frame_repr", False)
pd.set_option("display.max_rows", None)
pd.set_option("display.max_columns", None)
pd.set_option("display.width", None)
pd.set_option("display.max_colwidth", None)
pd.set_option("display.expand_frame_repr", False)


def perftest(
Expand Down
4 changes: 2 additions & 2 deletions csrc/ck_gemm_moe_2stages_codegen/gemm_moe_ck2stages_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ def get_gemm1_kernels_list(
QuantType: str,
ActOP: str,
MulRoutedWeight: bool,
preshuffle: bool,
preshuffle: bool = False,
) -> list:
arch = get_gfx()
if Adtype in bit16_list and Bdtype in bit16_list and Adtype == Adtype:
Expand Down Expand Up @@ -393,7 +393,7 @@ def get_gemm2_kernels_list(
Nswizzle: bool,
QuantType: str,
MulRoutedWeight: bool,
preshuffle: bool,
preshuffle: bool = False,
) -> list:
arch = get_gfx()

Expand Down
8 changes: 5 additions & 3 deletions hsa/gfx942/fmoe_2stages/tune.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1539,6 +1539,7 @@ def gen_2stages_task(self, key, blockMs):
int(q_type),
str(act_type).split(".")[-1].lower(),
doweight_stage1,
False,
Comment thread
xudoyuan marked this conversation as resolved.
Outdated
)
_, ck_stage2_kernels = get_gemm2_kernels_list(
dtype2str_dict[q_dtype_a],
Expand All @@ -1547,6 +1548,7 @@ def gen_2stages_task(self, key, blockMs):
False,
int(q_type),
not doweight_stage1,
False,
)
for blockM in blockMs:
if blockM in [16, 32, 64, 128] and use_g1u1:
Expand Down Expand Up @@ -1815,9 +1817,9 @@ def post_process(self, results, args, topk=-1, fast_mode=False):
)
prorfiles.append(profileDF)

profileDF = profileDF.sort_values("us").drop_duplicates(
["stage", "block_m"], keep="first"
)
# profileDF = profileDF.sort_values("us").drop_duplicates(
Comment thread
xudoyuan marked this conversation as resolved.
Outdated
# ["stage", "block_m"], keep="first"
# )
stage1_profileDF = profileDF[profileDF["stage"] == "stage1"].drop(
columns=["stage"], axis=1
)
Expand Down