diff --git a/aiter/ops/flydsl/kernels/mixed_moe_gemm_2stage_common.py b/aiter/ops/flydsl/kernels/mixed_moe_gemm_2stage_common.py index 83e08d75427..b3175328ecb 100644 --- a/aiter/ops/flydsl/kernels/mixed_moe_gemm_2stage_common.py +++ b/aiter/ops/flydsl/kernels/mixed_moe_gemm_2stage_common.py @@ -430,6 +430,20 @@ def x_lds_elem(): fp4_ratio = 2 if a_dtype == "fp4" else 1 gui_ratio = 1 if gate_up_interleave else 2 + # Stage 1 writes the intermediate's unpadded columns rounded up to stage 2's + # K tile (tile_k // 2): stage 2 reads that far, and columns left unwritten + # are stale memory (NaN through mfma_scale even against zero weights). The + # grid skips the rest of the padding, counted in w1 rows: two per column, + # (gate, up), in both the separated and the interleaved layouts. + inter_dim_written = inter_dim - inter_dim_pad + if not gate_only: + tile_k_stage2 = tile_k // 2 + inter_dim_written = min( + inter_dim, + -(-inter_dim_written // tile_k_stage2) * tile_k_stage2, + ) + n_rows_skipped = 2 * (inter_dim - inter_dim_written) + if True: def _emit_moe_gemm1( @@ -458,7 +472,6 @@ def _emit_moe_gemm1( ): tokens_in = fx.Index(i32_tokens_in) - n_in = fx.Index(i32_n_in) k_in = fx.Index(i32_k_in) size_expert_ids_in = fx.Index(i32_size_expert_ids_in) # Runtime clamp bound for the activation. Host passes the configured @@ -520,20 +533,8 @@ def _emit_moe_gemm1( if const_expr(xcd_swizzle > 0): num_xcds = 8 - one = arith.constant(1, index=True) - tile_n_idx = arith.constant(tile_n, index=True) - inter_pad_idx = arith.constant(2 * inter_dim_pad, index=True) - if const_expr(mock_gate_only or gate_up_interleave): - grid_x = (n_in - inter_pad_idx + tile_n_idx - one) // tile_n_idx - else: - two = arith.constant(2, index=True) - grid_x = ( - (n_in - inter_pad_idx + two * tile_n_idx - one) - // tile_n_idx - // two - ) - persist_m_idx = arith.constant(persist_m, index=True) - grid_y = (size_expert_ids_in + persist_m_idx - one) // persist_m_idx + grid_x = fx.Index(gpu.grid_dim.x) + grid_y = fx.Index(gpu.grid_dim.y) linear_id = bx_persist * grid_x + by num_workgroups = grid_x * grid_y num_xcds_idx = arith.constant(num_xcds, index=True) @@ -2952,23 +2953,14 @@ def _launch_mixed_moe_gemm1( # manual finalize is needed. ctx = CompilationContext.get_current() - inter_dim_pad_total = arith.constant(2 * inter_dim_pad, index=True) - tile2_pad = 0 - if const_expr(not gate_only): - tile_k_stage2 = tile_k // 2 - tile2_pad = ( - tile_k_stage2 - (inter_dim - inter_dim_pad) % tile_k_stage2 - ) % tile_k_stage2 - + n_rows_skipped_idx = arith.constant(n_rows_skipped, index=True) inter_in = fx.Index(i32_inter_in) tile_n_index = arith.constant(tile_n, index=True) if const_expr(mock_gate_only or gate_up_interleave): - gx = ( - inter_in - inter_dim_pad_total + tile2_pad + tile_n_index - 1 - ) // tile_n_index + gx = (inter_in - n_rows_skipped_idx + tile_n_index - 1) // tile_n_index else: gx = ( - (inter_in - inter_dim_pad_total + tile2_pad + 2 * tile_n_index - 1) + (inter_in - n_rows_skipped_idx + 2 * tile_n_index - 1) // tile_n_index // arith.constant(2, index=True) )