Minor modifications

Signed-off-by: Jinyang Yuan <[email protected]>

Author: jinyangyuan-nvidia

tensorrt_llm/_torch/custom_ops/torch_custom_ops.py

@@ -88,7 +88,7 @@ def forward(
     ):
         x, fc1_expert_weights, fc1_expert_biases, fc2_expert_weights, fc2_expert_biases = inputs
         self.fused_moe_runner.run_gemm_profile(
-            x.contiguous(),
+            x,
             fc1_expert_weights,
             fc1_expert_biases,
             fc2_expert_weights,
@@ -148,6 +148,8 @@ def fused_moe(
     tuner = AutoTuner.get()
     MoERunner.refine_tuning_config(tune_max_num_tokens)
 
+    # Only the non-alltoall case is considered for profiling in the warmup phase.
+    # Therefore, to get the correct tactics during the actual inference, the inputs to the tuner should be the same as when not using alltoall.
     if enable_alltoall:
         assert tuner_num_tokens is not None
         assert tuner_top_k is not None

tensorrt_llm/_torch/modules/fused_moe/fused_moe_wide_ep.py

@@ -438,7 +438,9 @@ def forward_chunk(
         # If alltoall is disabled, we need also disable use_postquant_alltoall
         use_postquant_alltoall = self.use_postquant_alltoall and use_all_to_all
 
-        # Prepare additional information for profiling in case padding is applied in all-to-all
+        # Prepare additional information for profiling in case padding is applied when using alltoall.
+        # Only the non-alltoall case is considered for profiling in the warmup phase.
+        # Therefore, to get the correct tactics during the actual inference, the inputs to the tuner should be the same as when not using alltoall.
         if use_all_to_all:
             if all_rank_num_tokens is not None:
                 tuner_num_tokens = sum(all_rank_num_tokens)