[fix] fix the bugs/outdated commands in .sh scripts and the corresponding snapshots

docs/models/deepseek/deepseek.md

@@ -45,15 +45,25 @@ dynamic sampling and eval.
 
 ### Multi-node fan-out
 
-The launcher starts the Ray head on the invoking node only. Join every other node to the cluster
-first:
+The `torch_dist` conversion and the node-local rsync fan out across every node of the Ray
+cluster, so a multi-node run needs the whole cluster joined **before** the launcher starts, and
+the launcher must be told not to replace it. Bring up the head, join the workers, then run with
+`MILES_SCRIPT_EXTERNAL_RAY=1`:
 
 ```bash
+# on node 0
+ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 8 --disable-usage-stats
+
+# on every other node
 ray start --address=${MASTER_ADDR}:6379 --num-gpus 8 \
           --node-ip-address ${WORKER_IP} --disable-usage-stats
+
+# back on node 0
+MILES_SCRIPT_EXTERNAL_RAY=1 python scripts/run_deepseek.py train \
+   --num-nodes 16 --num-gpus-per-node 8
 ```
 
-With an MPI-style hostfile (each line `ip slot=8`), fan out from node 0:
+With an MPI-style hostfile (each line `ip slot=8`), fan the workers out from node 0:
 
 ```bash
 for WORKER_IP in $(awk '{print $1}' $BASE_DIR/mpi_hostfile); do
@@ -68,6 +78,10 @@ done
 wait
 ```
 
+Without `MILES_SCRIPT_EXTERNAL_RAY=1` the launcher runs `ray stop --force` and starts its own
+single-node head, which is what the single-node invocation above relies on — the conversion then
+runs on that one node.
+
 ## 4. Checkpoint conversion
 
 `train` performs the two conversion steps for you; the equivalent manual commands are below.

miles/utils/external_utils/command_utils.py

@@ -169,7 +169,8 @@ def execute_train(
                 "CUDA_DEVICE_MAX_CONNECTIONS": "1",
             }
         ),
-        "NCCL_NVLS_ENABLE": os.environ.get("NCCL_NVLS_ENABLE", str(int(check_has_nvlink()))),
+        # a get() default is evaluated eagerly, which would probe even when already decided
+        "NCCL_NVLS_ENABLE": os.environ.get("NCCL_NVLS_ENABLE") or str(int(check_has_nvlink())),
         **{
             k: os.environ[k]
             for k in ("NCCL_SOCKET_IFNAME", "GLOO_SOCKET_IFNAME", "NCCL_DEBUG", "NCCL_DEBUG_FILE")

scripts/amd/run_qwen3_30b_a3b.py

@@ -222,6 +222,7 @@ def execute(args: ScriptArgs):
 
     U.execute_train(
         train_args=train_args,
+        config=args,
         num_gpus_per_node=args.num_gpus_per_node,
         megatron_model_type=args.megatron_model_type,
         extra_env_vars={**misc_env_vars},

scripts/run-glm4.5-355B-A32B.sh

@@ -15,13 +15,6 @@ set -ex
 # will prevent ray from buffering stdout/stderr
 export PYTHONUNBUFFERED=1
 
-NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l)
-if [ "$NVLINK_COUNT" -gt 0 ]; then
-    HAS_NVLINK=1
-else
-    HAS_NVLINK=0
-fi
-echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"
 
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 MODEL_ARGS_LINE="$(python3 "${SCRIPT_DIR}/../miles/utils/external_utils/model_args_utils.py" "glm4.5-355B-A32B")" || exit 1
@@ -189,7 +182,6 @@ ray job submit --address="http://127.0.0.1:8265" \
    --actor-num-nodes 8 \
    --actor-num-gpus-per-node 8 \
    --colocate \
-   --save-debug-rollout-data /mnt/zhuzilin/github-miles/data.pt \
    ${MODEL_ARGS[@]} \
    ${CKPT_ARGS[@]} \
    ${ROLLOUT_ARGS[@]} \

scripts/run-gpt-oss-20b-bf16.sh

@@ -16,6 +16,14 @@ set -ex
 export PYTHONUNBUFFERED=1
 export HF_HOME=/workspace/hf_cache
 
+NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l)
+if [ "$NVLINK_COUNT" -gt 0 ]; then
+    HAS_NVLINK=1
+else
+    HAS_NVLINK=0
+fi
+echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"
+
 # Load model architecture config
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 MODEL_ARGS_LINE="$(python3 "${SCRIPT_DIR}/../miles/utils/external_utils/model_args_utils.py" "gpt-oss-20b")" || exit 1
@@ -143,6 +151,5 @@ ray job submit --address="http://127.0.0.1:8265" \
    ${GRPO_ARGS[@]} \
    ${WANDB_ARGS[@]} \
    ${PERF_ARGS[@]} \
-   ${EVAL_ARGS[@]} \
    ${SGLANG_ARGS[@]} \
    ${MISC_ARGS[@]}

scripts/run-kimi-k25.sh

@@ -52,7 +52,6 @@ ROLLOUT_ARGS=(
    --rollout-temperature 1
 
    --global-batch-size 256
-   --filter-zero-reward-samples
    --use-dynamic-global-batch-size
 )
 

scripts/run-qwen3-235B-A22B-sft.sh

@@ -147,5 +147,4 @@ ray job submit --address="http://127.0.0.1:8265" \
    ${OPTIMIZER_ARGS[@]} \
    ${WANDB_ARGS[@]} \
    ${PERF_ARGS[@]} \
-   ${EVAL_ARGS[@]} \
    ${MISC_ARGS[@]}

scripts/run-qwen3-4B-base-sft.sh

@@ -123,5 +123,4 @@ ray job submit --address="http://127.0.0.1:8265" \
    ${OPTIMIZER_ARGS[@]} \
    ${WANDB_ARGS[@]} \
    ${PERF_ARGS[@]} \
-   ${EVAL_ARGS[@]} \
    ${MISC_ARGS[@]}

scripts/run-qwen3-4B_4xgpu.sh

@@ -100,12 +100,15 @@ OPTIMIZER_ARGS=(
    --adam-beta2 0.98
 )
 
-WANDB_ARGS=(
-   --use-wandb
-   --wandb-project miles-dev-qwen3-radix
-   --wandb-group qwen3-4B-4xgpu
-   --wandb-key ${WANDB_KEY}
-)
+WANDB_ARGS=()
+if [ -n "${WANDB_KEY:-}" ]; then
+   WANDB_ARGS=(
+      --use-wandb
+      --wandb-project miles-dev-qwen3-radix
+      --wandb-group qwen3-4B-4xgpu
+      --wandb-key "${WANDB_KEY}"
+   )
+fi
 
 SGLANG_ARGS=(
    --rollout-num-gpus-per-engine 2
@@ -140,6 +143,7 @@ ray job submit --address="http://127.0.0.1:8265" \
    -- python3 train.py \
    --actor-num-nodes 1 \
    --actor-num-gpus-per-node 4 \
+   --num-gpus-per-node 4 \
    --colocate \
    ${MODEL_ARGS[@]} \
    ${CKPT_ARGS[@]} \

scripts/run-qwen3-next-80B-A3B.sh

@@ -166,7 +166,7 @@ RUNTIME_ENV_JSON="{
   \"env_vars\": {
     \"PYTHONPATH\": \"/root/Megatron-LM/\",
     \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\",
-    \"NCCL_NVLS_ENABLE\": \"${HAS_NVLINK}\"
+    \"NCCL_NVLS_ENABLE\": \"${HAS_NVLINK}\",
     \"no_proxy\": \"${no_proxy}\",
     \"MASTER_ADDR\": \"${MASTER_ADDR}\"
   }

scripts/run_deepseek.py

@@ -4,6 +4,7 @@
 
 import re
 from dataclasses import dataclass
+from functools import partial
 from typing import Literal
 
 import typer
@@ -103,7 +104,7 @@ def _prepare_cp(args: ScriptArgs):
     )
 
 
-def _execute_train(args: ScriptArgs):
+def _execute_train(args: ScriptArgs, before_ray_job_submit=None):
     load_save_path = f"{args.output_dir}/{args.run_id}/checkpoints"
     ckpt_args = (
         f"--hf-checkpoint {args.model_local_dir}/{args.model_name} "
@@ -304,6 +305,7 @@ def _execute_train(args: ScriptArgs):
         megatron_model_type=args.megatron_model_type,
         extra_env_vars={**sglang_extra_env_vars},
         megatron_path=args.megatron_path,
+        before_ray_job_submit=before_ray_job_submit,
     )
 
 
@@ -312,9 +314,12 @@ def _execute_train(args: ScriptArgs):
 def train(args: ScriptArgs):
     _prepare_download(args)
     _prepare_bf16_ckpt(args)
+    _execute_train(args, before_ray_job_submit=partial(_prepare_ray_dependent, args))
+
+
+def _prepare_ray_dependent(args: ScriptArgs):
     _prepare_megatron_ckpt(args)
     _prepare_cp(args)
-    _execute_train(args)
 
 
 @app.callback()

scripts/run_deepseek_v32.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from functools import partial
 from pathlib import Path
 from typing import Literal
 
@@ -166,7 +167,7 @@ def _prepare_cp(args: ScriptArgs, skip_existing: bool = False):
         )
 
 
-def _execute_train(args: ScriptArgs):
+def _execute_train(args: ScriptArgs, before_ray_job_submit=None):
     ref_load_path = f"{args.model_dir}/{args.model_name}_torch_dist"
     load_save_path = f"{args.output_dir}/{args.run_id}/checkpoints"
 
@@ -443,6 +444,7 @@ def _execute_train(args: ScriptArgs):
         megatron_model_type=args.megatron_model_type,
         extra_env_vars={**misc_env_vars},
         megatron_path=args.megatron_path,
+        before_ray_job_submit=before_ray_job_submit,
     )
 
 
@@ -454,9 +456,7 @@ def full_train(args: ScriptArgs):
     _prepare_bf16_ckpt(args)
     _prepare_mxfp8_ckpt(args)
     _prepare_fp8_ckpt(args)
-    _prepare_megatron_ckpt(args)
-    # _prepare_cp(args, skip_existing=True)
-    _execute_train(args)
+    _execute_train(args, before_ray_job_submit=partial(_prepare_megatron_ckpt, args))
 
 
 @app.command()

scripts/run_glm45_355b_a32b.py

@@ -21,7 +21,7 @@ class ScriptArgs(U.ExecuteTrainConfig):
     model_name: str = "GLM-4.5"
     megatron_model_type: str = "glm4.5-355B-A32B"
     num_gpus_per_node: int = 4
-    hardware: Literal["H100", "GB200", "GB300"] = "H100"
+    hardware: Literal["H100", "GB200", "GB300"] = "GB200"
     enable_eval: bool = True
     extra_args: str = ""
     data_dir: str = "/root/datasets"

scripts/run_glm47_flash.py

@@ -72,7 +72,7 @@ def execute(args: ScriptArgs):
     eval_args = ""
     if (args.mode != "debug_minimal") and args.enable_eval:
         eval_args += (
-            # "--eval-interval 20 "
+            "--eval-interval 20 "
             f"--eval-prompt-data aime24 {args.data_dir}/aime-2024/aime-2024.jsonl "
             "--n-samples-per-eval-prompt 16 "
             "--eval-max-response-len 16384 "

scripts/run_qwen3_30b_a3b.py

@@ -413,6 +413,7 @@ def execute(args: ScriptArgs):
 
     U.execute_train(
         train_args=train_args,
+        config=args,
         num_gpus_per_node=args.num_gpus_per_node,
         megatron_model_type=args.megatron_model_type,
         extra_env_vars={**misc_env_vars},

tests/e2e/megatron/test_mimo_7B_mtp_only_grad.py

@@ -1,3 +1,4 @@
+# FIXME
 """End-to-end test for MTP-only gradient verification.
 
 This test verifies that when MTP training is enabled and all outputs are truncated

tests/fast/launch_scripts/py_harness.py

@@ -155,11 +155,17 @@ def exists(self: Path, **kwargs: object) -> bool:
 def call_entrypoint(module: ModuleType, name: str, overrides: dict[str, object], sandbox: Path) -> None:
     entrypoint = getattr(module, name)
     first = next(iter(inspect.signature(entrypoint).parameters.values()), None)
-    with host_filesystem_frozen(sandbox):
-        if first is not None and first.name == "args":
-            entrypoint(module.ScriptArgs(**overrides))
-        else:
-            entrypoint(**overrides)
+    saved_env = dict(os.environ)
+    try:
+        with host_filesystem_frozen(sandbox):
+            if first is not None and first.name == "args":
+                entrypoint(module.ScriptArgs(**overrides))
+            else:
+                entrypoint(**overrides)
+    finally:
+        # a leaked knob would make later recordings depend on which launcher ran first
+        os.environ.clear()
+        os.environ.update(saved_env)
 
 
 def format_recording(recording: Recording, sandbox: Path) -> str:

tests/manual/launch_scripts/test_py_launch_scripts.py

@@ -54,7 +54,6 @@ def _nemotron_checkpoint(sandbox: Path) -> dict[str, object]:
 
 _SCRIPTS_WHOSE_DEFAULTS_ARE_UNSUPPORTED: dict[str, Callable[[Path], dict[str, object]]] = {
     "scripts/run_deepseek_v4.py": lambda sandbox: {"model_name": "DeepSeek-V4-Flash-FP8-4layer"},
-    "scripts/run_glm45_355b_a32b.py": lambda sandbox: {"hardware": "GB200"},
     "scripts/run_glm5_744b_a40b.py": lambda sandbox: _glm_checkpoint(sandbox, "GLM-5", 78),
     "scripts/run_glm5_2_744b_a40b.py": lambda sandbox: _glm_checkpoint(sandbox, "GLM-5.2", 78),
     "scripts/run_inkling.py": lambda sandbox: {"model_name": "Inkling-4layer"},

tests/snapshots/launch_scripts/py/scripts/run_deepseek.py/train.txt

@@ -21,6 +21,17 @@ python <REPO_ROOT>/tools/fp8_cast_bf16.py
   --output-bf16-hf-path /root/models/DeepSeek-V3-bf16/ 
 
 ### 5
+pkill -9 sglang; sleep 3; ray stop
+  --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; 
+
+### 6
+export PYTHONUNBUFFERED=1 && ray start
+  --head
+  --node-ip-address 127.0.0.1
+  --num-gpus 4
+  --disable-usage-stats
+
+### 7
 [multi_node num_nodes=None] PYTHONPATH=<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath torchrun
   --nproc-per-node 4
   --master-addr {{master_addr}}
@@ -79,25 +90,14 @@ python <REPO_ROOT>/tools/fp8_cast_bf16.py
   --decoder-first-pipeline-num-layers 7
   --decoder-last-pipeline-num-layers 6 
 
-### 6
+### 8
 [multi_node num_nodes=None] mkdir -p /root/local_data/DeepSeek-V3_torch_dist && rsync -a
   --info=progress2 /root/models/DeepSeek-V3_torch_dist/ /root/local_data/DeepSeek-V3_torch_dist
 
-### 7
+### 9
 [multi_node num_nodes=None] mkdir -p /root/local_data/DeepSeek-V3 && rsync -a
   --info=progress2 /root/models/DeepSeek-V3/ /root/local_data/DeepSeek-V3
 
-### 8
-pkill -9 sglang; sleep 3; ray stop
-  --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; 
-
-### 9
-export PYTHONUNBUFFERED=1 && ray start
-  --head
-  --node-ip-address 127.0.0.1
-  --num-gpus 4
-  --disable-usage-stats
-
 ### 10
 nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l
 

tests/snapshots/launch_scripts/py/scripts/run_deepseek_v32.py/full_train.txt

@@ -21,6 +21,17 @@ python <REPO_ROOT>/tools/fp8_cast_bf16.py
   --output-bf16-hf-path /root/models/DeepSeek-V3.2-bf16/ 
 
 ### 5
+pkill -9 sglang; sleep 3; ray stop
+  --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; 
+
+### 6
+export PYTHONUNBUFFERED=1 && ray start
+  --head
+  --node-ip-address 127.0.0.1
+  --num-gpus 8
+  --disable-usage-stats
+
+### 7
 [multi_node num_nodes=None] PYTHONPATH=<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath torchrun
   --nproc-per-node 8
   --master-addr {{master_addr}}
@@ -79,17 +90,6 @@ python <REPO_ROOT>/tools/fp8_cast_bf16.py
   --decoder-last-pipeline-num-layers 13
   --expert-tensor-parallel-size 1 
 
-### 6
-pkill -9 sglang; sleep 3; ray stop
-  --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; 
-
-### 7
-export PYTHONUNBUFFERED=1 && ray start
-  --head
-  --node-ip-address 127.0.0.1
-  --num-gpus 8
-  --disable-usage-stats
-
 ### 8
 nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l
 

tests/snapshots/launch_scripts/py/scripts/run_glm47_flash.py/execute.txt

@@ -107,6 +107,7 @@ export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit
   --recompute-num-layers 1
   --use-dynamic-batch-size
   --max-tokens-per-gpu 32768 
+  --eval-interval 20
   --eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl
   --n-samples-per-eval-prompt 16
   --eval-max-response-len 16384