diff --git a/docs/models/deepseek/deepseek.md b/docs/models/deepseek/deepseek.md index cbf2f0b66a3..08a0d05e62a 100644 --- a/docs/models/deepseek/deepseek.md +++ b/docs/models/deepseek/deepseek.md @@ -45,15 +45,25 @@ dynamic sampling and eval. ### Multi-node fan-out -The launcher starts the Ray head on the invoking node only. Join every other node to the cluster -first: +The `torch_dist` conversion and the node-local rsync fan out across every node of the Ray +cluster, so a multi-node run needs the whole cluster joined **before** the launcher starts, and +the launcher must be told not to replace it. Bring up the head, join the workers, then run with +`MILES_SCRIPT_EXTERNAL_RAY=1`: ```bash +# on node 0 +ray start --head --node-ip-address ${MASTER_ADDR} --num-gpus 8 --disable-usage-stats + +# on every other node ray start --address=${MASTER_ADDR}:6379 --num-gpus 8 \ --node-ip-address ${WORKER_IP} --disable-usage-stats + +# back on node 0 +MILES_SCRIPT_EXTERNAL_RAY=1 python scripts/run_deepseek.py train \ + --num-nodes 16 --num-gpus-per-node 8 ``` -With an MPI-style hostfile (each line `ip slot=8`), fan out from node 0: +With an MPI-style hostfile (each line `ip slot=8`), fan the workers out from node 0: ```bash for WORKER_IP in $(awk '{print $1}' $BASE_DIR/mpi_hostfile); do @@ -68,6 +78,10 @@ done wait ``` +Without `MILES_SCRIPT_EXTERNAL_RAY=1` the launcher runs `ray stop --force` and starts its own +single-node head, which is what the single-node invocation above relies on — the conversion then +runs on that one node. + ## 4. Checkpoint conversion `train` performs the two conversion steps for you; the equivalent manual commands are below. diff --git a/miles/utils/external_utils/command_utils.py b/miles/utils/external_utils/command_utils.py index cae5a492637..a0ea8f9cd9a 100644 --- a/miles/utils/external_utils/command_utils.py +++ b/miles/utils/external_utils/command_utils.py @@ -169,7 +169,8 @@ def execute_train( "CUDA_DEVICE_MAX_CONNECTIONS": "1", } ), - "NCCL_NVLS_ENABLE": os.environ.get("NCCL_NVLS_ENABLE", str(int(check_has_nvlink()))), + # a get() default is evaluated eagerly, which would probe even when already decided + "NCCL_NVLS_ENABLE": os.environ.get("NCCL_NVLS_ENABLE") or str(int(check_has_nvlink())), **{ k: os.environ[k] for k in ("NCCL_SOCKET_IFNAME", "GLOO_SOCKET_IFNAME", "NCCL_DEBUG", "NCCL_DEBUG_FILE") diff --git a/scripts/amd/run_qwen3_30b_a3b.py b/scripts/amd/run_qwen3_30b_a3b.py index e270eeab022..8c35ea0df18 100644 --- a/scripts/amd/run_qwen3_30b_a3b.py +++ b/scripts/amd/run_qwen3_30b_a3b.py @@ -222,6 +222,7 @@ def execute(args: ScriptArgs): U.execute_train( train_args=train_args, + config=args, num_gpus_per_node=args.num_gpus_per_node, megatron_model_type=args.megatron_model_type, extra_env_vars={**misc_env_vars}, diff --git a/scripts/run-glm4.5-355B-A32B.sh b/scripts/run-glm4.5-355B-A32B.sh index dc0c1a69441..ff7b6dbaff4 100644 --- a/scripts/run-glm4.5-355B-A32B.sh +++ b/scripts/run-glm4.5-355B-A32B.sh @@ -15,13 +15,6 @@ set -ex # will prevent ray from buffering stdout/stderr export PYTHONUNBUFFERED=1 -NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) -if [ "$NVLINK_COUNT" -gt 0 ]; then - HAS_NVLINK=1 -else - HAS_NVLINK=0 -fi -echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)" SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" MODEL_ARGS_LINE="$(python3 "${SCRIPT_DIR}/../miles/utils/external_utils/model_args_utils.py" "glm4.5-355B-A32B")" || exit 1 @@ -189,7 +182,6 @@ ray job submit --address="http://127.0.0.1:8265" \ --actor-num-nodes 8 \ --actor-num-gpus-per-node 8 \ --colocate \ - --save-debug-rollout-data /mnt/zhuzilin/github-miles/data.pt \ ${MODEL_ARGS[@]} \ ${CKPT_ARGS[@]} \ ${ROLLOUT_ARGS[@]} \ diff --git a/scripts/run-gpt-oss-20b-bf16.sh b/scripts/run-gpt-oss-20b-bf16.sh index 4cf823ee1b4..5600d1fb84e 100644 --- a/scripts/run-gpt-oss-20b-bf16.sh +++ b/scripts/run-gpt-oss-20b-bf16.sh @@ -16,6 +16,14 @@ set -ex export PYTHONUNBUFFERED=1 export HF_HOME=/workspace/hf_cache +NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) +if [ "$NVLINK_COUNT" -gt 0 ]; then + HAS_NVLINK=1 +else + HAS_NVLINK=0 +fi +echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)" + # Load model architecture config SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" MODEL_ARGS_LINE="$(python3 "${SCRIPT_DIR}/../miles/utils/external_utils/model_args_utils.py" "gpt-oss-20b")" || exit 1 @@ -143,6 +151,5 @@ ray job submit --address="http://127.0.0.1:8265" \ ${GRPO_ARGS[@]} \ ${WANDB_ARGS[@]} \ ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ ${SGLANG_ARGS[@]} \ ${MISC_ARGS[@]} diff --git a/scripts/run-kimi-k25.sh b/scripts/run-kimi-k25.sh index 0f796e66031..3b8e182f92c 100755 --- a/scripts/run-kimi-k25.sh +++ b/scripts/run-kimi-k25.sh @@ -52,7 +52,6 @@ ROLLOUT_ARGS=( --rollout-temperature 1 --global-batch-size 256 - --filter-zero-reward-samples --use-dynamic-global-batch-size ) diff --git a/scripts/run-qwen3-235B-A22B-sft.sh b/scripts/run-qwen3-235B-A22B-sft.sh index 3233241189c..44a2e6828e3 100644 --- a/scripts/run-qwen3-235B-A22B-sft.sh +++ b/scripts/run-qwen3-235B-A22B-sft.sh @@ -147,5 +147,4 @@ ray job submit --address="http://127.0.0.1:8265" \ ${OPTIMIZER_ARGS[@]} \ ${WANDB_ARGS[@]} \ ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ ${MISC_ARGS[@]} diff --git a/scripts/run-qwen3-4B-base-sft.sh b/scripts/run-qwen3-4B-base-sft.sh index e8acefd9ea4..9bba25b3eca 100644 --- a/scripts/run-qwen3-4B-base-sft.sh +++ b/scripts/run-qwen3-4B-base-sft.sh @@ -123,5 +123,4 @@ ray job submit --address="http://127.0.0.1:8265" \ ${OPTIMIZER_ARGS[@]} \ ${WANDB_ARGS[@]} \ ${PERF_ARGS[@]} \ - ${EVAL_ARGS[@]} \ ${MISC_ARGS[@]} diff --git a/scripts/run-qwen3-4B_4xgpu.sh b/scripts/run-qwen3-4B_4xgpu.sh index 54a268e657b..f94f12b5ffe 100755 --- a/scripts/run-qwen3-4B_4xgpu.sh +++ b/scripts/run-qwen3-4B_4xgpu.sh @@ -100,12 +100,15 @@ OPTIMIZER_ARGS=( --adam-beta2 0.98 ) -WANDB_ARGS=( - --use-wandb - --wandb-project miles-dev-qwen3-radix - --wandb-group qwen3-4B-4xgpu - --wandb-key ${WANDB_KEY} -) +WANDB_ARGS=() +if [ -n "${WANDB_KEY:-}" ]; then + WANDB_ARGS=( + --use-wandb + --wandb-project miles-dev-qwen3-radix + --wandb-group qwen3-4B-4xgpu + --wandb-key "${WANDB_KEY}" + ) +fi SGLANG_ARGS=( --rollout-num-gpus-per-engine 2 @@ -140,6 +143,7 @@ ray job submit --address="http://127.0.0.1:8265" \ -- python3 train.py \ --actor-num-nodes 1 \ --actor-num-gpus-per-node 4 \ + --num-gpus-per-node 4 \ --colocate \ ${MODEL_ARGS[@]} \ ${CKPT_ARGS[@]} \ diff --git a/scripts/run-qwen3-next-80B-A3B.sh b/scripts/run-qwen3-next-80B-A3B.sh index 6973de821f2..2b5e086865c 100644 --- a/scripts/run-qwen3-next-80B-A3B.sh +++ b/scripts/run-qwen3-next-80B-A3B.sh @@ -166,7 +166,7 @@ RUNTIME_ENV_JSON="{ \"env_vars\": { \"PYTHONPATH\": \"/root/Megatron-LM/\", \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\", - \"NCCL_NVLS_ENABLE\": \"${HAS_NVLINK}\" + \"NCCL_NVLS_ENABLE\": \"${HAS_NVLINK}\", \"no_proxy\": \"${no_proxy}\", \"MASTER_ADDR\": \"${MASTER_ADDR}\" } diff --git a/scripts/run_deepseek.py b/scripts/run_deepseek.py index 693068116b2..8f055a21c80 100644 --- a/scripts/run_deepseek.py +++ b/scripts/run_deepseek.py @@ -4,6 +4,7 @@ import re from dataclasses import dataclass +from functools import partial from typing import Literal import typer @@ -103,7 +104,7 @@ def _prepare_cp(args: ScriptArgs): ) -def _execute_train(args: ScriptArgs): +def _execute_train(args: ScriptArgs, before_ray_job_submit=None): load_save_path = f"{args.output_dir}/{args.run_id}/checkpoints" ckpt_args = ( f"--hf-checkpoint {args.model_local_dir}/{args.model_name} " @@ -304,6 +305,7 @@ def _execute_train(args: ScriptArgs): megatron_model_type=args.megatron_model_type, extra_env_vars={**sglang_extra_env_vars}, megatron_path=args.megatron_path, + before_ray_job_submit=before_ray_job_submit, ) @@ -312,9 +314,12 @@ def _execute_train(args: ScriptArgs): def train(args: ScriptArgs): _prepare_download(args) _prepare_bf16_ckpt(args) + _execute_train(args, before_ray_job_submit=partial(_prepare_ray_dependent, args)) + + +def _prepare_ray_dependent(args: ScriptArgs): _prepare_megatron_ckpt(args) _prepare_cp(args) - _execute_train(args) @app.callback() diff --git a/scripts/run_deepseek_v32.py b/scripts/run_deepseek_v32.py index 76a2adf4642..26e923f6522 100644 --- a/scripts/run_deepseek_v32.py +++ b/scripts/run_deepseek_v32.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from functools import partial from pathlib import Path from typing import Literal @@ -166,7 +167,7 @@ def _prepare_cp(args: ScriptArgs, skip_existing: bool = False): ) -def _execute_train(args: ScriptArgs): +def _execute_train(args: ScriptArgs, before_ray_job_submit=None): ref_load_path = f"{args.model_dir}/{args.model_name}_torch_dist" load_save_path = f"{args.output_dir}/{args.run_id}/checkpoints" @@ -443,6 +444,7 @@ def _execute_train(args: ScriptArgs): megatron_model_type=args.megatron_model_type, extra_env_vars={**misc_env_vars}, megatron_path=args.megatron_path, + before_ray_job_submit=before_ray_job_submit, ) @@ -454,9 +456,7 @@ def full_train(args: ScriptArgs): _prepare_bf16_ckpt(args) _prepare_mxfp8_ckpt(args) _prepare_fp8_ckpt(args) - _prepare_megatron_ckpt(args) - # _prepare_cp(args, skip_existing=True) - _execute_train(args) + _execute_train(args, before_ray_job_submit=partial(_prepare_megatron_ckpt, args)) @app.command() diff --git a/scripts/run_glm45_355b_a32b.py b/scripts/run_glm45_355b_a32b.py index c85f9faffd3..6e85fa1f750 100644 --- a/scripts/run_glm45_355b_a32b.py +++ b/scripts/run_glm45_355b_a32b.py @@ -21,7 +21,7 @@ class ScriptArgs(U.ExecuteTrainConfig): model_name: str = "GLM-4.5" megatron_model_type: str = "glm4.5-355B-A32B" num_gpus_per_node: int = 4 - hardware: Literal["H100", "GB200", "GB300"] = "H100" + hardware: Literal["H100", "GB200", "GB300"] = "GB200" enable_eval: bool = True extra_args: str = "" data_dir: str = "/root/datasets" diff --git a/scripts/run_glm47_flash.py b/scripts/run_glm47_flash.py index 673b2513dee..c3b2cb2e766 100644 --- a/scripts/run_glm47_flash.py +++ b/scripts/run_glm47_flash.py @@ -72,7 +72,7 @@ def execute(args: ScriptArgs): eval_args = "" if (args.mode != "debug_minimal") and args.enable_eval: eval_args += ( - # "--eval-interval 20 " + "--eval-interval 20 " f"--eval-prompt-data aime24 {args.data_dir}/aime-2024/aime-2024.jsonl " "--n-samples-per-eval-prompt 16 " "--eval-max-response-len 16384 " diff --git a/scripts/run_qwen3_30b_a3b.py b/scripts/run_qwen3_30b_a3b.py index 1a23d8d64aa..5098f7fb4e6 100644 --- a/scripts/run_qwen3_30b_a3b.py +++ b/scripts/run_qwen3_30b_a3b.py @@ -413,6 +413,7 @@ def execute(args: ScriptArgs): U.execute_train( train_args=train_args, + config=args, num_gpus_per_node=args.num_gpus_per_node, megatron_model_type=args.megatron_model_type, extra_env_vars={**misc_env_vars}, diff --git a/tests/e2e/megatron/test_mimo_7B_mtp_only_grad.py b/tests/e2e/megatron/test_mimo_7B_mtp_only_grad.py index 9568953d960..01d63369889 100644 --- a/tests/e2e/megatron/test_mimo_7B_mtp_only_grad.py +++ b/tests/e2e/megatron/test_mimo_7B_mtp_only_grad.py @@ -1,3 +1,4 @@ +# FIXME """End-to-end test for MTP-only gradient verification. This test verifies that when MTP training is enabled and all outputs are truncated diff --git a/tests/fast/launch_scripts/py_harness.py b/tests/fast/launch_scripts/py_harness.py index 9690989669c..03811c3451f 100644 --- a/tests/fast/launch_scripts/py_harness.py +++ b/tests/fast/launch_scripts/py_harness.py @@ -155,11 +155,17 @@ def exists(self: Path, **kwargs: object) -> bool: def call_entrypoint(module: ModuleType, name: str, overrides: dict[str, object], sandbox: Path) -> None: entrypoint = getattr(module, name) first = next(iter(inspect.signature(entrypoint).parameters.values()), None) - with host_filesystem_frozen(sandbox): - if first is not None and first.name == "args": - entrypoint(module.ScriptArgs(**overrides)) - else: - entrypoint(**overrides) + saved_env = dict(os.environ) + try: + with host_filesystem_frozen(sandbox): + if first is not None and first.name == "args": + entrypoint(module.ScriptArgs(**overrides)) + else: + entrypoint(**overrides) + finally: + # a leaked knob would make later recordings depend on which launcher ran first + os.environ.clear() + os.environ.update(saved_env) def format_recording(recording: Recording, sandbox: Path) -> str: diff --git a/tests/manual/launch_scripts/test_py_launch_scripts.py b/tests/manual/launch_scripts/test_py_launch_scripts.py index 59bbbb72ef3..dab2a3c1433 100644 --- a/tests/manual/launch_scripts/test_py_launch_scripts.py +++ b/tests/manual/launch_scripts/test_py_launch_scripts.py @@ -54,7 +54,6 @@ def _nemotron_checkpoint(sandbox: Path) -> dict[str, object]: _SCRIPTS_WHOSE_DEFAULTS_ARE_UNSUPPORTED: dict[str, Callable[[Path], dict[str, object]]] = { "scripts/run_deepseek_v4.py": lambda sandbox: {"model_name": "DeepSeek-V4-Flash-FP8-4layer"}, - "scripts/run_glm45_355b_a32b.py": lambda sandbox: {"hardware": "GB200"}, "scripts/run_glm5_744b_a40b.py": lambda sandbox: _glm_checkpoint(sandbox, "GLM-5", 78), "scripts/run_glm5_2_744b_a40b.py": lambda sandbox: _glm_checkpoint(sandbox, "GLM-5.2", 78), "scripts/run_inkling.py": lambda sandbox: {"model_name": "Inkling-4layer"}, diff --git a/tests/snapshots/launch_scripts/py/scripts/run_deepseek.py/train.txt b/tests/snapshots/launch_scripts/py/scripts/run_deepseek.py/train.txt index 3f47ebe2308..1d6d9499cc3 100644 --- a/tests/snapshots/launch_scripts/py/scripts/run_deepseek.py/train.txt +++ b/tests/snapshots/launch_scripts/py/scripts/run_deepseek.py/train.txt @@ -21,6 +21,17 @@ python /tools/fp8_cast_bf16.py --output-bf16-hf-path /root/models/DeepSeek-V3-bf16/ ### 5 +pkill -9 sglang; sleep 3; ray stop + --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; + +### 6 +export PYTHONUNBUFFERED=1 && ray start + --head + --node-ip-address 127.0.0.1 + --num-gpus 4 + --disable-usage-stats + +### 7 [multi_node num_nodes=None] PYTHONPATH=:/root/Megatron-LM:/frozen/pythonpath torchrun --nproc-per-node 4 --master-addr {{master_addr}} @@ -79,25 +90,14 @@ python /tools/fp8_cast_bf16.py --decoder-first-pipeline-num-layers 7 --decoder-last-pipeline-num-layers 6 -### 6 +### 8 [multi_node num_nodes=None] mkdir -p /root/local_data/DeepSeek-V3_torch_dist && rsync -a --info=progress2 /root/models/DeepSeek-V3_torch_dist/ /root/local_data/DeepSeek-V3_torch_dist -### 7 +### 9 [multi_node num_nodes=None] mkdir -p /root/local_data/DeepSeek-V3 && rsync -a --info=progress2 /root/models/DeepSeek-V3/ /root/local_data/DeepSeek-V3 -### 8 -pkill -9 sglang; sleep 3; ray stop - --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; - -### 9 -export PYTHONUNBUFFERED=1 && ray start - --head - --node-ip-address 127.0.0.1 - --num-gpus 4 - --disable-usage-stats - ### 10 nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l diff --git a/tests/snapshots/launch_scripts/py/scripts/run_deepseek_v32.py/full_train.txt b/tests/snapshots/launch_scripts/py/scripts/run_deepseek_v32.py/full_train.txt index 1f7c0df46fc..f3a7d85fae6 100644 --- a/tests/snapshots/launch_scripts/py/scripts/run_deepseek_v32.py/full_train.txt +++ b/tests/snapshots/launch_scripts/py/scripts/run_deepseek_v32.py/full_train.txt @@ -21,6 +21,17 @@ python /tools/fp8_cast_bf16.py --output-bf16-hf-path /root/models/DeepSeek-V3.2-bf16/ ### 5 +pkill -9 sglang; sleep 3; ray stop + --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; + +### 6 +export PYTHONUNBUFFERED=1 && ray start + --head + --node-ip-address 127.0.0.1 + --num-gpus 8 + --disable-usage-stats + +### 7 [multi_node num_nodes=None] PYTHONPATH=:/root/Megatron-LM:/frozen/pythonpath torchrun --nproc-per-node 8 --master-addr {{master_addr}} @@ -79,17 +90,6 @@ python /tools/fp8_cast_bf16.py --decoder-last-pipeline-num-layers 13 --expert-tensor-parallel-size 1 -### 6 -pkill -9 sglang; sleep 3; ray stop - --force; pkill -9 ray; pkill -9 miles; sleep 3; pkill -9 ray; pkill -9 miles; pkill -9 redis; true; - -### 7 -export PYTHONUNBUFFERED=1 && ray start - --head - --node-ip-address 127.0.0.1 - --num-gpus 8 - --disable-usage-stats - ### 8 nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l diff --git a/tests/snapshots/launch_scripts/py/scripts/run_glm47_flash.py/execute.txt b/tests/snapshots/launch_scripts/py/scripts/run_glm47_flash.py/execute.txt index 1099658c7c3..934c4e364b7 100644 --- a/tests/snapshots/launch_scripts/py/scripts/run_glm47_flash.py/execute.txt +++ b/tests/snapshots/launch_scripts/py/scripts/run_glm47_flash.py/execute.txt @@ -107,6 +107,7 @@ export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit --recompute-num-layers 1 --use-dynamic-batch-size --max-tokens-per-gpu 32768 + --eval-interval 20 --eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl --n-samples-per-eval-prompt 16 --eval-max-response-len 16384 diff --git a/tests/snapshots/launch_scripts/sh/scripts/run-glm4.5-355B-A32B.sh.txt b/tests/snapshots/launch_scripts/sh/scripts/run-glm4.5-355B-A32B.sh.txt index c7af53a0b7c..c4a08f76629 100644 --- a/tests/snapshots/launch_scripts/sh/scripts/run-glm4.5-355B-A32B.sh.txt +++ b/tests/snapshots/launch_scripts/sh/scripts/run-glm4.5-355B-A32B.sh.txt @@ -39,16 +39,11 @@ "python" ### 8 -"nvidia-smi" -"topo" -"-m" - -### 9 "python3" "/scripts/../miles/utils/external_utils/model_args_utils.py" "glm4.5-355B-A32B" -### 10 +### 9 "ray" "start" "--head" @@ -59,12 +54,12 @@ "--dashboard-host=0.0.0.0" "--dashboard-port=8265" -### 11 +### 10 "awk" "{print $1}" "/root/mpi_rack_hostfile" -### 12 +### 11 "ray" "job" "submit" @@ -78,8 +73,6 @@ "--actor-num-gpus-per-node" "8" "--colocate" -"--save-debug-rollout-data" -"/mnt/zhuzilin/github-miles/data.pt" "--disable-bias-linear" "--qk-layernorm" "--group-query-attention" diff --git a/tests/snapshots/launch_scripts/sh/scripts/run-gpt-oss-20b-bf16.sh.txt b/tests/snapshots/launch_scripts/sh/scripts/run-gpt-oss-20b-bf16.sh.txt index 093da50e7a6..8d334ab8d72 100644 --- a/tests/snapshots/launch_scripts/sh/scripts/run-gpt-oss-20b-bf16.sh.txt +++ b/tests/snapshots/launch_scripts/sh/scripts/run-gpt-oss-20b-bf16.sh.txt @@ -39,11 +39,16 @@ "python" ### 8 +"nvidia-smi" +"topo" +"-m" + +### 9 "python3" "/scripts/../miles/utils/external_utils/model_args_utils.py" "gpt-oss-20b" -### 9 +### 10 "ray" "start" "--head" @@ -55,12 +60,12 @@ "--dashboard-host=0.0.0.0" "--dashboard-port=8265" -### 10 +### 11 "ray" "job" "submit" "--address=http://127.0.0.1:8265" -"--runtime-env-json={\n \"env_vars\": {\n \"PYTHONPATH\": \"/root/Megatron-LM/\",\n \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\",\n \"NCCL_NVLS_ENABLE\": \"\"\n }\n}" +"--runtime-env-json={\n \"env_vars\": {\n \"PYTHONPATH\": \"/root/Megatron-LM/\",\n \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\",\n \"NCCL_NVLS_ENABLE\": \"0\"\n }\n}" "--" "python3" "train.py" diff --git a/tests/snapshots/launch_scripts/sh/scripts/run-kimi-k25.sh.txt b/tests/snapshots/launch_scripts/sh/scripts/run-kimi-k25.sh.txt index c7546661d87..afdd3676089 100644 --- a/tests/snapshots/launch_scripts/sh/scripts/run-kimi-k25.sh.txt +++ b/tests/snapshots/launch_scripts/sh/scripts/run-kimi-k25.sh.txt @@ -178,7 +178,6 @@ "1" "--global-batch-size" "256" -"--filter-zero-reward-samples" "--use-dynamic-global-batch-size" "--optimizer" "adam" diff --git a/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-4B_4xgpu.sh.txt b/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-4B_4xgpu.sh.txt index 0ac3ee9d6ad..74f219b66b6 100644 --- a/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-4B_4xgpu.sh.txt +++ b/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-4B_4xgpu.sh.txt @@ -71,6 +71,8 @@ "1" "--actor-num-gpus-per-node" "4" +"--num-gpus-per-node" +"4" "--colocate" "--swiglu" "--num-layers" diff --git a/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-next-80B-A3B.sh.txt b/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-next-80B-A3B.sh.txt index d579cbfe6a5..c81b0c3dbd7 100644 --- a/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-next-80B-A3B.sh.txt +++ b/tests/snapshots/launch_scripts/sh/scripts/run-qwen3-next-80B-A3B.sh.txt @@ -70,7 +70,7 @@ "job" "submit" "--address=http://127.0.0.1:8265" -"--runtime-env-json={\n \"env_vars\": {\n \"PYTHONPATH\": \"/root/Megatron-LM/\",\n \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\",\n \"NCCL_NVLS_ENABLE\": \"0\"\n \"no_proxy\": \"127.0.0.1,127.0.0.1\",\n \"MASTER_ADDR\": \"127.0.0.1\"\n }\n}" +"--runtime-env-json={\n \"env_vars\": {\n \"PYTHONPATH\": \"/root/Megatron-LM/\",\n \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\",\n \"NCCL_NVLS_ENABLE\": \"0\",\n \"no_proxy\": \"127.0.0.1,127.0.0.1\",\n \"MASTER_ADDR\": \"127.0.0.1\"\n }\n}" "--" "python3" "train.py"