diff --git a/tests/fast/launch_scripts/py_harness.py b/tests/fast/launch_scripts/py_harness.py index 44785cc5d02..5c2597a5dd2 100644 --- a/tests/fast/launch_scripts/py_harness.py +++ b/tests/fast/launch_scripts/py_harness.py @@ -1,8 +1,11 @@ import ast import importlib.util import inspect +import os import re +import subprocess import sys +import time from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass @@ -16,6 +19,10 @@ FROZEN_RUN_ID = "260101-000000-000" +_GPU_COUNT_ANY_WAIT_LOOP_ACCEPTS = "1000000" +_FROZEN_PID = 1000 +_FROZEN_PPID = 1001 + _FROZEN_ENV = { "MASTER_ADDR": "127.0.0.1", "MILES_SCRIPT_ENABLE_RAY_SUBMIT": "1", @@ -61,6 +68,38 @@ def iter_py_launch_scripts() -> list[PyLaunchScript]: return [PyLaunchScript(path=path, entrypoints=tuple(_entrypoint_names(path))) for path in paths] +def iter_self_executing_launchers() -> list[Path]: + """Launchers that reach the shell themselves rather than through command_utils.""" + roots = [REPO_ROOT / root for root in ("scripts", "examples", "tools")] + convention = {script.path for script in iter_py_launch_scripts()} + return sorted( + path + for root in roots + for path in root.rglob("*.py") + if path not in convention and "ray job submit" in path.read_text(errors="replace") + ) + + +def install_shell_recorder(monkeypatch, sandbox: Path) -> Recording: + """A launcher holding its own subprocess handle never touches the recorded command_utils helpers.""" + recording = Recording(commands=[], pseudo_files=[]) + + def fake_run(command, *args, **kwargs): + recording.commands.append(command if isinstance(command, str) else " ".join(command)) + return subprocess.CompletedProcess( + args=command, returncode=0, stdout=_GPU_COUNT_ANY_WAIT_LOOP_ACCEPTS, stderr="" + ) + + monkeypatch.setenv("MILES_LOG_DIR", str(sandbox)) + monkeypatch.setattr(subprocess, "run", fake_run) + monkeypatch.setattr(time, "sleep", lambda seconds: None) + monkeypatch.setattr(os, "makedirs", lambda path, **kwargs: None) + monkeypatch.setattr(os, "getpid", lambda: _FROZEN_PID) + monkeypatch.setattr(os, "getppid", lambda: _FROZEN_PPID) + + return recording + + def freeze_environment(monkeypatch) -> None: for key, value in _FROZEN_ENV.items(): monkeypatch.setenv(key, value) diff --git a/tests/fast/launch_scripts/test_self_executing_launchers.py b/tests/fast/launch_scripts/test_self_executing_launchers.py new file mode 100644 index 00000000000..ec50b9396d9 --- /dev/null +++ b/tests/fast/launch_scripts/test_self_executing_launchers.py @@ -0,0 +1,110 @@ +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +from tests.fast.launch_scripts.py_harness import ( + format_recording, + freeze_environment, + import_launch_script, + install_shell_recorder, + iter_self_executing_launchers, +) +from tests.fast.launch_scripts.sh_harness import REPO_ROOT, assert_matches_snapshot + +_SNAPSHOT_DIR = REPO_ROOT / "tests" / "snapshots" / "launch_scripts" / "self_executing" + +_P2P = "examples/infra_features/p2p_weight_transfer/run.py" +_FORMAL_MATH = "examples/experimental/formal_math/single_round/run_minimal.py" + + +@dataclass(frozen=True) +class LauncherCase: + rel: str + name: str + entrypoint: str | None = None + kwargs: dict[str, object] = field(default_factory=dict) + + +_P2P_PROFILES = ( + "GLM-4.5-Air", + "GLM-4.7-Flash", + "GLM-5", + "GLM-5_20layer", + "GLM-5_4layer", + "GLM-Z1-9B-0414", + "Kimi-K2-Instruct", + "Moonlight-16B-A3B-Instruct", + "Qwen3-235B-A22B-Instruct-2507", + "Qwen3-30B-A3B", + "Qwen3-4B", +) + +_CASES = [ + LauncherCase( + rel=_P2P, + name=f"run/{profile}/{mode}", + entrypoint="cmd_run", + kwargs={"model_name": profile, "mode": mode, "node_rank": 0, "head_ip": "10.0.0.1"}, + ) + for profile in _P2P_PROFILES + for mode in ("p2p", "broadcast") +] + [LauncherCase(rel=_FORMAL_MATH, name="import")] + +_ENTRYPOINTS_THE_HARNESS_CANNOT_SANDBOX = {(_P2P, "cmd_prepare")} + + +@pytest.fixture(params=_CASES, ids=[f"{case.rel}::{case.name}" for case in _CASES]) +def recorded(request, monkeypatch, tmp_path): + case = request.param + freeze_environment(monkeypatch) + monkeypatch.setenv("SKIP_VALIDATION", "1") + recording = install_shell_recorder(monkeypatch, sandbox=tmp_path) + module = import_launch_script(REPO_ROOT / case.rel) + if case.entrypoint is not None: + getattr(module, case.entrypoint)(**case.kwargs) + return case, recording, tmp_path + + +class TestEverySelfExecutingLauncher: + def test_commands_match_snapshot(self, recorded): + """These launchers build their whole command line by hand, so only a snapshot pins it.""" + case, recording, sandbox = recorded + snapshot = _SNAPSHOT_DIR / case.rel / f"{case.name}.txt" + + assert_matches_snapshot(snapshot, format_recording(recording, sandbox=sandbox), f"{case.rel}::{case.name}") + + def test_reruns_produce_identical_recordings(self, recorded, monkeypatch, tmp_path): + """These launchers embed their own pid, so a snapshot is only stable if the harness freezes it.""" + case, recording, _ = recorded + freeze_environment(monkeypatch) + monkeypatch.setenv("SKIP_VALIDATION", "1") + again = install_shell_recorder(monkeypatch, sandbox=tmp_path) + module = import_launch_script(REPO_ROOT / case.rel) + if case.entrypoint is not None: + getattr(module, case.entrypoint)(**case.kwargs) + + assert again.commands == recording.commands + + def test_the_launcher_submits_a_ray_job(self, recorded): + """A launcher that stops reaching `ray job submit` is broken, whatever else it records.""" + _, recording, _ = recorded + + assert [command for command in recording.commands if "ray job submit" in command] + + +class TestDiscovery: + def test_every_self_executing_launcher_has_at_least_one_case(self): + """Discovery is by behaviour, not by path, so a new hand-rolled launcher shows up here.""" + discovered = {path.relative_to(REPO_ROOT).as_posix() for path in iter_self_executing_launchers()} + + assert discovered == {case.rel for case in _CASES} + + def test_the_uncovered_entrypoint_is_named_and_still_uncoverable(self): + """cmd_prepare rewrites a checkout under a hardcoded /root/models, which no fixture can redirect.""" + module = import_launch_script(REPO_ROOT / _P2P) + + assert {(_P2P, name) for name in ("cmd_run", "cmd_prepare")} - { + (case.rel, case.entrypoint) for case in _CASES + } == _ENTRYPOINTS_THE_HARNESS_CANNOT_SANDBOX + assert '"/root/models"' in Path(module.__file__).read_text() diff --git a/tests/snapshots/launch_scripts/self_executing/examples/experimental/formal_math/single_round/run_minimal.py/import.txt b/tests/snapshots/launch_scripts/self_executing/examples/experimental/formal_math/single_round/run_minimal.py/import.txt new file mode 100644 index 00000000000..ad94fb21e00 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/experimental/formal_math/single_round/run_minimal.py/import.txt @@ -0,0 +1,68 @@ +### 0 +bash -c export PYTHONUNBUFFERED=1 && source "/scripts/models/qwen3-8B.sh" && ray job submit + --address="http://127.0.0.1:8265" + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1"}}' + -- python3 train.py ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Qwen3-8B/ + --ref-load /root/models/Qwen3-8B_torch_dist + --save-interval 20 + --load /root/Qwen3-8B_miles + --save /root/Qwen3-8B_miles + --prompt-data /root/datasets/formal_math_single_round/minimal_demo/flc_train.jsonl + --input-key prompt + --apply-chat-template + --rollout-shuffle + --custom-rm-path examples.experimental.formal_math.single_round.reward_fn.reward_fn + --reward-key reward_value + --log-reward-category reward_cat + --rollout-batch-size 32 + --n-samples-per-prompt 8 + --rollout-max-response-len 8192 + --rollout-temperature 1 + --global-batch-size 256 + --balance-data + --num-rollout 3000 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --use-wandb + --wandb-project miles-formal-math-run-minimal + --wandb-group demo + --wandb-key 'frozen-wandb-api-key' + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 1 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 6144 + --eval-interval 20 + --n-samples-per-eval-prompt 1 + --eval-max-response-len 16384 + --eval-top-p 1 + --eval-prompt-data minif2f /root/datasets/formal_math_single_round/minimal_demo/minif2f_test.jsonl + --rollout-num-gpus-per-engine 8 + --sglang-mem-fraction-static 0.7 + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --colocate + --log-passrate diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.5-Air/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.5-Air/broadcast.txt new file mode 100644 index 00000000000..0985581a519 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.5-Air/broadcast.txt @@ -0,0 +1,106 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm4.5-106B-A12B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300", "MODEL_ARGS_ROTARY_BASE": "1000000"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-4.5-Air + --ref-load /root/GLM-4.5-Air_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 16 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 1 + --sequence-parallel + --pipeline-model-parallel-size 4 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 10 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator gspo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 4e-4 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 32 + --sglang-mem-fraction-static 0.8 + --sglang-ep-size 8 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 4 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 4294967296 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.5-Air/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.5-Air/p2p.txt new file mode 100644 index 00000000000..83b71704059 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.5-Air/p2p.txt @@ -0,0 +1,107 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm4.5-106B-A12B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300", "MODEL_ARGS_ROTARY_BASE": "1000000"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-4.5-Air + --ref-load /root/GLM-4.5-Air_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 16 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 1 + --sequence-parallel + --pipeline-model-parallel-size 4 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 10 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator gspo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 4e-4 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 32 + --sglang-mem-fraction-static 0.8 + --sglang-ep-size 8 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 4 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.7-Flash/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.7-Flash/broadcast.txt new file mode 100644 index 00000000000..433f05bf261 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.7-Flash/broadcast.txt @@ -0,0 +1,108 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm4.7-flash.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-4.7-Flash + --ref-load /root/multinode/GLM-4.7-Flash_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-temperature 0.6 + --eval-top-p 0.95 + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 4 + --rollout-num-gpus 8 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 4 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.7-Flash/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.7-Flash/p2p.txt new file mode 100644 index 00000000000..fd2c92f05f4 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.7-Flash/p2p.txt @@ -0,0 +1,109 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm4.7-flash.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-4.7-Flash + --ref-load /root/multinode/GLM-4.7-Flash_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --eval-prompt-data aime24 /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-temperature 0.6 + --eval-top-p 0.95 + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 4 + --rollout-num-gpus 8 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 4 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5/broadcast.txt new file mode 100644 index 00000000000..878aae279ca --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5/broadcast.txt @@ -0,0 +1,118 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm5-744B-A40B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "600", "INDEXER_ROPE_NEOX_STYLE": "0", "NVSHMEM_DISABLE_NCCL": "1"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-5 + --ref-load /root/GLM-5_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 8 + --context-parallel-size 2 + --expert-model-parallel-size 16 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 8 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 256 + --data-pad-size-multiplier 4096 + --log-probs-chunk-size 1024 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 64 + --rollout-num-gpus 128 + --sglang-mem-fraction-static 0.9 + --sglang-ep-size 64 + --sglang-dp-size 64 + --sglang-moe-dense-tp-size 1 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-page-size 64 + --sglang-nsa-decode-backend flashmla_sparse + --sglang-nsa-prefill-backend flashmla_sparse + --sglang-attention-backend nsa + --sglang-cuda-graph-max-bs 8 + --sglang-max-running-requests 512 + --sglang-chunked-prefill-size 131072 + --sglang-watchdog-timeout 3600 + --sglang-disable-cuda-graph '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --allgather-cp + --moe-token-dispatcher-type alltoall + --actor-num-nodes 16 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5/p2p.txt new file mode 100644 index 00000000000..84ba31d3922 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5/p2p.txt @@ -0,0 +1,119 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm5-744B-A40B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "600", "INDEXER_ROPE_NEOX_STYLE": "0", "NVSHMEM_DISABLE_NCCL": "1"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-5 + --ref-load /root/GLM-5_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 8 + --context-parallel-size 2 + --expert-model-parallel-size 16 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 8 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 256 + --data-pad-size-multiplier 4096 + --log-probs-chunk-size 1024 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 64 + --rollout-num-gpus 128 + --sglang-mem-fraction-static 0.9 + --sglang-ep-size 64 + --sglang-dp-size 64 + --sglang-moe-dense-tp-size 1 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-page-size 64 + --sglang-nsa-decode-backend flashmla_sparse + --sglang-nsa-prefill-backend flashmla_sparse + --sglang-attention-backend nsa + --sglang-cuda-graph-max-bs 8 + --sglang-max-running-requests 512 + --sglang-chunked-prefill-size 131072 + --sglang-watchdog-timeout 3600 + --sglang-disable-cuda-graph + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --allgather-cp + --moe-token-dispatcher-type alltoall + --actor-num-nodes 16 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_20layer/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_20layer/broadcast.txt new file mode 100644 index 00000000000..f2e0345b5d0 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_20layer/broadcast.txt @@ -0,0 +1,118 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm5-744B-A40B_20layer.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "600", "INDEXER_ROPE_NEOX_STYLE": "0", "NVSHMEM_DISABLE_NCCL": "1"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-5_20layer + --ref-load /root/GLM-5_20layer_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 3 + --context-parallel-size 1 + --expert-model-parallel-size 16 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 6 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 1024 + --data-pad-size-multiplier 4096 + --log-probs-chunk-size 1024 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 16 + --rollout-num-gpus 48 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 16 + --sglang-dp-size 16 + --sglang-moe-dense-tp-size 1 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-page-size 64 + --sglang-nsa-decode-backend flashmla_sparse + --sglang-nsa-prefill-backend flashmla_sparse + --sglang-attention-backend nsa + --sglang-cuda-graph-max-bs 8 + --sglang-max-running-requests 512 + --sglang-chunked-prefill-size 32768 + --sglang-watchdog-timeout 3600 + --sglang-disable-cuda-graph '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --allgather-cp + --moe-token-dispatcher-type alltoall + --actor-num-nodes 6 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_20layer/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_20layer/p2p.txt new file mode 100644 index 00000000000..060b1c407c5 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_20layer/p2p.txt @@ -0,0 +1,119 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm5-744B-A40B_20layer.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "600", "INDEXER_ROPE_NEOX_STYLE": "0", "NVSHMEM_DISABLE_NCCL": "1"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-5_20layer + --ref-load /root/GLM-5_20layer_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 3 + --context-parallel-size 1 + --expert-model-parallel-size 16 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 6 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 1024 + --data-pad-size-multiplier 4096 + --log-probs-chunk-size 1024 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 16 + --rollout-num-gpus 48 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 16 + --sglang-dp-size 16 + --sglang-moe-dense-tp-size 1 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-page-size 64 + --sglang-nsa-decode-backend flashmla_sparse + --sglang-nsa-prefill-backend flashmla_sparse + --sglang-attention-backend nsa + --sglang-cuda-graph-max-bs 8 + --sglang-max-running-requests 512 + --sglang-chunked-prefill-size 32768 + --sglang-watchdog-timeout 3600 + --sglang-disable-cuda-graph + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --allgather-cp + --moe-token-dispatcher-type alltoall + --actor-num-nodes 6 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_4layer/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_4layer/broadcast.txt new file mode 100644 index 00000000000..76fdfe325cd --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_4layer/broadcast.txt @@ -0,0 +1,114 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm5-744B-A40B_4layer.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "600", "INDEXER_ROPE_NEOX_STYLE": "0", "NVSHMEM_DISABLE_NCCL": "1"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-5_4layer + --ref-load /root/GLM-5_4layer_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --data-pad-size-multiplier 4096 + --log-probs-chunk-size 1024 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 8 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 8 + --sglang-dp-size 8 + --sglang-moe-dense-tp-size 1 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-page-size 64 + --sglang-nsa-decode-backend flashmla_sparse + --sglang-nsa-prefill-backend flashmla_sparse + --sglang-attention-backend nsa + --sglang-cuda-graph-max-bs 8 + --sglang-max-running-requests 512 + --sglang-chunked-prefill-size 16384 + --sglang-watchdog-timeout 3600 + --sglang-disable-cuda-graph '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --allgather-cp + --moe-token-dispatcher-type alltoall + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_4layer/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_4layer/p2p.txt new file mode 100644 index 00000000000..d1012211f08 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-5_4layer/p2p.txt @@ -0,0 +1,115 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/glm5-744B-A40B_4layer.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "600", "INDEXER_ROPE_NEOX_STYLE": "0", "NVSHMEM_DISABLE_NCCL": "1"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-5_4layer + --ref-load /root/GLM-5_4layer_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --data-pad-size-multiplier 4096 + --log-probs-chunk-size 1024 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 8 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 8 + --sglang-dp-size 8 + --sglang-moe-dense-tp-size 1 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-page-size 64 + --sglang-nsa-decode-backend flashmla_sparse + --sglang-nsa-prefill-backend flashmla_sparse + --sglang-attention-backend nsa + --sglang-cuda-graph-max-bs 8 + --sglang-max-running-requests 512 + --sglang-chunked-prefill-size 16384 + --sglang-watchdog-timeout 3600 + --sglang-disable-cuda-graph + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --allgather-cp + --moe-token-dispatcher-type alltoall + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-Z1-9B-0414/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-Z1-9B-0414/broadcast.txt new file mode 100644 index 00000000000..560206d4205 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-Z1-9B-0414/broadcast.txt @@ -0,0 +1,92 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +ray start + --head + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +source "/scripts/models/glm4-9B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": ""}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-Z1-9B-0414 + --ref-load /root/multinode/GLM-Z1-9B-0414_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 32 + --balance-data + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 2 + --expert-model-parallel-size 1 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 2 + --rollout-num-gpus 4 + --sglang-mem-fraction-static 0.8 '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 4 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-Z1-9B-0414/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-Z1-9B-0414/p2p.txt new file mode 100644 index 00000000000..cb2a9a61cbc --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-Z1-9B-0414/p2p.txt @@ -0,0 +1,93 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +ray start + --head + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +source "/scripts/models/glm4-9B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": ""}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/GLM-Z1-9B-0414 + --ref-load /root/multinode/GLM-Z1-9B-0414_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 32 + --balance-data + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 2 + --expert-model-parallel-size 1 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 2 + --rollout-num-gpus 4 + --sglang-mem-fraction-static 0.8 + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 4 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Kimi-K2-Instruct/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Kimi-K2-Instruct/broadcast.txt new file mode 100644 index 00000000000..115505f9e0f --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Kimi-K2-Instruct/broadcast.txt @@ -0,0 +1,114 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/kimi-k2.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Kimi-K2-Instruct + --ref-load /root/multinode/Kimi-K2-Instruct_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 64 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 8 + --sequence-parallel + --pipeline-model-parallel-size 8 + --context-parallel-size 4 + --expert-model-parallel-size 32 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 5 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 16384 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 32 + --rollout-num-gpus 256 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 32 + --sglang-dp-size 8 + --sglang-moe-dense-tp-size 1 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-server-concurrency 1024 + --sglang-moe-runner-backend triton + --sglang-fp8-gemm-backend triton '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --moe-token-dispatcher-type alltoall + --actor-num-nodes 32 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Kimi-K2-Instruct/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Kimi-K2-Instruct/p2p.txt new file mode 100644 index 00000000000..52de614a9da --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Kimi-K2-Instruct/p2p.txt @@ -0,0 +1,115 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/kimi-k2.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Kimi-K2-Instruct + --ref-load /root/multinode/Kimi-K2-Instruct_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 64 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 8 + --sequence-parallel + --pipeline-model-parallel-size 8 + --context-parallel-size 4 + --expert-model-parallel-size 32 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 5 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 16384 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 32 + --rollout-num-gpus 256 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 32 + --sglang-dp-size 8 + --sglang-moe-dense-tp-size 1 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-server-concurrency 1024 + --sglang-moe-runner-backend triton + --sglang-fp8-gemm-backend triton + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --moe-token-dispatcher-type alltoall + --actor-num-nodes 32 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Moonlight-16B-A3B-Instruct/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Moonlight-16B-A3B-Instruct/broadcast.txt new file mode 100644 index 00000000000..ee975b6212c --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Moonlight-16B-A3B-Instruct/broadcast.txt @@ -0,0 +1,100 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/moonlight.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Moonlight-16B-A3B-Instruct + --ref-load /root/multinode/Moonlight-16B-A3B-Instruct_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 8 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 8 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Moonlight-16B-A3B-Instruct/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Moonlight-16B-A3B-Instruct/p2p.txt new file mode 100644 index 00000000000..5886226820a --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Moonlight-16B-A3B-Instruct/p2p.txt @@ -0,0 +1,101 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/moonlight.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Moonlight-16B-A3B-Instruct + --ref-load /root/multinode/Moonlight-16B-A3B-Instruct_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 1.0 + --global-batch-size 16 + --balance-data + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --use-kl-loss + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 8 + --sglang-mem-fraction-static 0.7 + --sglang-ep-size 8 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-235B-A22B-Instruct-2507/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-235B-A22B-Instruct-2507/broadcast.txt new file mode 100644 index 00000000000..63455c46129 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-235B-A22B-Instruct-2507/broadcast.txt @@ -0,0 +1,107 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/qwen3-235B-A22B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300", "MODEL_ARGS_ROTARY_BASE": "5000000"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Qwen3-235B-A22B-Instruct-2507 + --ref-load /root/multinode/Qwen3-235B-A22B-Instruct-2507_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 64 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 4 + --context-parallel-size 2 + --expert-model-parallel-size 16 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 22 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 8192 + --advantage-estimator gspo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 4e-4 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 32 + --rollout-num-gpus 64 + --sglang-mem-fraction-static 0.75 + --sglang-ep-size 32 + --sglang-dp-size 1 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 8 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-235B-A22B-Instruct-2507/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-235B-A22B-Instruct-2507/p2p.txt new file mode 100644 index 00000000000..0812884fa38 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-235B-A22B-Instruct-2507/p2p.txt @@ -0,0 +1,108 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/qwen3-235B-A22B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300", "MODEL_ARGS_ROTARY_BASE": "5000000"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Qwen3-235B-A22B-Instruct-2507 + --ref-load /root/multinode/Qwen3-235B-A22B-Instruct-2507_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 64 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 4 + --context-parallel-size 2 + --expert-model-parallel-size 16 + --expert-tensor-parallel-size 1 + --decoder-last-pipeline-num-layers 22 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 8192 + --advantage-estimator gspo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 4e-4 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 32 + --rollout-num-gpus 64 + --sglang-mem-fraction-static 0.75 + --sglang-ep-size 32 + --sglang-dp-size 1 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 8 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-30B-A3B/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-30B-A3B/broadcast.txt new file mode 100644 index 00000000000..62e0f4baf30 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-30B-A3B/broadcast.txt @@ -0,0 +1,105 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/qwen3-30B-A3B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300", "MODEL_ARGS_ROTARY_BASE": "1000000"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Qwen3-30B-A3B + --ref-load /root/multinode/Qwen3-30B-A3B_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 16 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator gspo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 4e-4 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 16 + --sglang-mem-fraction-static 0.8 + --sglang-ep-size 8 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 2 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-30B-A3B/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-30B-A3B/p2p.txt new file mode 100644 index 00000000000..45a3b63863c --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-30B-A3B/p2p.txt @@ -0,0 +1,106 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +RAY_memory_monitor_refresh_ms=0 ray start + --head + --node-ip-address 10.0.0.1 + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()" + +### 11 +source "/scripts/models/qwen3-30B-A3B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1", "MILES_LOG_DIR": "", "MC_TRANSFER_TIMEOUT": "300", "MODEL_ARGS_ROTARY_BASE": "1000000"}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Qwen3-30B-A3B + --ref-load /root/multinode/Qwen3-30B-A3B_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 4 + --n-samples-per-prompt 4 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 16 + --balance-data + --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl + --n-samples-per-eval-prompt 16 + --eval-max-response-len 16384 + --eval-top-p 0.7 + --tensor-model-parallel-size 4 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator gspo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 4e-4 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + --rollout-num-gpus-per-engine 8 + --rollout-num-gpus 16 + --sglang-mem-fraction-static 0.8 + --sglang-ep-size 8 + --sglang-cuda-graph-bs 1 2 4 8 16 + --sglang-enable-dp-attention + --sglang-enable-dp-lm-head + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 2 + --actor-num-gpus-per-node 8 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-4B/broadcast.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-4B/broadcast.txt new file mode 100644 index 00000000000..40fc6151875 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-4B/broadcast.txt @@ -0,0 +1,91 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +ray start + --head + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +source "/scripts/models/qwen3-4B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": ""}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Qwen3-4B + --ref-load /root/multinode/Qwen3-4B_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 32 + --balance-data + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 2 + --expert-model-parallel-size 1 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 2 + --rollout-num-gpus 4 + --sglang-mem-fraction-static 0.8 '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 4 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode broadcast diff --git a/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-4B/p2p.txt b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-4B/p2p.txt new file mode 100644 index 00000000000..c89b79aaf14 --- /dev/null +++ b/tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/Qwen3-4B/p2p.txt @@ -0,0 +1,92 @@ +### 0 +pkill -9 sglang || true + +### 1 +sleep 3 + +### 2 +ray stop + --force || true + +### 3 +pkill -9 ray || true + +### 4 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 5 +sleep 3 + +### 6 +pkill -9 ray || true + +### 7 +pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true + +### 8 +pkill -9 redis || true + +### 9 +ray start + --head + --num-gpus 8 + --disable-usage-stats + --dashboard-host=0.0.0.0 + --dashboard-port=8265 + +### 10 +source "/scripts/models/qwen3-4B.sh" && ray job submit + --address='http://127.0.0.1:8265' + --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": ""}}' + -- python3 "/train.py" ${MODEL_ARGS[@]} + --hf-checkpoint /root/models/Qwen3-4B + --ref-load /root/multinode/Qwen3-4B_torch_dist + --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout 13 + --rollout-batch-size 8 + --n-samples-per-prompt 8 + --rollout-max-response-len 100 + --rollout-temperature 0.8 + --global-batch-size 32 + --balance-data + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 2 + --expert-model-parallel-size 1 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 + --advantage-estimator grpo + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --entropy-coef 0.00 + --eps-clip 0.2 + --eps-clip-high 0.28 + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 + --rollout-num-gpus-per-engine 2 + --rollout-num-gpus 4 + --sglang-mem-fraction-static 0.8 + --sglang-remote-instance-weight-loader-start-seed-via-transfer-engine '--sglang-load-format dummy' + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --actor-num-nodes 1 + --actor-num-gpus-per-node 4 + --update-weight-buffer-size 1073741824 + --update-weight-transfer-mode p2p