Snapshot the launchers that build their own command line

tests/fast/launch_scripts/py_harness.py

@@ -1,8 +1,11 @@
 import ast
 import importlib.util
 import inspect
+import os
 import re
+import subprocess
 import sys
+import time
 from collections.abc import Iterator
 from contextlib import contextmanager
 from dataclasses import dataclass
@@ -16,6 +19,10 @@
 
 FROZEN_RUN_ID = "260101-000000-000"
 
+_GPU_COUNT_ANY_WAIT_LOOP_ACCEPTS = "1000000"
+_FROZEN_PID = 1000
+_FROZEN_PPID = 1001
+
 _FROZEN_ENV = {
     "MASTER_ADDR": "127.0.0.1",
     "MILES_SCRIPT_ENABLE_RAY_SUBMIT": "1",
@@ -61,6 +68,38 @@ def iter_py_launch_scripts() -> list[PyLaunchScript]:
     return [PyLaunchScript(path=path, entrypoints=tuple(_entrypoint_names(path))) for path in paths]
 
 
+def iter_self_executing_launchers() -> list[Path]:
+    """Launchers that reach the shell themselves rather than through command_utils."""
+    roots = [REPO_ROOT / root for root in ("scripts", "examples", "tools")]
+    convention = {script.path for script in iter_py_launch_scripts()}
+    return sorted(
+        path
+        for root in roots
+        for path in root.rglob("*.py")
+        if path not in convention and "ray job submit" in path.read_text(errors="replace")
+    )
+
+
+def install_shell_recorder(monkeypatch, sandbox: Path) -> Recording:
+    """A launcher holding its own subprocess handle never touches the recorded command_utils helpers."""
+    recording = Recording(commands=[], pseudo_files=[])
+
+    def fake_run(command, *args, **kwargs):
+        recording.commands.append(command if isinstance(command, str) else " ".join(command))
+        return subprocess.CompletedProcess(
+            args=command, returncode=0, stdout=_GPU_COUNT_ANY_WAIT_LOOP_ACCEPTS, stderr=""
+        )
+
+    monkeypatch.setenv("MILES_LOG_DIR", str(sandbox))
+    monkeypatch.setattr(subprocess, "run", fake_run)
+    monkeypatch.setattr(time, "sleep", lambda seconds: None)
+    monkeypatch.setattr(os, "makedirs", lambda path, **kwargs: None)
+    monkeypatch.setattr(os, "getpid", lambda: _FROZEN_PID)
+    monkeypatch.setattr(os, "getppid", lambda: _FROZEN_PPID)
+
+    return recording
+
+
 def freeze_environment(monkeypatch) -> None:
     for key, value in _FROZEN_ENV.items():
         monkeypatch.setenv(key, value)

tests/fast/launch_scripts/test_self_executing_launchers.py

@@ -0,0 +1,110 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import pytest
+
+from tests.fast.launch_scripts.py_harness import (
+    format_recording,
+    freeze_environment,
+    import_launch_script,
+    install_shell_recorder,
+    iter_self_executing_launchers,
+)
+from tests.fast.launch_scripts.sh_harness import REPO_ROOT, assert_matches_snapshot
+
+_SNAPSHOT_DIR = REPO_ROOT / "tests" / "snapshots" / "launch_scripts" / "self_executing"
+
+_P2P = "examples/infra_features/p2p_weight_transfer/run.py"
+_FORMAL_MATH = "examples/experimental/formal_math/single_round/run_minimal.py"
+
+
+@dataclass(frozen=True)
+class LauncherCase:
+    rel: str
+    name: str
+    entrypoint: str | None = None
+    kwargs: dict[str, object] = field(default_factory=dict)
+
+
+_P2P_PROFILES = (
+    "GLM-4.5-Air",
+    "GLM-4.7-Flash",
+    "GLM-5",
+    "GLM-5_20layer",
+    "GLM-5_4layer",
+    "GLM-Z1-9B-0414",
+    "Kimi-K2-Instruct",
+    "Moonlight-16B-A3B-Instruct",
+    "Qwen3-235B-A22B-Instruct-2507",
+    "Qwen3-30B-A3B",
+    "Qwen3-4B",
+)
+
+_CASES = [
+    LauncherCase(
+        rel=_P2P,
+        name=f"run/{profile}/{mode}",
+        entrypoint="cmd_run",
+        kwargs={"model_name": profile, "mode": mode, "node_rank": 0, "head_ip": "10.0.0.1"},
+    )
+    for profile in _P2P_PROFILES
+    for mode in ("p2p", "broadcast")
+] + [LauncherCase(rel=_FORMAL_MATH, name="import")]
+
+_ENTRYPOINTS_THE_HARNESS_CANNOT_SANDBOX = {(_P2P, "cmd_prepare")}
+
+
+@pytest.fixture(params=_CASES, ids=[f"{case.rel}::{case.name}" for case in _CASES])
+def recorded(request, monkeypatch, tmp_path):
+    case = request.param
+    freeze_environment(monkeypatch)
+    monkeypatch.setenv("SKIP_VALIDATION", "1")
+    recording = install_shell_recorder(monkeypatch, sandbox=tmp_path)
+    module = import_launch_script(REPO_ROOT / case.rel)
+    if case.entrypoint is not None:
+        getattr(module, case.entrypoint)(**case.kwargs)
+    return case, recording, tmp_path
+
+
+class TestEverySelfExecutingLauncher:
+    def test_commands_match_snapshot(self, recorded):
+        """These launchers build their whole command line by hand, so only a snapshot pins it."""
+        case, recording, sandbox = recorded
+        snapshot = _SNAPSHOT_DIR / case.rel / f"{case.name}.txt"
+
+        assert_matches_snapshot(snapshot, format_recording(recording, sandbox=sandbox), f"{case.rel}::{case.name}")
+
+    def test_reruns_produce_identical_recordings(self, recorded, monkeypatch, tmp_path):
+        """These launchers embed their own pid, so a snapshot is only stable if the harness freezes it."""
+        case, recording, _ = recorded
+        freeze_environment(monkeypatch)
+        monkeypatch.setenv("SKIP_VALIDATION", "1")
+        again = install_shell_recorder(monkeypatch, sandbox=tmp_path)
+        module = import_launch_script(REPO_ROOT / case.rel)
+        if case.entrypoint is not None:
+            getattr(module, case.entrypoint)(**case.kwargs)
+
+        assert again.commands == recording.commands
+
+    def test_the_launcher_submits_a_ray_job(self, recorded):
+        """A launcher that stops reaching `ray job submit` is broken, whatever else it records."""
+        _, recording, _ = recorded
+
+        assert [command for command in recording.commands if "ray job submit" in command]
+
+
+class TestDiscovery:
+    def test_every_self_executing_launcher_has_at_least_one_case(self):
+        """Discovery is by behaviour, not by path, so a new hand-rolled launcher shows up here."""
+        discovered = {path.relative_to(REPO_ROOT).as_posix() for path in iter_self_executing_launchers()}
+
+        assert discovered == {case.rel for case in _CASES}
+
+    def test_the_uncovered_entrypoint_is_named_and_still_uncoverable(self):
+        """cmd_prepare rewrites a checkout under a hardcoded /root/models, which no fixture can redirect."""
+        module = import_launch_script(REPO_ROOT / _P2P)
+
+        assert {(_P2P, name) for name in ("cmd_run", "cmd_prepare")} - {
+            (case.rel, case.entrypoint) for case in _CASES
+        } == _ENTRYPOINTS_THE_HARNESS_CANNOT_SANDBOX
+        assert '"/root/models"' in Path(module.__file__).read_text()

tests/snapshots/launch_scripts/self_executing/examples/experimental/formal_math/single_round/run_minimal.py/import.txt

@@ -0,0 +1,68 @@
+### 0
+bash -c export PYTHONUNBUFFERED=1 && source "<REPO_ROOT>/scripts/models/qwen3-8B.sh" && ray job submit
+  --address="http://127.0.0.1:8265"
+  --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "1"}}'
+  -- python3 train.py ${MODEL_ARGS[@]}
+  --hf-checkpoint /root/models/Qwen3-8B/
+  --ref-load /root/models/Qwen3-8B_torch_dist
+  --save-interval 20
+  --load /root/Qwen3-8B_miles
+  --save /root/Qwen3-8B_miles 
+  --prompt-data /root/datasets/formal_math_single_round/minimal_demo/flc_train.jsonl
+  --input-key prompt
+  --apply-chat-template
+  --rollout-shuffle
+  --custom-rm-path examples.experimental.formal_math.single_round.reward_fn.reward_fn
+  --reward-key reward_value
+  --log-reward-category reward_cat
+  --rollout-batch-size 32
+  --n-samples-per-prompt 8
+  --rollout-max-response-len 8192
+  --rollout-temperature 1
+  --global-batch-size 256
+  --balance-data
+  --num-rollout 3000 
+  --optimizer adam
+  --lr 1e-6
+  --lr-decay-style constant
+  --weight-decay 0.1
+  --adam-beta1 0.9
+  --adam-beta2 0.98 
+  --advantage-estimator grpo
+  --use-kl-loss
+  --kl-loss-coef 0.00
+  --kl-loss-type low_var_kl
+  --entropy-coef 0.00
+  --eps-clip 0.2
+  --eps-clip-high 0.28 
+  --use-wandb
+  --wandb-project miles-formal-math-run-minimal
+  --wandb-group demo
+  --wandb-key 'frozen-wandb-api-key' 
+  --tensor-model-parallel-size 2
+  --sequence-parallel
+  --pipeline-model-parallel-size 1
+  --context-parallel-size 1
+  --expert-model-parallel-size 1
+  --expert-tensor-parallel-size 1
+  --recompute-granularity full
+  --recompute-method uniform
+  --recompute-num-layers 1
+  --use-dynamic-batch-size
+  --max-tokens-per-gpu 6144 
+  --eval-interval 20
+  --n-samples-per-eval-prompt 1
+  --eval-max-response-len 16384
+  --eval-top-p 1
+  --eval-prompt-data minif2f /root/datasets/formal_math_single_round/minimal_demo/minif2f_test.jsonl 
+  --rollout-num-gpus-per-engine 8
+  --sglang-mem-fraction-static 0.7 
+  --attention-dropout 0.0
+  --hidden-dropout 0.0
+  --accumulate-allreduce-grads-in-fp32
+  --attention-softmax-in-fp32
+  --attention-backend flash
+  --actor-num-nodes 1
+  --actor-num-gpus-per-node 8
+  --colocate
+  --log-passrate  

tests/snapshots/launch_scripts/self_executing/examples/infra_features/p2p_weight_transfer/run.py/run/GLM-4.5-Air/broadcast.txt

@@ -0,0 +1,106 @@
+### 0
+pkill -9 sglang || true
+
+### 1
+sleep 3
+
+### 2
+ray stop
+  --force || true
+
+### 3
+pkill -9 ray || true
+
+### 4
+pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true
+
+### 5
+sleep 3
+
+### 6
+pkill -9 ray || true
+
+### 7
+pgrep -x 'python|python3' | grep -v -w 1000 | grep -v -w 1001 | xargs -r kill -9 || true
+
+### 8
+pkill -9 redis || true
+
+### 9
+RAY_memory_monitor_refresh_ms=0 ray start
+  --head
+  --node-ip-address 10.0.0.1
+  --num-gpus 8
+  --disable-usage-stats
+  --dashboard-host=0.0.0.0
+  --dashboard-port=8265
+
+### 10
+python3 -c "import ray; ray.init(address='auto', ignore_reinit_error=True); print(int(ray.cluster_resources().get('GPU', 0))); ray.shutdown()"
+
+### 11
+source "<REPO_ROOT>/scripts/models/glm4.5-106B-A12B.sh" && ray job submit
+  --address='http://127.0.0.1:8265'
+  --runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "RAY_DEBUG": "1", "PYTHONPATH": "/root/Megatron-LM/", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "MILES_LOG_DIR": "<SANDBOX>", "MC_TRANSFER_TIMEOUT": "300", "MODEL_ARGS_ROTARY_BASE": "1000000"}}'
+  -- python3 "<REPO_ROOT>/train.py" ${MODEL_ARGS[@]}
+  --hf-checkpoint /root/models/GLM-4.5-Air
+  --ref-load /root/GLM-4.5-Air_torch_dist
+  --prompt-data /root/datasets/dapo-math-17k/dapo-math-17k.jsonl
+  --input-key prompt
+  --label-key label
+  --apply-chat-template
+  --rollout-shuffle
+  --rm-type deepscaler
+  --num-rollout 13
+  --rollout-batch-size 4
+  --n-samples-per-prompt 4
+  --rollout-max-response-len 100
+  --rollout-temperature 0.8
+  --global-batch-size 16
+  --balance-data
+  --eval-prompt-data aime /root/datasets/aime-2024/aime-2024.jsonl
+  --n-samples-per-eval-prompt 16
+  --eval-max-response-len 16384
+  --eval-top-p 0.7
+  --tensor-model-parallel-size 1
+  --sequence-parallel
+  --pipeline-model-parallel-size 4
+  --context-parallel-size 1
+  --expert-model-parallel-size 8
+  --expert-tensor-parallel-size 1
+  --decoder-last-pipeline-num-layers 10
+  --recompute-granularity full
+  --recompute-method uniform
+  --recompute-num-layers 1
+  --use-dynamic-batch-size
+  --max-tokens-per-gpu 2048
+  --advantage-estimator gspo
+  --kl-loss-coef 0.00
+  --kl-loss-type low_var_kl
+  --entropy-coef 0.00
+  --eps-clip 4e-4
+  --optimizer adam
+  --lr 1e-6
+  --lr-decay-style constant
+  --weight-decay 0.1
+  --adam-beta1 0.9
+  --adam-beta2 0.98
+  --optimizer-cpu-offload
+  --overlap-cpu-optimizer-d2h-h2d
+  --use-precision-aware-optimizer
+  --rollout-num-gpus-per-engine 8
+  --rollout-num-gpus 32
+  --sglang-mem-fraction-static 0.8
+  --sglang-ep-size 8
+  --sglang-cuda-graph-bs 1 2 4 8 16
+  --sglang-enable-dp-attention
+  --sglang-enable-dp-lm-head '--sglang-load-format dummy'
+  --attention-dropout 0.0
+  --hidden-dropout 0.0
+  --accumulate-allreduce-grads-in-fp32
+  --attention-softmax-in-fp32
+  --attention-backend flash
+  --actor-num-nodes 4
+  --actor-num-gpus-per-node 8
+  --update-weight-buffer-size 4294967296
+  --update-weight-transfer-mode broadcast