From 4db930eb42834bfacf249c46056955eaaa0d98be Mon Sep 17 00:00:00 2001 From: Shi Dong Date: Tue, 28 Jul 2026 14:08:55 -0700 Subject: [PATCH 1/6] examples: add swe-agent-daytona (Harbor sandboxes on Daytona) Adds a SWE-agent example whose task sandboxes run on Daytona cloud sandboxes instead of local Docker, so the trainer host needs no Docker daemon and no local task-image builds. The agent is terminus-2, which runs as a host process and appends both user and tool turns, so the session server needs --tito-allowed-append-roles user tool. That flag is the reason this needs its own launcher rather than reusing examples/swe-agent/run.py. generate.py and swe_agent_function.py are unchanged for Daytona-backed runs, so run.py puts the sibling example on PYTHONPATH instead of duplicating them. --- examples/README.md | 1 + examples/swe-agent-daytona/README.md | 186 ++++++++++++ .../swe-agent-daytona/launch_agent_server.sh | 40 +++ examples/swe-agent-daytona/run.py | 284 ++++++++++++++++++ 4 files changed, 511 insertions(+) create mode 100644 examples/swe-agent-daytona/README.md create mode 100755 examples/swe-agent-daytona/launch_agent_server.sh create mode 100644 examples/swe-agent-daytona/run.py diff --git a/examples/README.md b/examples/README.md index b0eb986e8fe..a7bd86decd9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,6 +19,7 @@ These examples provide concrete examples to leverage miles in your own RL workfl - **[search-r1](./search-r1)**: A minimal reproduction of Search-R1, featuring multi-turn conversation and tool-calling. - **[strands-agents](./strands-agents)**: Integration example with the Strands-Agents scaffolding framework. - **[swe-agent](./swe-agent)**: Trains coding and terminal agents with Harbor-managed sandboxes and verifier rewards. +- **[swe-agent-daytona](./swe-agent-daytona)**: The same Harbor pipeline with task sandboxes hosted on Daytona instead of local Docker. - **[tau-bench](./tau-bench)**: Training in an agentic multi-turn tool use environment (Tau-bench). - **[train_infer_mismatch_helper](./train_infer_mismatch_helper)**: Algorithmic methods for rollout correction (e.g., TIS, MIS). - **[true_on_policy](./true_on_policy)**: Ensures strictly equal log probabilities between inference (SGLang) and training engines. diff --git a/examples/swe-agent-daytona/README.md b/examples/swe-agent-daytona/README.md new file mode 100644 index 00000000000..5b854ac5cb1 --- /dev/null +++ b/examples/swe-agent-daytona/README.md @@ -0,0 +1,186 @@ +# SWE-Agent training with Harbor on Daytona sandboxes + +This example trains GLM-4.7-Flash on agentic terminal and coding tasks, with +task sandboxes hosted on [Daytona](https://www.daytona.io/) instead of local +Docker. Miles runs synchronous GRPO and serves the policy through its session +server; a Harbor agent server drives the agent and returns verifier rewards. + +It is the [`../swe-agent`](../swe-agent) pipeline with two changes: + +- **Daytona sandboxes.** The agent-server host needs outbound HTTPS but no + Docker daemon, no local image builds, and no local disk for task images. This + is the practical option when the trainer runs on a GPU node where you cannot + or do not want to run Docker-in-Docker. +- **terminus-2 agent.** terminus-2 runs as a host process and calls the model + endpoint itself, rather than from inside the sandbox. It appends both `user` + and `tool` turns, which the session server must be told to allow. + +Everything else — TITO, the session server, GRPO, the reward path — is shared +with `../swe-agent`, and `run.py` imports that example's `generate.py` and +`swe_agent_function.py` rather than duplicating them. + +## Files + +| File | Purpose | +| --- | --- | +| `run.py` | Training launcher for Daytona-backed terminus-2 runs. | +| `launch_agent_server.sh` | Starts the Harbor agent server in Daytona mode. | + +## 1. Provision Daytona + +Create an API key in the Daytona dashboard and export it on the agent-server +host: + +```bash +export DAYTONA_API_KEY= +``` + +Daytona enforces an **account-wide total-disk quota**, so size the run against +it: concurrent sandboxes times `HARBOR_DAYTONA_DISK_GB` must stay under the cap, +with headroom. Two things surprise people here: + +- A `stopped` sandbox still consumes quota. Only deletion, or expiry of the + sandbox's auto-delete interval, frees it. +- The quota is shared by everything using the same account. If other jobs use + the key, your sandbox creations will start failing for reasons that have + nothing to do with your run — so check total usage by owner before assuming + you are leaking sandboxes. + +An over-quota creation is rejected quickly rather than hanging, and surfaces as +`DaytonaValidationError`, or as `EnvironmentStartTimeoutError` when creations +are merely slow because the account is near its limit. + +## 2. Start the Harbor agent server + +Use the `harbor-miles-v0.20.0` branch of `harbor-framework/harbor`, which +carries the Miles integration: + +```bash +git clone https://github.com/harbor-framework/harbor.git +cd harbor +git checkout harbor-miles-v0.20.0 +uv sync + +export DAYTONA_API_KEY= +export HARBOR_TASKS_DIR=/path/to/harbor_tasks +export TRIALS_DIR=/path/to/trials +bash /path/to/miles/examples/swe-agent-daytona/launch_agent_server.sh +``` + +`HARBOR_TASKS_DIR` must contain one Harbor task directory for every +`metadata.instance_id` in the training data; a missing directory makes the trial +score 0 rather than raise. Set `--max-concurrent` to at least one sandbox per +trajectory in a rollout step (`--rollout-batch-size` times +`--n-samples-per-prompt`). Keep the agent timeout generous — agentic trials +routinely run past an hour. + +Run the agent server under a process supervisor or a detached terminal +multiplexer on its own host, not in a foreground shell over SSH: if that shell +dies it takes the agent server and every live sandbox with it, and the trainer +then starves without an obvious error. + +Verify `http://:11000/health` before launching Miles. + +## 3. Prepare data + +`../swe-agent/download_and_process_data.py` converts a local JSONL into Miles +format. For terminus-2, set the agent name accordingly: + +```bash +python examples/swe-agent/download_and_process_data.py \ + --input /path/to/terminal-bench.jsonl \ + --output /path/to/tb2_train.jsonl \ + --agent-name terminus-2 \ + --prompt-key instruction +``` + +## 4. Launch training + +The shape below is what a multi-day Terminal-Bench 2 run used on one node of 8 +H200 GPUs, with the agent server colocated on the same host as the trainer: + +```bash +export WANDB_API_KEY= + +python examples/swe-agent-daytona/run.py \ + --num-nodes 1 \ + --num-gpus-per-node 8 \ + --skip-prepare \ + --megatron-path /root/Megatron-LM \ + --hf-checkpoint /path/to/GLM-4.7-Flash \ + --ref-load /path/to/GLM-4.7-Flash_torch_dist \ + --save-dir /path/to/checkpoints \ + --prompt-data /path/to/tb2_train.jsonl \ + --max-seq-len 65536 \ + --rollout-batch-size 4 \ + --n-samples-per-prompt 8 \ + --global-batch-size 32 \ + --num-rollout 200 \ + --save-interval 10 \ + --agent-server-url http://127.0.0.1:11000 \ + --router-external-host \ + --save-traces-dir /path/to/traces \ + --wandb-project +``` + +For a smoke test, set `--num-rollout 1`. + +`--router-external-host` is the address the agent server uses to reach the Miles +session server and SGLang router. **It must be a numeric IP**: sgl-router parses +it into a Rust `SocketAddr` and a hostname fails to bind. Ports 30000 and 31000 +must be reachable from the agent-server host. + +## Sizing the per-turn response cap + +`--rollout-max-response-len` and the agent server's `AGENT_MAX_OUTPUT_TOKENS` +both cap a **single turn**, not the whole trajectory. Agentic trajectories are +routinely several times longer than one turn, so a cap that looks generous +against `--max-seq-len` can still abort most trials. + +When a turn exceeds the cap, `HARBOR_RESPONSE_LENGTH_POLICY=abort` ends the +trial and **none of that turn's tool calls are performed**, so the trial scores +0 and dilutes its GRPO group. The symptoms are +`SingleTurnMaxSeqLenExceededError` and `ContextLengthExceededError` in the trial +exception files, with `rollout/truncated_ratio` well above 0. + +To size these, compare `rollout/response_len/mean` and `.../max` against the +cap, and keep `AGENT_MAX_INPUT_TOKENS` above the largest observed context. +`--max-seq-len 65536` leaves plenty of headroom to raise both. + +## Verify progress + +Read `rollout/raw_reward` for the task solve rate. `rollout/rewards` is the +GRPO-centered advantage and sits near zero by construction, so it never shows +learning. + +Two properties of this shape make per-step reward misleading: + +- With `--rollout-batch-size 4` there are only 4 GRPO groups per step. Uniform + groups (all solved or all failed) contribute no gradient, and which 4 tasks + were drawn dominates the step reward. Judge progress on repeated tasks across + many batches, never on consecutive steps. +- A long run's headline health number is the fraction of trials that return + successfully, not the reward. Census the trial directories under + `--trials-dir` by outcome: no `exception.txt` means success, and the last + exception class named in that file is the failure mode. Key the census on the + **trial start time** — the mtime of the trial's `config.json`, written at + launch — because writing `exception.txt` bumps the directory mtime and makes + any mtime-sorted listing look like everything is failing. + +Confirm a suspected stall on disk before believing a dashboard. W&B uploads can +fail partway through a long run, dropping some metric rows while others keep +arriving, which looks exactly like a frozen reward curve. The per-step +`train_data/` and `rollout_data/.pt` dumps under `--save-traces-dir` +are written by the trainer itself and are authoritative. + +## Troubleshooting + +| Symptom | Cause | +| --- | --- | +| `DaytonaValidationError` on sandbox create | Account-wide disk quota exhausted; see step 1. | +| `EnvironmentStartTimeoutError` in bursts | Sandbox creation is slow because the account is near its quota. | +| `SingleTurnMaxSeqLenExceededError` | Per-turn output cap too low; see the sizing section. | +| `ContextLengthExceededError` | `AGENT_MAX_INPUT_TOKENS` below the observed context length. | +| Trajectories all one turn, many session rollbacks | `--tito-allowed-append-roles user tool` missing, so terminus-2's appends are rejected. | +| sgl-router fails to bind | `--router-external-host` is a hostname; it must be a numeric IP. | +| Every trial scores 0 | `metadata.instance_id` values have no matching directory under `HARBOR_TASKS_DIR`. | diff --git a/examples/swe-agent-daytona/launch_agent_server.sh b/examples/swe-agent-daytona/launch_agent_server.sh new file mode 100755 index 00000000000..7c84e49d31b --- /dev/null +++ b/examples/swe-agent-daytona/launch_agent_server.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Start the Harbor agent server in Daytona mode. +# +# Run this from a harbor checkout, before launching run.py. Trials are graded +# inside Daytona cloud sandboxes, so this host needs outbound HTTPS but no +# Docker daemon. +set -euo pipefail + +: "${DAYTONA_API_KEY:?set DAYTONA_API_KEY to a Daytona API key}" +: "${HARBOR_TASKS_DIR:?set HARBOR_TASKS_DIR to the directory holding Harbor task dirs}" + +TRIALS_DIR="${TRIALS_DIR:-/tmp/harbor_trials}" +PORT="${PORT:-11000}" +MAX_CONCURRENT="${MAX_CONCURRENT:-32}" + +export HARBOR_ENV_TYPE=daytona +export HARBOR_DAYTONA_DISK_GB="${HARBOR_DAYTONA_DISK_GB:-10}" +# Snapshot each task image on first use so later trials skip the build. +export HARBOR_DAYTONA_AUTO_SNAPSHOT=1 + +# terminus-2 runs as a host process and calls the model itself, so the model +# endpoint must be reachable from here rather than from inside the sandbox. +export OPENAI_API_KEY="${OPENAI_API_KEY:-dummy}" +export OPENAI_API_BASE="${OPENAI_API_BASE:-http://127.0.0.1:30000/v1}" +export OPENAI_BASE_URL="$OPENAI_API_BASE" + +# Keep these consistent with --rollout-max-response-len and --max-seq-len on the +# trainer side; see the README section on sizing them. +export AGENT_MAX_INPUT_TOKENS="${AGENT_MAX_INPUT_TOKENS:-32768}" +export AGENT_MAX_OUTPUT_TOKENS="${AGENT_MAX_OUTPUT_TOKENS:-8192}" +export HARBOR_RESPONSE_LENGTH_POLICY=abort + +mkdir -p "$TRIALS_DIR" + +exec python miles_agent_server.py \ + --host 0.0.0.0 \ + --port "$PORT" \ + --max-concurrent "$MAX_CONCURRENT" \ + --agent-timeout 5400 \ + --trials-dir "$TRIALS_DIR" diff --git a/examples/swe-agent-daytona/run.py b/examples/swe-agent-daytona/run.py new file mode 100644 index 00000000000..8c264e07c16 --- /dev/null +++ b/examples/swe-agent-daytona/run.py @@ -0,0 +1,284 @@ +"""SWE-Agent launcher (GLM-4.7-Flash) for Harbor tasks on Daytona sandboxes. + +Same Miles <-> Harbor pipeline as ../swe-agent, with two differences: + * sandboxes are Daytona cloud sandboxes rather than local Docker containers, + so the trainer host needs no Docker daemon and no per-task image pulls + * the agent is terminus-2, which needs --tito-allowed-append-roles + +Usage: + python run.py --prompt-data /path/to/tb2_train.jsonl +""" + +import os +import socket +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import typer + +import miles.utils.external_utils.command_utils as U + +SCRIPT_DIR = Path(__file__).resolve().parent +# generate.py and swe_agent_function.py are identical for Docker- and +# Daytona-backed runs, so import them from the sibling example instead of +# keeping a second copy in sync. +SHARED_DIR = SCRIPT_DIR.parent / "swe-agent" + + +def _default_router_host() -> str: + """sgl-router parses this into a Rust SocketAddr, so a hostname will not bind.""" + try: + return socket.gethostbyname(socket.gethostname()) + except OSError: + return "" + + +@dataclass +class ScriptArgs(U.ExecuteTrainConfig): + mode: Literal["normal", "debug_rollout_only"] = "normal" + run_id: str = U.create_run_id() + megatron_model_type: str = "glm4.7-flash" + num_gpus_per_node: int = 8 + megatron_path: str = "/root/Megatron-LM" + + # Paths + skip_prepare: bool = False + base_dir: str = "/root" + model_name: str = "GLM-4.7-Flash" + hf_checkpoint: str = "zai-org/GLM-4.7-Flash" + ref_load: str = "/root/GLM-4.7-Flash_torch_dist" + save_dir: str = "/root/GLM-4.7-Flash_swe_agent_daytona/" + save_traces_dir: str = "" + prompt_data: str = "/root/tb2_train.jsonl" + + # Training settings + max_seq_len: int = 65536 + num_rollout: int = 200 + rollout_batch_size: int = 4 + n_samples_per_prompt: int = 8 + global_batch_size: int = 32 + # Per-turn generation cap, not a whole-trajectory cap. See the README -- + # raising this is usually the first thing to try if trials abort with + # SingleTurnMaxSeqLenExceededError. + rollout_max_response_len: int = 8192 + max_tokens_per_gpu: int = 16384 + save_interval: int = 20 + lr: str = "1e-6" + sglang_mem_fraction_static: float = 0.7 + + # Agent settings + agent_server_url: str = os.environ.get("AGENT_SERVER_URL", "http://127.0.0.1:11000") + agent_model_name: str = os.environ.get("AGENT_MODEL_NAME", "model") + harbor_tasks_dir: str = os.environ.get("HARBOR_TASKS_DIR", "/root/harbor_tasks") + router_external_host: str = os.environ.get("MILES_ROUTER_EXTERNAL_HOST", "") or _default_router_host() + miles_host_ip: str = os.environ.get("MILES_HOST_IP", "") + + # W&B settings + wandb_key: str = os.environ.get("WANDB_KEY", os.environ.get("WANDB_API_KEY", "")) + wandb_project: str = os.environ.get("WANDB_PROJECT", "my-wandb-project") + wandb_team: str = os.environ.get("WANDB_TEAM", "") + wandb_run_name: str = "glm47-flash-swe-agent-daytona" + + # Prometheus settings + use_prometheus: bool = True + prometheus_port: int = 9090 + prometheus_run_name: str = "glm47-flash-swe-agent-daytona" + + +def cleanup(): + """Kill old Ray jobs and stale processes to free GPU resources.""" + my_pid = os.getpid() + ppid = os.getppid() + print(f"Cleanup starting (pid={my_pid}, ppid={ppid})") + targets = ["sglang", "train.py", "MegatronTrain"] + exclude = f"grep -v '^{my_pid}$' | grep -v '^{ppid}$'" + for t in targets: + subprocess.run( + f"pgrep -f '{t}' | {exclude} | xargs -r kill 2>/dev/null || true", + shell=True, + ) + time.sleep(5) + print(f"Cleanup complete (pid={my_pid}) — old processes killed.") + + +def prepare(args: ScriptArgs): + """Convert HF checkpoint to torch_dist format if not already done.""" + U.convert_checkpoint( + model_name=args.model_name, + megatron_model_type=args.megatron_model_type, + num_gpus_per_node=args.num_gpus_per_node, + dir_dst=args.base_dir, + hf_checkpoint=args.hf_checkpoint, + megatron_path=args.megatron_path, + ) + + +def execute(args: ScriptArgs): + ckpt_args = ( + f"--hf-checkpoint {args.hf_checkpoint} " + f"--ref-load {args.ref_load} " + f"--save {args.save_dir} " + f"--save-interval {args.save_interval} " + ) + + rollout_args = ( + f"--prompt-data {args.prompt_data} " + "--input-key prompt " + "--metadata-key metadata " + "--rollout-shuffle " + f"--num-rollout {args.num_rollout} " + f"--rollout-batch-size {args.rollout_batch_size} " + f"--n-samples-per-prompt {args.n_samples_per_prompt} " + "--rollout-temperature 0.8 " + f"--rollout-max-response-len {args.rollout_max_response_len} " + f"--max-seq-len {args.max_seq_len} " + f"--global-batch-size {args.global_batch_size} " + "--balance-data " + ) + + perf_args = ( + "--tensor-model-parallel-size 4 " + "--sequence-parallel " + "--pipeline-model-parallel-size 1 " + "--context-parallel-size 1 " + "--expert-model-parallel-size 8 " + "--expert-tensor-parallel-size 1 " + "--recompute-granularity full " + "--recompute-method uniform " + "--recompute-num-layers 1 " + "--use-dynamic-batch-size " + f"--max-tokens-per-gpu {args.max_tokens_per_gpu} " + "--optimizer-cpu-offload " + "--overlap-cpu-optimizer-d2h-h2d " + "--use-precision-aware-optimizer " + ) + + grpo_args = ( + "--advantage-estimator grpo " + "--use-kl-loss " + "--kl-loss-coef 0.01 " + "--kl-loss-type low_var_kl " + "--entropy-coef 0.0 " + "--eps-clip 0.2 " + "--eps-clip-high 0.28 " + ) + + optimizer_args = ( + "--optimizer adam " + f"--lr {args.lr} " + "--lr-decay-style constant " + "--weight-decay 0.1 " + "--adam-beta1 0.9 " + "--adam-beta2 0.98 " + ) + + sglang_args = ( + "--rollout-num-gpus-per-engine 1 " + f"--sglang-mem-fraction-static {args.sglang_mem_fraction_static} " + "--sglang-tool-call-parser glm47 " + "--sglang-reasoning-parser glm45 " + "--sglang-router-port 31000 " + ) + + agent_args = ( + "--custom-generate-function-path miles.rollout.generate_hub.agentic_tool_call.generate " + "--custom-agent-function-path swe_agent_function.run " + "--custom-rm-path generate.reward_func " + "--rollout-function-path generate.RolloutFn " + "--dynamic-sampling-filter-path miles.rollout.filter_hub.dynamic_sampling_filters.check_no_aborted " + "--tito-model glm47 " + "--use-session-server " + "--session-server-port 30000 " + # terminus-2 appends both user and tool turns; without this the session + # server rejects the append and rolls the trajectory back. + "--tito-allowed-append-roles user tool " + ) + + misc_args = ( + "--attention-dropout 0.0 " + "--hidden-dropout 0.0 " + "--accumulate-allreduce-grads-in-fp32 " + "--attention-softmax-in-fp32 " + "--attention-backend flash " + "--colocate " + f"--actor-num-nodes {args.num_nodes} " + f"--actor-num-gpus-per-node {args.num_gpus_per_node} " + f"--rollout-num-gpus {args.num_gpus_per_node} " + ) + + debug_args = "--debug-rollout-only " if args.mode == "debug_rollout_only" else "" + + trace_args = "" + if args.save_traces_dir: + trace_args = f"--dump-details {args.save_traces_dir} " + + wandb_args = "" + if args.wandb_key: + wandb_args = ( + "--use-wandb " + f"--wandb-project {args.wandb_project} " + f"--wandb-group {args.wandb_run_name} " + f"--wandb-key {args.wandb_key} " + ) + if args.wandb_team: + wandb_args += f"--wandb-team {args.wandb_team} " + + prometheus_args = "" + if args.use_prometheus: + prometheus_args = ( + "--use-prometheus " + f"--prometheus-port {args.prometheus_port} " + f"--prometheus-run-name {args.prometheus_run_name} " + ) + + train_args = ( + f"{ckpt_args}" + f"{rollout_args}" + f"{optimizer_args}" + f"{grpo_args}" + f"{wandb_args}" + f"{prometheus_args}" + f"{trace_args}" + f"{perf_args}" + f"{sglang_args}" + f"{agent_args}" + f"{misc_args}" + f"{debug_args}" + ) + + miles_root = U.repo_base_dir + + extra_env_vars = { + "PYTHONPATH": f"{args.megatron_path}:{SHARED_DIR}:{miles_root}", + "MILES_EXPERIMENTAL_ROLLOUT_REFACTOR": "1", + "AGENT_SERVER_URL": args.agent_server_url, + "AGENT_MODEL_NAME": args.agent_model_name, + "MILES_ROUTER_EXTERNAL_HOST": args.router_external_host, + "HARBOR_TASKS_DIR": args.harbor_tasks_dir, + } + if args.miles_host_ip: + extra_env_vars["MILES_HOST_IP"] = args.miles_host_ip + + U.execute_train( + train_args=train_args, + config=args, + num_gpus_per_node=args.num_gpus_per_node, + megatron_model_type=args.megatron_model_type, + megatron_path=args.megatron_path, + extra_env_vars=extra_env_vars, + ) + + +@U.dataclass_cli +def main(args: ScriptArgs): + cleanup() + if not args.skip_prepare: + prepare(args) + execute(args) + + +if __name__ == "__main__": + typer.run(main) From e78e1cf40361d85d651fdeafcd1cb73da95fc099 Mon Sep 17 00:00:00 2001 From: Shi Dong Date: Tue, 28 Jul 2026 15:02:27 -0700 Subject: [PATCH 2/6] examples: rename to swe-agent-harbor-daytona and address review - rename the example directory to swe-agent-harbor-daytona - name the harbor branch (harbor-miles-v0.20.0) in launch_agent_server.sh - trim the Daytona disk-quota discussion down to the sizing rule - refer to sibling files by their path from the repo root, not ../swe-agent - drop the troubleshooting row for a missing --tito-allowed-append-roles, which run.py always passes --- .../{swe-agent-daytona => swe-agent-harbor-daytona}/README.md | 0 .../launch_agent_server.sh | 0 examples/{swe-agent-daytona => swe-agent-harbor-daytona}/run.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename examples/{swe-agent-daytona => swe-agent-harbor-daytona}/README.md (100%) rename examples/{swe-agent-daytona => swe-agent-harbor-daytona}/launch_agent_server.sh (100%) rename examples/{swe-agent-daytona => swe-agent-harbor-daytona}/run.py (100%) diff --git a/examples/swe-agent-daytona/README.md b/examples/swe-agent-harbor-daytona/README.md similarity index 100% rename from examples/swe-agent-daytona/README.md rename to examples/swe-agent-harbor-daytona/README.md diff --git a/examples/swe-agent-daytona/launch_agent_server.sh b/examples/swe-agent-harbor-daytona/launch_agent_server.sh similarity index 100% rename from examples/swe-agent-daytona/launch_agent_server.sh rename to examples/swe-agent-harbor-daytona/launch_agent_server.sh diff --git a/examples/swe-agent-daytona/run.py b/examples/swe-agent-harbor-daytona/run.py similarity index 100% rename from examples/swe-agent-daytona/run.py rename to examples/swe-agent-harbor-daytona/run.py From 5a4ae071a1364974b8cfa3bfbd020ebf4bad77d3 Mon Sep 17 00:00:00 2001 From: Shi Dong Date: Tue, 28 Jul 2026 15:03:13 -0700 Subject: [PATCH 3/6] examples: address review comments on swe-agent-harbor-daytona - name the harbor branch (harbor-miles-v0.20.0) in launch_agent_server.sh - trim the Daytona disk-quota discussion down to the sizing rule - refer to sibling files by their path from the repo root, not ../swe-agent - drop the troubleshooting row for a missing --tito-allowed-append-roles, which run.py always passes - follow the directory rename through paths and default run names --- examples/README.md | 2 +- examples/swe-agent-harbor-daytona/README.md | 33 ++++++------------- .../launch_agent_server.sh | 8 +++-- examples/swe-agent-harbor-daytona/run.py | 14 ++++---- 4 files changed, 23 insertions(+), 34 deletions(-) diff --git a/examples/README.md b/examples/README.md index a7bd86decd9..9d2c7ef20a3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,7 +19,7 @@ These examples provide concrete examples to leverage miles in your own RL workfl - **[search-r1](./search-r1)**: A minimal reproduction of Search-R1, featuring multi-turn conversation and tool-calling. - **[strands-agents](./strands-agents)**: Integration example with the Strands-Agents scaffolding framework. - **[swe-agent](./swe-agent)**: Trains coding and terminal agents with Harbor-managed sandboxes and verifier rewards. -- **[swe-agent-daytona](./swe-agent-daytona)**: The same Harbor pipeline with task sandboxes hosted on Daytona instead of local Docker. +- **[swe-agent-harbor-daytona](./swe-agent-harbor-daytona)**: The same Harbor pipeline with task sandboxes hosted on Daytona instead of local Docker. - **[tau-bench](./tau-bench)**: Training in an agentic multi-turn tool use environment (Tau-bench). - **[train_infer_mismatch_helper](./train_infer_mismatch_helper)**: Algorithmic methods for rollout correction (e.g., TIS, MIS). - **[true_on_policy](./true_on_policy)**: Ensures strictly equal log probabilities between inference (SGLang) and training engines. diff --git a/examples/swe-agent-harbor-daytona/README.md b/examples/swe-agent-harbor-daytona/README.md index 5b854ac5cb1..192bf84fa08 100644 --- a/examples/swe-agent-harbor-daytona/README.md +++ b/examples/swe-agent-harbor-daytona/README.md @@ -5,7 +5,7 @@ task sandboxes hosted on [Daytona](https://www.daytona.io/) instead of local Docker. Miles runs synchronous GRPO and serves the policy through its session server; a Harbor agent server drives the agent and returns verifier rewards. -It is the [`../swe-agent`](../swe-agent) pipeline with two changes: +It is the [`examples/swe-agent`](../swe-agent) pipeline with two changes: - **Daytona sandboxes.** The agent-server host needs outbound HTTPS but no Docker daemon, no local image builds, and no local disk for task images. This @@ -16,7 +16,7 @@ It is the [`../swe-agent`](../swe-agent) pipeline with two changes: and `tool` turns, which the session server must be told to allow. Everything else — TITO, the session server, GRPO, the reward path — is shared -with `../swe-agent`, and `run.py` imports that example's `generate.py` and +with `examples/swe-agent`, and `run.py` imports that example's `generate.py` and `swe_agent_function.py` rather than duplicating them. ## Files @@ -35,20 +35,8 @@ host: export DAYTONA_API_KEY= ``` -Daytona enforces an **account-wide total-disk quota**, so size the run against -it: concurrent sandboxes times `HARBOR_DAYTONA_DISK_GB` must stay under the cap, -with headroom. Two things surprise people here: - -- A `stopped` sandbox still consumes quota. Only deletion, or expiry of the - sandbox's auto-delete interval, frees it. -- The quota is shared by everything using the same account. If other jobs use - the key, your sandbox creations will start failing for reasons that have - nothing to do with your run — so check total usage by owner before assuming - you are leaking sandboxes. - -An over-quota creation is rejected quickly rather than hanging, and surfaces as -`DaytonaValidationError`, or as `EnvironmentStartTimeoutError` when creations -are merely slow because the account is near its limit. +Daytona accounts have a total-disk quota, so keep concurrent sandboxes times +`HARBOR_DAYTONA_DISK_GB` under it. ## 2. Start the Harbor agent server @@ -64,7 +52,7 @@ uv sync export DAYTONA_API_KEY= export HARBOR_TASKS_DIR=/path/to/harbor_tasks export TRIALS_DIR=/path/to/trials -bash /path/to/miles/examples/swe-agent-daytona/launch_agent_server.sh +bash /path/to/miles/examples/swe-agent-harbor-daytona/launch_agent_server.sh ``` `HARBOR_TASKS_DIR` must contain one Harbor task directory for every @@ -83,8 +71,8 @@ Verify `http://:11000/health` before launching Miles. ## 3. Prepare data -`../swe-agent/download_and_process_data.py` converts a local JSONL into Miles -format. For terminus-2, set the agent name accordingly: +`examples/swe-agent/download_and_process_data.py` converts a local JSONL into +Miles format. For terminus-2, set the agent name accordingly: ```bash python examples/swe-agent/download_and_process_data.py \ @@ -102,7 +90,7 @@ H200 GPUs, with the agent server colocated on the same host as the trainer: ```bash export WANDB_API_KEY= -python examples/swe-agent-daytona/run.py \ +python examples/swe-agent-harbor-daytona/run.py \ --num-nodes 1 \ --num-gpus-per-node 8 \ --skip-prepare \ @@ -177,10 +165,9 @@ are written by the trainer itself and are authoritative. | Symptom | Cause | | --- | --- | -| `DaytonaValidationError` on sandbox create | Account-wide disk quota exhausted; see step 1. | -| `EnvironmentStartTimeoutError` in bursts | Sandbox creation is slow because the account is near its quota. | +| `DaytonaValidationError` on sandbox create | Daytona disk quota exhausted. | +| `EnvironmentStartTimeoutError` in bursts | Sandbox creation is slow because the account is near its disk quota. | | `SingleTurnMaxSeqLenExceededError` | Per-turn output cap too low; see the sizing section. | | `ContextLengthExceededError` | `AGENT_MAX_INPUT_TOKENS` below the observed context length. | -| Trajectories all one turn, many session rollbacks | `--tito-allowed-append-roles user tool` missing, so terminus-2's appends are rejected. | | sgl-router fails to bind | `--router-external-host` is a hostname; it must be a numeric IP. | | Every trial scores 0 | `metadata.instance_id` values have no matching directory under `HARBOR_TASKS_DIR`. | diff --git a/examples/swe-agent-harbor-daytona/launch_agent_server.sh b/examples/swe-agent-harbor-daytona/launch_agent_server.sh index 7c84e49d31b..7a977180242 100755 --- a/examples/swe-agent-harbor-daytona/launch_agent_server.sh +++ b/examples/swe-agent-harbor-daytona/launch_agent_server.sh @@ -1,9 +1,11 @@ #!/bin/bash # Start the Harbor agent server in Daytona mode. # -# Run this from a harbor checkout, before launching run.py. Trials are graded -# inside Daytona cloud sandboxes, so this host needs outbound HTTPS but no -# Docker daemon. +# Run this from the root of a harbor-framework/harbor checkout on the +# harbor-miles-v0.20.0 branch, which carries the Miles integration, before +# launching examples/swe-agent-harbor-daytona/run.py. Trials are graded inside +# Daytona cloud sandboxes, so this host needs outbound HTTPS but no Docker +# daemon. set -euo pipefail : "${DAYTONA_API_KEY:?set DAYTONA_API_KEY to a Daytona API key}" diff --git a/examples/swe-agent-harbor-daytona/run.py b/examples/swe-agent-harbor-daytona/run.py index 8c264e07c16..8dc62746438 100644 --- a/examples/swe-agent-harbor-daytona/run.py +++ b/examples/swe-agent-harbor-daytona/run.py @@ -1,12 +1,12 @@ """SWE-Agent launcher (GLM-4.7-Flash) for Harbor tasks on Daytona sandboxes. -Same Miles <-> Harbor pipeline as ../swe-agent, with two differences: +Same Miles <-> Harbor pipeline as examples/swe-agent, with two differences: * sandboxes are Daytona cloud sandboxes rather than local Docker containers, so the trainer host needs no Docker daemon and no per-task image pulls * the agent is terminus-2, which needs --tito-allowed-append-roles Usage: - python run.py --prompt-data /path/to/tb2_train.jsonl + python examples/swe-agent-harbor-daytona/run.py --prompt-data /path/to/tb2_train.jsonl """ import os @@ -23,8 +23,8 @@ SCRIPT_DIR = Path(__file__).resolve().parent # generate.py and swe_agent_function.py are identical for Docker- and -# Daytona-backed runs, so import them from the sibling example instead of -# keeping a second copy in sync. +# Daytona-backed runs, so import them from examples/swe-agent instead of keeping +# a second copy in sync. SHARED_DIR = SCRIPT_DIR.parent / "swe-agent" @@ -50,7 +50,7 @@ class ScriptArgs(U.ExecuteTrainConfig): model_name: str = "GLM-4.7-Flash" hf_checkpoint: str = "zai-org/GLM-4.7-Flash" ref_load: str = "/root/GLM-4.7-Flash_torch_dist" - save_dir: str = "/root/GLM-4.7-Flash_swe_agent_daytona/" + save_dir: str = "/root/GLM-4.7-Flash_swe_agent_harbor_daytona/" save_traces_dir: str = "" prompt_data: str = "/root/tb2_train.jsonl" @@ -80,12 +80,12 @@ class ScriptArgs(U.ExecuteTrainConfig): wandb_key: str = os.environ.get("WANDB_KEY", os.environ.get("WANDB_API_KEY", "")) wandb_project: str = os.environ.get("WANDB_PROJECT", "my-wandb-project") wandb_team: str = os.environ.get("WANDB_TEAM", "") - wandb_run_name: str = "glm47-flash-swe-agent-daytona" + wandb_run_name: str = "glm47-flash-swe-agent-harbor-daytona" # Prometheus settings use_prometheus: bool = True prometheus_port: int = 9090 - prometheus_run_name: str = "glm47-flash-swe-agent-daytona" + prometheus_run_name: str = "glm47-flash-swe-agent-harbor-daytona" def cleanup(): From be3b592b3b1cd6d0a18183fd6abf32d30541a328 Mon Sep 17 00:00:00 2001 From: Shi Dong Date: Tue, 28 Jul 2026 15:13:20 -0700 Subject: [PATCH 4/6] examples: drop remaining relative paths and state the node shape - say up front that the example targets a single node of 8 H200 GPUs, and drop the now-duplicated hardware note from the launch section - reference examples/swe-agent as plain text rather than a relative link - spell out rollout/response_len/max instead of abbreviating it as .../max --- examples/swe-agent-harbor-daytona/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/swe-agent-harbor-daytona/README.md b/examples/swe-agent-harbor-daytona/README.md index 192bf84fa08..108a17fd297 100644 --- a/examples/swe-agent-harbor-daytona/README.md +++ b/examples/swe-agent-harbor-daytona/README.md @@ -4,8 +4,9 @@ This example trains GLM-4.7-Flash on agentic terminal and coding tasks, with task sandboxes hosted on [Daytona](https://www.daytona.io/) instead of local Docker. Miles runs synchronous GRPO and serves the policy through its session server; a Harbor agent server drives the agent and returns verifier rewards. +It is meant to run on a single node of 8 H200 GPUs. -It is the [`examples/swe-agent`](../swe-agent) pipeline with two changes: +It is the `examples/swe-agent` pipeline with two changes: - **Daytona sandboxes.** The agent-server host needs outbound HTTPS but no Docker daemon, no local image builds, and no local disk for task images. This @@ -84,8 +85,8 @@ python examples/swe-agent/download_and_process_data.py \ ## 4. Launch training -The shape below is what a multi-day Terminal-Bench 2 run used on one node of 8 -H200 GPUs, with the agent server colocated on the same host as the trainer: +The shape below is what a multi-day Terminal-Bench 2 run used, with the agent +server colocated on the same host as the trainer: ```bash export WANDB_API_KEY= @@ -131,8 +132,9 @@ trial and **none of that turn's tool calls are performed**, so the trial scores `SingleTurnMaxSeqLenExceededError` and `ContextLengthExceededError` in the trial exception files, with `rollout/truncated_ratio` well above 0. -To size these, compare `rollout/response_len/mean` and `.../max` against the -cap, and keep `AGENT_MAX_INPUT_TOKENS` above the largest observed context. +To size these, compare `rollout/response_len/mean` and `rollout/response_len/max` +against the cap, and keep `AGENT_MAX_INPUT_TOKENS` above the largest observed +context. `--max-seq-len 65536` leaves plenty of headroom to raise both. ## Verify progress From 20976ba056ce9bd431989881129b0e7680e78d36 Mon Sep 17 00:00:00 2001 From: Shi Dong Date: Mon, 3 Aug 2026 13:15:40 -0700 Subject: [PATCH 5/6] Move swe-agent-harbor-daytona under examples/experimental/ --- .../{ => experimental}/swe-agent-harbor-daytona/README.md | 4 ++-- .../swe-agent-harbor-daytona/launch_agent_server.sh | 3 ++- examples/{ => experimental}/swe-agent-harbor-daytona/run.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) rename examples/{ => experimental}/swe-agent-harbor-daytona/README.md (97%) rename examples/{ => experimental}/swe-agent-harbor-daytona/launch_agent_server.sh (94%) rename examples/{ => experimental}/swe-agent-harbor-daytona/run.py (98%) diff --git a/examples/swe-agent-harbor-daytona/README.md b/examples/experimental/swe-agent-harbor-daytona/README.md similarity index 97% rename from examples/swe-agent-harbor-daytona/README.md rename to examples/experimental/swe-agent-harbor-daytona/README.md index 108a17fd297..cfbfaccfd4f 100644 --- a/examples/swe-agent-harbor-daytona/README.md +++ b/examples/experimental/swe-agent-harbor-daytona/README.md @@ -53,7 +53,7 @@ uv sync export DAYTONA_API_KEY= export HARBOR_TASKS_DIR=/path/to/harbor_tasks export TRIALS_DIR=/path/to/trials -bash /path/to/miles/examples/swe-agent-harbor-daytona/launch_agent_server.sh +bash /path/to/miles/examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh ``` `HARBOR_TASKS_DIR` must contain one Harbor task directory for every @@ -91,7 +91,7 @@ server colocated on the same host as the trainer: ```bash export WANDB_API_KEY= -python examples/swe-agent-harbor-daytona/run.py \ +python examples/experimental/swe-agent-harbor-daytona/run.py \ --num-nodes 1 \ --num-gpus-per-node 8 \ --skip-prepare \ diff --git a/examples/swe-agent-harbor-daytona/launch_agent_server.sh b/examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh similarity index 94% rename from examples/swe-agent-harbor-daytona/launch_agent_server.sh rename to examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh index 7a977180242..1d3c6bf1c03 100755 --- a/examples/swe-agent-harbor-daytona/launch_agent_server.sh +++ b/examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh @@ -3,7 +3,8 @@ # # Run this from the root of a harbor-framework/harbor checkout on the # harbor-miles-v0.20.0 branch, which carries the Miles integration, before -# launching examples/swe-agent-harbor-daytona/run.py. Trials are graded inside +# launching examples/experimental/swe-agent-harbor-daytona/run.py. Trials are +# graded inside # Daytona cloud sandboxes, so this host needs outbound HTTPS but no Docker # daemon. set -euo pipefail diff --git a/examples/swe-agent-harbor-daytona/run.py b/examples/experimental/swe-agent-harbor-daytona/run.py similarity index 98% rename from examples/swe-agent-harbor-daytona/run.py rename to examples/experimental/swe-agent-harbor-daytona/run.py index 8dc62746438..b40d19f7cf7 100644 --- a/examples/swe-agent-harbor-daytona/run.py +++ b/examples/experimental/swe-agent-harbor-daytona/run.py @@ -6,7 +6,7 @@ * the agent is terminus-2, which needs --tito-allowed-append-roles Usage: - python examples/swe-agent-harbor-daytona/run.py --prompt-data /path/to/tb2_train.jsonl + python examples/experimental/swe-agent-harbor-daytona/run.py --prompt-data /path/to/tb2_train.jsonl """ import os @@ -25,7 +25,7 @@ # generate.py and swe_agent_function.py are identical for Docker- and # Daytona-backed runs, so import them from examples/swe-agent instead of keeping # a second copy in sync. -SHARED_DIR = SCRIPT_DIR.parent / "swe-agent" +SHARED_DIR = SCRIPT_DIR.parent.parent / "swe-agent" def _default_router_host() -> str: From 12e7742536ee445d0dc066639604381eb111c072 Mon Sep 17 00:00:00 2001 From: Shi Dong Date: Wed, 5 Aug 2026 20:01:50 -0700 Subject: [PATCH 6/6] examples: drop the Daytona example's launcher, reuse examples/swe-agent/run.py The launcher only existed because terminus-2 needed --tito-allowed-append-roles. #1818 removed that flag and added a test asserting it is rejected, so the launcher could no longer start a run. Nothing else in it was Daytona-specific: Daytona is selected entirely by the agent server's environment, which the trainer never sees. The four values it exposed as flags all defaulted to what examples/swe-agent/run.py already hardcodes, so reusing that launcher is behaviourally identical. Also corrects --router-external-host: it is substituted into the base URL handed to the agent and only has to resolve from the agent-server host, so a hostname is fine. The bind-time constraint belongs to --miles-host-ip. --- .../swe-agent-harbor-daytona/README.md | 33 +- .../launch_agent_server.sh | 6 +- .../swe-agent-harbor-daytona/run.py | 284 ------------------ 3 files changed, 23 insertions(+), 300 deletions(-) delete mode 100644 examples/experimental/swe-agent-harbor-daytona/run.py diff --git a/examples/experimental/swe-agent-harbor-daytona/README.md b/examples/experimental/swe-agent-harbor-daytona/README.md index cfbfaccfd4f..3d40c79da55 100644 --- a/examples/experimental/swe-agent-harbor-daytona/README.md +++ b/examples/experimental/swe-agent-harbor-daytona/README.md @@ -13,18 +13,19 @@ It is the `examples/swe-agent` pipeline with two changes: is the practical option when the trainer runs on a GPU node where you cannot or do not want to run Docker-in-Docker. - **terminus-2 agent.** terminus-2 runs as a host process and calls the model - endpoint itself, rather than from inside the sandbox. It appends both `user` - and `tool` turns, which the session server must be told to allow. + endpoint itself, rather than from inside the sandbox, so the model endpoint + must be reachable from the agent-server host. -Everything else — TITO, the session server, GRPO, the reward path — is shared -with `examples/swe-agent`, and `run.py` imports that example's `generate.py` and -`swe_agent_function.py` rather than duplicating them. +Everything else — TITO, the session server, GRPO, the reward path — is identical +to `examples/swe-agent`, and so is the trainer side: this example has no +launcher of its own and runs `examples/swe-agent/run.py` unchanged. Daytona is +selected entirely by the agent server's environment, which the trainer never +sees. ## Files | File | Purpose | | --- | --- | -| `run.py` | Training launcher for Daytona-backed terminus-2 runs. | | `launch_agent_server.sh` | Starts the Harbor agent server in Daytona mode. | ## 1. Provision Daytona @@ -91,7 +92,7 @@ server colocated on the same host as the trainer: ```bash export WANDB_API_KEY= -python examples/experimental/swe-agent-harbor-daytona/run.py \ +python examples/swe-agent/run.py \ --num-nodes 1 \ --num-gpus-per-node 8 \ --skip-prepare \ @@ -107,7 +108,7 @@ python examples/experimental/swe-agent-harbor-daytona/run.py \ --num-rollout 200 \ --save-interval 10 \ --agent-server-url http://127.0.0.1:11000 \ - --router-external-host \ + --router-external-host \ --save-traces-dir /path/to/traces \ --wandb-project ``` @@ -115,9 +116,12 @@ python examples/experimental/swe-agent-harbor-daytona/run.py \ For a smoke test, set `--num-rollout 1`. `--router-external-host` is the address the agent server uses to reach the Miles -session server and SGLang router. **It must be a numeric IP**: sgl-router parses -it into a Rust `SocketAddr` and a hostname fails to bind. Ports 30000 and 31000 -must be reachable from the agent-server host. +session server, substituted into the base URL handed to the agent. It only has +to resolve from the agent-server host, so a hostname is fine — use one when the +agent server reaches the trainer over a tailnet or other overlay. Do not confuse +it with `--miles-host-ip`, which is bound locally on the trainer and must be an +address that already exists on one of its interfaces. Ports 30000 and 31000 must +be reachable from the agent-server host. ## Sizing the per-turn response cap @@ -137,6 +141,11 @@ against the cap, and keep `AGENT_MAX_INPUT_TOKENS` above the largest observed context. `--max-seq-len 65536` leaves plenty of headroom to raise both. +`examples/swe-agent/run.py` hardcodes `--rollout-max-response-len 8192`, so raise +it there; `AGENT_MAX_OUTPUT_TOKENS` is an environment variable on the agent +server and is set in `launch_agent_server.sh`. Raise the two together — leaving +either one behind reintroduces the aborts. + ## Verify progress Read `rollout/raw_reward` for the task solve rate. `rollout/rewards` is the @@ -171,5 +180,5 @@ are written by the trainer itself and are authoritative. | `EnvironmentStartTimeoutError` in bursts | Sandbox creation is slow because the account is near its disk quota. | | `SingleTurnMaxSeqLenExceededError` | Per-turn output cap too low; see the sizing section. | | `ContextLengthExceededError` | `AGENT_MAX_INPUT_TOKENS` below the observed context length. | -| sgl-router fails to bind | `--router-external-host` is a hostname; it must be a numeric IP. | +| sgl-router fails to bind | `--miles-host-ip` is not an address the trainer host can bind; leave it unset to auto-detect. | | Every trial scores 0 | `metadata.instance_id` values have no matching directory under `HARBOR_TASKS_DIR`. | diff --git a/examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh b/examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh index 1d3c6bf1c03..44c89927303 100755 --- a/examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh +++ b/examples/experimental/swe-agent-harbor-daytona/launch_agent_server.sh @@ -3,10 +3,8 @@ # # Run this from the root of a harbor-framework/harbor checkout on the # harbor-miles-v0.20.0 branch, which carries the Miles integration, before -# launching examples/experimental/swe-agent-harbor-daytona/run.py. Trials are -# graded inside -# Daytona cloud sandboxes, so this host needs outbound HTTPS but no Docker -# daemon. +# launching examples/swe-agent/run.py. Trials are graded inside Daytona cloud +# sandboxes, so this host needs outbound HTTPS but no Docker daemon. set -euo pipefail : "${DAYTONA_API_KEY:?set DAYTONA_API_KEY to a Daytona API key}" diff --git a/examples/experimental/swe-agent-harbor-daytona/run.py b/examples/experimental/swe-agent-harbor-daytona/run.py deleted file mode 100644 index b40d19f7cf7..00000000000 --- a/examples/experimental/swe-agent-harbor-daytona/run.py +++ /dev/null @@ -1,284 +0,0 @@ -"""SWE-Agent launcher (GLM-4.7-Flash) for Harbor tasks on Daytona sandboxes. - -Same Miles <-> Harbor pipeline as examples/swe-agent, with two differences: - * sandboxes are Daytona cloud sandboxes rather than local Docker containers, - so the trainer host needs no Docker daemon and no per-task image pulls - * the agent is terminus-2, which needs --tito-allowed-append-roles - -Usage: - python examples/experimental/swe-agent-harbor-daytona/run.py --prompt-data /path/to/tb2_train.jsonl -""" - -import os -import socket -import subprocess -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Literal - -import typer - -import miles.utils.external_utils.command_utils as U - -SCRIPT_DIR = Path(__file__).resolve().parent -# generate.py and swe_agent_function.py are identical for Docker- and -# Daytona-backed runs, so import them from examples/swe-agent instead of keeping -# a second copy in sync. -SHARED_DIR = SCRIPT_DIR.parent.parent / "swe-agent" - - -def _default_router_host() -> str: - """sgl-router parses this into a Rust SocketAddr, so a hostname will not bind.""" - try: - return socket.gethostbyname(socket.gethostname()) - except OSError: - return "" - - -@dataclass -class ScriptArgs(U.ExecuteTrainConfig): - mode: Literal["normal", "debug_rollout_only"] = "normal" - run_id: str = U.create_run_id() - megatron_model_type: str = "glm4.7-flash" - num_gpus_per_node: int = 8 - megatron_path: str = "/root/Megatron-LM" - - # Paths - skip_prepare: bool = False - base_dir: str = "/root" - model_name: str = "GLM-4.7-Flash" - hf_checkpoint: str = "zai-org/GLM-4.7-Flash" - ref_load: str = "/root/GLM-4.7-Flash_torch_dist" - save_dir: str = "/root/GLM-4.7-Flash_swe_agent_harbor_daytona/" - save_traces_dir: str = "" - prompt_data: str = "/root/tb2_train.jsonl" - - # Training settings - max_seq_len: int = 65536 - num_rollout: int = 200 - rollout_batch_size: int = 4 - n_samples_per_prompt: int = 8 - global_batch_size: int = 32 - # Per-turn generation cap, not a whole-trajectory cap. See the README -- - # raising this is usually the first thing to try if trials abort with - # SingleTurnMaxSeqLenExceededError. - rollout_max_response_len: int = 8192 - max_tokens_per_gpu: int = 16384 - save_interval: int = 20 - lr: str = "1e-6" - sglang_mem_fraction_static: float = 0.7 - - # Agent settings - agent_server_url: str = os.environ.get("AGENT_SERVER_URL", "http://127.0.0.1:11000") - agent_model_name: str = os.environ.get("AGENT_MODEL_NAME", "model") - harbor_tasks_dir: str = os.environ.get("HARBOR_TASKS_DIR", "/root/harbor_tasks") - router_external_host: str = os.environ.get("MILES_ROUTER_EXTERNAL_HOST", "") or _default_router_host() - miles_host_ip: str = os.environ.get("MILES_HOST_IP", "") - - # W&B settings - wandb_key: str = os.environ.get("WANDB_KEY", os.environ.get("WANDB_API_KEY", "")) - wandb_project: str = os.environ.get("WANDB_PROJECT", "my-wandb-project") - wandb_team: str = os.environ.get("WANDB_TEAM", "") - wandb_run_name: str = "glm47-flash-swe-agent-harbor-daytona" - - # Prometheus settings - use_prometheus: bool = True - prometheus_port: int = 9090 - prometheus_run_name: str = "glm47-flash-swe-agent-harbor-daytona" - - -def cleanup(): - """Kill old Ray jobs and stale processes to free GPU resources.""" - my_pid = os.getpid() - ppid = os.getppid() - print(f"Cleanup starting (pid={my_pid}, ppid={ppid})") - targets = ["sglang", "train.py", "MegatronTrain"] - exclude = f"grep -v '^{my_pid}$' | grep -v '^{ppid}$'" - for t in targets: - subprocess.run( - f"pgrep -f '{t}' | {exclude} | xargs -r kill 2>/dev/null || true", - shell=True, - ) - time.sleep(5) - print(f"Cleanup complete (pid={my_pid}) — old processes killed.") - - -def prepare(args: ScriptArgs): - """Convert HF checkpoint to torch_dist format if not already done.""" - U.convert_checkpoint( - model_name=args.model_name, - megatron_model_type=args.megatron_model_type, - num_gpus_per_node=args.num_gpus_per_node, - dir_dst=args.base_dir, - hf_checkpoint=args.hf_checkpoint, - megatron_path=args.megatron_path, - ) - - -def execute(args: ScriptArgs): - ckpt_args = ( - f"--hf-checkpoint {args.hf_checkpoint} " - f"--ref-load {args.ref_load} " - f"--save {args.save_dir} " - f"--save-interval {args.save_interval} " - ) - - rollout_args = ( - f"--prompt-data {args.prompt_data} " - "--input-key prompt " - "--metadata-key metadata " - "--rollout-shuffle " - f"--num-rollout {args.num_rollout} " - f"--rollout-batch-size {args.rollout_batch_size} " - f"--n-samples-per-prompt {args.n_samples_per_prompt} " - "--rollout-temperature 0.8 " - f"--rollout-max-response-len {args.rollout_max_response_len} " - f"--max-seq-len {args.max_seq_len} " - f"--global-batch-size {args.global_batch_size} " - "--balance-data " - ) - - perf_args = ( - "--tensor-model-parallel-size 4 " - "--sequence-parallel " - "--pipeline-model-parallel-size 1 " - "--context-parallel-size 1 " - "--expert-model-parallel-size 8 " - "--expert-tensor-parallel-size 1 " - "--recompute-granularity full " - "--recompute-method uniform " - "--recompute-num-layers 1 " - "--use-dynamic-batch-size " - f"--max-tokens-per-gpu {args.max_tokens_per_gpu} " - "--optimizer-cpu-offload " - "--overlap-cpu-optimizer-d2h-h2d " - "--use-precision-aware-optimizer " - ) - - grpo_args = ( - "--advantage-estimator grpo " - "--use-kl-loss " - "--kl-loss-coef 0.01 " - "--kl-loss-type low_var_kl " - "--entropy-coef 0.0 " - "--eps-clip 0.2 " - "--eps-clip-high 0.28 " - ) - - optimizer_args = ( - "--optimizer adam " - f"--lr {args.lr} " - "--lr-decay-style constant " - "--weight-decay 0.1 " - "--adam-beta1 0.9 " - "--adam-beta2 0.98 " - ) - - sglang_args = ( - "--rollout-num-gpus-per-engine 1 " - f"--sglang-mem-fraction-static {args.sglang_mem_fraction_static} " - "--sglang-tool-call-parser glm47 " - "--sglang-reasoning-parser glm45 " - "--sglang-router-port 31000 " - ) - - agent_args = ( - "--custom-generate-function-path miles.rollout.generate_hub.agentic_tool_call.generate " - "--custom-agent-function-path swe_agent_function.run " - "--custom-rm-path generate.reward_func " - "--rollout-function-path generate.RolloutFn " - "--dynamic-sampling-filter-path miles.rollout.filter_hub.dynamic_sampling_filters.check_no_aborted " - "--tito-model glm47 " - "--use-session-server " - "--session-server-port 30000 " - # terminus-2 appends both user and tool turns; without this the session - # server rejects the append and rolls the trajectory back. - "--tito-allowed-append-roles user tool " - ) - - misc_args = ( - "--attention-dropout 0.0 " - "--hidden-dropout 0.0 " - "--accumulate-allreduce-grads-in-fp32 " - "--attention-softmax-in-fp32 " - "--attention-backend flash " - "--colocate " - f"--actor-num-nodes {args.num_nodes} " - f"--actor-num-gpus-per-node {args.num_gpus_per_node} " - f"--rollout-num-gpus {args.num_gpus_per_node} " - ) - - debug_args = "--debug-rollout-only " if args.mode == "debug_rollout_only" else "" - - trace_args = "" - if args.save_traces_dir: - trace_args = f"--dump-details {args.save_traces_dir} " - - wandb_args = "" - if args.wandb_key: - wandb_args = ( - "--use-wandb " - f"--wandb-project {args.wandb_project} " - f"--wandb-group {args.wandb_run_name} " - f"--wandb-key {args.wandb_key} " - ) - if args.wandb_team: - wandb_args += f"--wandb-team {args.wandb_team} " - - prometheus_args = "" - if args.use_prometheus: - prometheus_args = ( - "--use-prometheus " - f"--prometheus-port {args.prometheus_port} " - f"--prometheus-run-name {args.prometheus_run_name} " - ) - - train_args = ( - f"{ckpt_args}" - f"{rollout_args}" - f"{optimizer_args}" - f"{grpo_args}" - f"{wandb_args}" - f"{prometheus_args}" - f"{trace_args}" - f"{perf_args}" - f"{sglang_args}" - f"{agent_args}" - f"{misc_args}" - f"{debug_args}" - ) - - miles_root = U.repo_base_dir - - extra_env_vars = { - "PYTHONPATH": f"{args.megatron_path}:{SHARED_DIR}:{miles_root}", - "MILES_EXPERIMENTAL_ROLLOUT_REFACTOR": "1", - "AGENT_SERVER_URL": args.agent_server_url, - "AGENT_MODEL_NAME": args.agent_model_name, - "MILES_ROUTER_EXTERNAL_HOST": args.router_external_host, - "HARBOR_TASKS_DIR": args.harbor_tasks_dir, - } - if args.miles_host_ip: - extra_env_vars["MILES_HOST_IP"] = args.miles_host_ip - - U.execute_train( - train_args=train_args, - config=args, - num_gpus_per_node=args.num_gpus_per_node, - megatron_model_type=args.megatron_model_type, - megatron_path=args.megatron_path, - extra_env_vars=extra_env_vars, - ) - - -@U.dataclass_cli -def main(args: ScriptArgs): - cleanup() - if not args.skip_prepare: - prepare(args) - execute(args) - - -if __name__ == "__main__": - typer.run(main)