diff --git a/.github/workflows/publish-envs.yml b/.github/workflows/publish-envs.yml index abcfef09d7..1925ae69bc 100644 --- a/.github/workflows/publish-envs.yml +++ b/.github/workflows/publish-envs.yml @@ -21,8 +21,9 @@ jobs: - name: Set matrix id: set-matrix run: | - # Get all environment directories - ENV_IDS=$(ls -d environments/*/ 2>/dev/null | xargs -n1 basename | jq -R . | jq -sc .) + # Auto-publish only the classic (v0) envs for now; v1 envs (the *_v1 packages + # and the `compact` harness example) aren't hub-published yet. + ENV_IDS=$(ls -d environments/*/ 2>/dev/null | xargs -n1 basename | { grep -vE '_v1$|^compact$' || true; } | jq -R . | jq -sc .) if [ "$ENV_IDS" = "[]" ] || [ "$ENV_IDS" = "null" ]; then echo "has_envs=false" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-harnesses.yml b/.github/workflows/publish-harnesses.yml new file mode 100644 index 0000000000..b035bbd363 --- /dev/null +++ b/.github/workflows/publish-harnesses.yml @@ -0,0 +1,210 @@ +name: Publish harnesses + +on: + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to release (e.g. harnesses-v0.1.1)' + required: true + type: string + push: + branches: + - main + tags: + - "harnesses-v*" + +jobs: + auto-tag-on-main: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + created: ${{ steps.tag.outputs.created }} + tag: ${{ steps.tag.outputs.tag }} + version: ${{ steps.tag.outputs.version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create release tag for untagged version + id: tag + run: | + echo "created=false" >> "$GITHUB_OUTPUT" + + VERSION=$(python - <<'PY' + import tomllib + from pathlib import Path + import sys + + data = tomllib.loads(Path("packages/harnesses/pyproject.toml").read_text()) + version = data.get("project", {}).get("version") + if not version: + sys.exit("Could not find [project].version in packages/harnesses/pyproject.toml") + print(version) + PY + ) + + TAG="harnesses-v${VERSION}" + + if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + echo "Tag ${TAG} already exists locally; skipping." + exit 0 + fi + + if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + echo "Tag ${TAG} already exists on origin; skipping." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + + echo "created=true" >> "$GITHUB_OUTPUT" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + build-from-auto-tag: + needs: auto-tag-on-main + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.auto-tag-on-main.outputs.created == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + tag: ${{ needs.auto-tag-on-main.outputs.tag }} + version: ${{ needs.auto-tag-on-main.outputs.version }} + steps: + - name: Checkout auto-created tag + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: refs/tags/${{ needs.auto-tag-on-main.outputs.tag }} + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Build harnesses + run: uv build packages/harnesses + + - name: Upload dist artifacts + uses: actions/upload-artifact@v4 + with: + name: harnesses-dist + path: packages/harnesses/dist/ + if-no-files-found: error + retention-days: 7 + + publish-from-auto-tag: + needs: build-from-auto-tag + runs-on: ubuntu-latest + environment: pypi-prod + permissions: + id-token: write + steps: + - name: Download dist artifacts + uses: actions/download-artifact@v4 + with: + name: harnesses-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + + build-tag: + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/harnesses-v') + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + tag: ${{ steps.release.outputs.tag }} + version: ${{ steps.release.outputs.version }} + steps: + - name: Checkout tagged release (dispatch) + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: refs/tags/${{ inputs.tag }} + + - name: Checkout tagged release (push) + if: github.event_name != 'workflow_dispatch' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve release tag + id: release + env: + EVENT_NAME: ${{ github.event_name }} + PUSHED_REF: ${{ github.ref_name }} + INPUT_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || '' }} + run: | + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + TAG="$INPUT_TAG" + else + TAG="$PUSHED_REF" + fi + + case "$TAG" in + harnesses-v*) ;; + *) + echo "Release tags must be prefixed with 'harnesses-v' (received '$TAG')" >&2 + exit 1 + ;; + esac + + VERSION="${TAG#harnesses-v}" + FILE_VERSION=$(python - <<'PY' + import tomllib + from pathlib import Path + import sys + + data = tomllib.loads(Path("packages/harnesses/pyproject.toml").read_text()) + version = data.get("project", {}).get("version") + if not version: + sys.exit("Could not find [project].version in packages/harnesses/pyproject.toml") + print(version) + PY + ) + + if [ "$FILE_VERSION" != "$VERSION" ]; then + echo "Version mismatch: tag requests '$VERSION' but packages/harnesses/pyproject.toml defines '$FILE_VERSION'" >&2 + exit 1 + fi + + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Build harnesses + run: uv build packages/harnesses + + - name: Upload dist artifacts + uses: actions/upload-artifact@v4 + with: + name: harnesses-dist + path: packages/harnesses/dist/ + if-no-files-found: error + retention-days: 7 + + publish-tag: + needs: build-tag + runs-on: ubuntu-latest + environment: pypi-prod + permissions: + id-token: write + steps: + - name: Download dist artifacts + uses: actions/download-artifact@v4 + with: + name: harnesses-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/publish-tasksets.yml b/.github/workflows/publish-tasksets.yml new file mode 100644 index 0000000000..01d1085bb9 --- /dev/null +++ b/.github/workflows/publish-tasksets.yml @@ -0,0 +1,210 @@ +name: Publish tasksets + +on: + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to release (e.g. tasksets-v0.1.1)' + required: true + type: string + push: + branches: + - main + tags: + - "tasksets-v*" + +jobs: + auto-tag-on-main: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + created: ${{ steps.tag.outputs.created }} + tag: ${{ steps.tag.outputs.tag }} + version: ${{ steps.tag.outputs.version }} + steps: + - name: Checkout main + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create release tag for untagged version + id: tag + run: | + echo "created=false" >> "$GITHUB_OUTPUT" + + VERSION=$(python - <<'PY' + import tomllib + from pathlib import Path + import sys + + data = tomllib.loads(Path("packages/tasksets/pyproject.toml").read_text()) + version = data.get("project", {}).get("version") + if not version: + sys.exit("Could not find [project].version in packages/tasksets/pyproject.toml") + print(version) + PY + ) + + TAG="tasksets-v${VERSION}" + + if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + echo "Tag ${TAG} already exists locally; skipping." + exit 0 + fi + + if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + echo "Tag ${TAG} already exists on origin; skipping." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + + echo "created=true" >> "$GITHUB_OUTPUT" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + build-from-auto-tag: + needs: auto-tag-on-main + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.auto-tag-on-main.outputs.created == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + tag: ${{ needs.auto-tag-on-main.outputs.tag }} + version: ${{ needs.auto-tag-on-main.outputs.version }} + steps: + - name: Checkout auto-created tag + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: refs/tags/${{ needs.auto-tag-on-main.outputs.tag }} + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Build tasksets + run: uv build packages/tasksets + + - name: Upload dist artifacts + uses: actions/upload-artifact@v4 + with: + name: tasksets-dist + path: packages/tasksets/dist/ + if-no-files-found: error + retention-days: 7 + + publish-from-auto-tag: + needs: build-from-auto-tag + runs-on: ubuntu-latest + environment: pypi-prod + permissions: + id-token: write + steps: + - name: Download dist artifacts + uses: actions/download-artifact@v4 + with: + name: tasksets-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + + build-tag: + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/tasksets-v') + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + tag: ${{ steps.release.outputs.tag }} + version: ${{ steps.release.outputs.version }} + steps: + - name: Checkout tagged release (dispatch) + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: refs/tags/${{ inputs.tag }} + + - name: Checkout tagged release (push) + if: github.event_name != 'workflow_dispatch' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Resolve release tag + id: release + env: + EVENT_NAME: ${{ github.event_name }} + PUSHED_REF: ${{ github.ref_name }} + INPUT_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || '' }} + run: | + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + TAG="$INPUT_TAG" + else + TAG="$PUSHED_REF" + fi + + case "$TAG" in + tasksets-v*) ;; + *) + echo "Release tags must be prefixed with 'tasksets-v' (received '$TAG')" >&2 + exit 1 + ;; + esac + + VERSION="${TAG#tasksets-v}" + FILE_VERSION=$(python - <<'PY' + import tomllib + from pathlib import Path + import sys + + data = tomllib.loads(Path("packages/tasksets/pyproject.toml").read_text()) + version = data.get("project", {}).get("version") + if not version: + sys.exit("Could not find [project].version in packages/tasksets/pyproject.toml") + print(version) + PY + ) + + if [ "$FILE_VERSION" != "$VERSION" ]; then + echo "Version mismatch: tag requests '$VERSION' but packages/tasksets/pyproject.toml defines '$FILE_VERSION'" >&2 + exit 1 + fi + + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Build tasksets + run: uv build packages/tasksets + + - name: Upload dist artifacts + uses: actions/upload-artifact@v4 + with: + name: tasksets-dist + path: packages/tasksets/dist/ + if-no-files-found: error + retention-days: 7 + + publish-tag: + needs: build-tag + runs-on: ubuntu-latest + environment: pypi-prod + permissions: + id-token: write + steps: + - name: Download dist artifacts + uses: actions/download-artifact@v4 + with: + name: tasksets-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/COMPARE.md b/COMPARE.md deleted file mode 100644 index b1b5a490e6..0000000000 --- a/COMPARE.md +++ /dev/null @@ -1,46 +0,0 @@ -# v1 PRs: #1559 vs #1576 - -Two open v1 refactors of verifiers (the `Taskset × Harness × Runtime` model), both against `main`. - -| | **#1559** — `codex/v1-nano-refactor-draft` | **#1576** — `feat/nano-as-v1` | -|---|---|---| -| Size | +17.6k / −34.1k, 422 files | +11.5k / −50k, 401 files (re-vendors vf-nano) | -| Thesis | Broad v1 surface — many harnesses, in-tree advantages, nested subagents | v0↔v1 bridge + training-readiness — legacy bridge, message graph, multiplexing, benchmarked | -| Rollout record | `State` + flat `Turn` list (serializable, no graph) | delta-native `MessageNode` graph (branches via leaves→root) | -| RL contract | token-level **advantages computed in-lib** (`@advantage`) | trainer (prime-rl) computes advantages; lib exposes trainable `Trace` | - -## Parity — supported by both - -- Core `Taskset × Harness × Runtime` over a typed (pydantic) rollout model -- Runtimes: **subprocess, docker, prime** -- Harnesses: a **default chat harness** + **rlm** -- Taskset authoring: `load_tasks` + `@reward` / `@metric` / `@stop`, **group rewards**, **runtime-based (in-sandbox) scoring**, per-task **image + resources** -- **MCP tool servers** exposed to the model + a first-class **user simulator** (framework-injected user turns) -- **Eval CLI + TOML config** (runtime/harness selected by config) -- **Trainable rollouts** — per-turn token ids + logprobs + mask -- **Interception server** proxying model calls; SIGTERM → graceful teardown -- v1 **unit tests** + live **eval reward** checks - -## Only in #1559 - -- Harness ecosystem: **`CommandHarness`** (agentic-CLI base) + **MiniSWEAgent / OpenCode / Pi / Terminus2 / Replay / NeMoGym** -- **In-tree token-level advantages** — `@advantage` (grpo / rloo / reinforce / sft) writing `Turn.tokens.*_advantages`; `advantage=None` defers to a trainer -- **Nested harnesses / subagents** — `Harness.run(context=parent)` reuses the parent's runtime, clients, toolsets -- **Richer MCP** — placement `dedicated` / `colocated` / `remote` × scope `rollout` / `env` (refcounted, start-once) + **bound-arg tools** (`args`/`sets`/`extends`) that hide state plumbing from the model -- **Replay harness** (SFT) - -## Only in #1576 - -- **Legacy v0 bridge** (`LegacyEnvServer`) — runs classic v0 envs over the **same ZMQ protocol** as native v1, indistinguishable to the trainer; token ids/logprobs carried 1:1; group scoring; eval-split fallback; renderer (train) vs chat-completions (eval) client dispatch. *The headline ("nano bridge").* -- **Delta-native message graph** — each message stored once, branches recovered leaves→root (linear, not quadratic in turns); one training sample per branch -- **Interception multiplexing** (`InterceptionPool`, `multiplex=32`) — N rollouts share servers + tunnels, to beat prime's 512/min tunnel cap -- **ZMQ env server is the v1 training path** (native + bridge both serve over it); prime-rl drives it [#1559's ZMQ is v0-only; its v1 eval runs in-process] -- **Modal runtime functional** (4 working runtimes vs 3 + 2 stubs) -- Framework-enforced limits (`max_turns` / token budgets / `@stop`) applied **harness-agnostically** in the interception server -- **Runtime + multiplex benchmark** (`bench/`) with committed numbers - -> Excluded from #1576's tip via reverts (in separate review — verifiers#1618): multimodal/VLM, user-sim colocation, color-codeword taskset. - ---- - -*Net:* **#1559** is the broader feature surface (harness ecosystem, in-lib advantages, nested subagents, richer MCP). **#1576** is narrower but is the only one that bridges v0→v1. diff --git a/bench/.gitignore b/bench/.gitignore deleted file mode 100644 index e33609d251..0000000000 --- a/bench/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.png diff --git a/bench/agentic_benchmark.sh b/bench/agentic_benchmark.sh deleted file mode 100755 index 39cf974e79..0000000000 --- a/bench/agentic_benchmark.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# Agentic benchmark: run ONE harbor task at group sizes (-r) across env-server modes and -# write bench/agentic_benchmark.json (per-rollout durations + e2e wall clock), which -# bench/agentic_aggregate.py summarizes. Each rollout is its own sandbox (a coding agent + -# the harbor verifier); with no group reward the rollouts are independent, so the worker -# pool round-robins them across workers — this stresses concurrent agentic execution + -# scoring (where the single-loop server is most likely to stall). -# -# bench/agentic_benchmark.sh -# ROLLOUTS="8 16" WORKERS="0 4" TASK=fix-git bench/agentic_benchmark.sh -# -# Compares WORKERS modes: 0 = single in-process server, N = an N-worker pool. Needs the -# `harbor` CLI (`uv tool install harbor`) and the `terminal-bench-2-v1` example taskset -# (an editable dep), plus a container runtime (prime default; PRIME_API_KEY in ~/.env). -set -uo pipefail - -TASKSET="${TASKSET:-terminal-bench-2-v1}" -TASK="${TASK:-fix-git}" -RUNTIME="${RUNTIME:-prime}" -ROLLOUTS="${ROLLOUTS:-32 64 128}" -WORKERS="${WORKERS:-0 4}" -MODEL="${MODEL:-deepseek/deepseek-v4-flash}" -MAX_TURNS="${MAX_TURNS:-30}" - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" -set -a; . "$HOME/.env" 2>/dev/null || true; set +a - -OUT="/tmp/vbench/agentic" -rm -rf "$OUT"; mkdir -p "$OUT"; : > "$OUT/e2e.txt" -for w in $WORKERS; do - for r in $ROLLOUTS; do - label="w$w-r$r" - echo "== $label (task=$TASK runtime=$RUNTIME max_turns=$MAX_TURNS) ==" - start=$(date +%s) - uv run eval "$TASKSET" --taskset.tasks "[\"$TASK\"]" \ - --harness.id default --harness.enable_bash true --harness.runtime.type "$RUNTIME" \ - --num_tasks 1 --num_rollouts "$r" --num_workers "$w" \ - --max_concurrent 512 --retry.attempts 1 --max_turns "$MAX_TURNS" \ - --rich false --output_dir "$OUT/$label" \ - > "$OUT/$label.stdout" 2> "$OUT/$label.log" - rc=$? - echo "$w $r $(( $(date +%s) - start ))" >> "$OUT/e2e.txt" - echo "rc=$rc e2e=$(tail -1 "$OUT/e2e.txt" | awk '{print $3}')s" - done -done - -# Aggregate into agentic_benchmark.json: per-(workers, rollouts) e2e + the per-rollout -# generation-duration list (p10/p50/p90), reward, and error count. -uv run python bench/bench_aggregate.py "$OUT" "$TASK ($RUNTIME, max_turns=$MAX_TURNS)" > "$OUT/agentic_benchmark.json" -echo "wrote $OUT/agentic_benchmark.json" diff --git a/bench/bench_aggregate.py b/bench/bench_aggregate.py deleted file mode 100644 index 854be00a74..0000000000 --- a/bench/bench_aggregate.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Aggregate a worker-pool benchmark run (single-turn or agentic) into JSON. - - python bench/bench_aggregate.py