diff --git a/.github/workflows/reusable-dispatch.yml b/.github/workflows/reusable-dispatch.yml index 37f4bd6f72..b7670afdde 100644 --- a/.github/workflows/reusable-dispatch.yml +++ b/.github/workflows/reusable-dispatch.yml @@ -640,6 +640,9 @@ jobs: TRIAGE_TARGET_REPO_DIR: target-repo TRIAGE_ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} TRIAGE_CLOUD_ML_REGION: ${{ inputs.gcp_region }} + # Per-run runtime/model/effort overrides from repository variables + # (FULLSEND_MODEL, TRIAGE_FULLSEND_MODEL, ...); see docs/runtimes.md. + FULLSEND_REPO_VARS: ${{ toJSON(vars) }} run: bash .github/scripts/setup-agent-env.sh - name: Run triage agent @@ -755,6 +758,9 @@ jobs: CODE_ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} CODE_CLOUD_ML_REGION: ${{ inputs.gcp_region }} CODE_ISSUE_NUMBER: ${{ fromJSON(needs.route.outputs.event_payload).issue.number }} + # Per-run runtime/model/effort overrides from repository variables + # (FULLSEND_MODEL, CODE_FULLSEND_MODEL, ...); see docs/runtimes.md. + FULLSEND_REPO_VARS: ${{ toJSON(vars) }} run: bash .github/scripts/setup-agent-env.sh - name: Run code agent @@ -877,6 +883,9 @@ jobs: REVIEW_TARGET_REPO_DIR: target-repo REVIEW_ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} REVIEW_CLOUD_ML_REGION: ${{ inputs.gcp_region }} + # Per-run runtime/model/effort overrides from repository variables + # (FULLSEND_MODEL, REVIEW_FULLSEND_MODEL, ...); see docs/runtimes.md. + FULLSEND_REPO_VARS: ${{ toJSON(vars) }} run: bash .github/scripts/setup-agent-env.sh - name: Run review agent @@ -1143,6 +1152,9 @@ jobs: FIX_HUMAN_INSTRUCTION: ${{ steps.context.outputs.instruction }} FIX_FIX_ITERATION: ${{ steps.context.outputs.iteration }} FIX_REPO_FULL_NAME: ${{ github.repository }} + # Per-run runtime/model/effort overrides from repository variables + # (FULLSEND_MODEL, FIX_FULLSEND_MODEL, ...); see docs/runtimes.md. + FULLSEND_REPO_VARS: ${{ toJSON(vars) }} run: bash .github/scripts/setup-agent-env.sh - name: Run fix agent @@ -1246,6 +1258,9 @@ jobs: RETRO_TARGET_REPO_DIR: target-repo RETRO_ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} RETRO_CLOUD_ML_REGION: ${{ inputs.gcp_region }} + # Per-run runtime/model/effort overrides from repository variables + # (FULLSEND_MODEL, RETRO_FULLSEND_MODEL, ...); see docs/runtimes.md. + FULLSEND_REPO_VARS: ${{ toJSON(vars) }} run: bash .github/scripts/setup-agent-env.sh - name: Run retro agent @@ -1329,6 +1344,9 @@ jobs: PRIORITIZE_PROJECT_NUMBER: ${{ inputs.project_number }} PRIORITIZE_ANTHROPIC_VERTEX_PROJECT_ID: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} PRIORITIZE_CLOUD_ML_REGION: ${{ inputs.gcp_region }} + # Per-run runtime/model/effort overrides from repository variables + # (FULLSEND_MODEL, PRIORITIZE_FULLSEND_MODEL, ...); see docs/runtimes.md. + FULLSEND_REPO_VARS: ${{ toJSON(vars) }} run: bash .github/scripts/setup-agent-env.sh - name: Create empty target-repo directory @@ -1620,6 +1638,9 @@ jobs: MATRIX_ROLE: ${{ matrix.role }} GCP_PROJECT: ${{ secrets.FULLSEND_GCP_PROJECT_ID }} GCP_REGION: ${{ inputs.gcp_region }} + # Per-run runtime/model/effort overrides from repository variables + # (FULLSEND_MODEL, _FULLSEND_MODEL, ...); see docs/runtimes.md. + FULLSEND_REPO_VARS: ${{ toJSON(vars) }} run: | set -euo pipefail ROLE_UPPER=$(echo "${MATRIX_ROLE}" | tr '[:lower:]' '[:upper:]') diff --git a/Makefile b/Makefile index 195a2359f5..18cc242fd8 100644 --- a/Makefile +++ b/Makefile @@ -193,6 +193,7 @@ script-test: $(call run-timed,bash .github/scripts/check-fix-eligibility-test.sh) $(call run-timed,bash internal/scaffold/fullsend-repo/scripts/reconcile-repos-test.sh) $(call run-timed,bash internal/scaffold/fullsend-repo/scripts/pre-fetch-prior-review-test.sh) + $(call run-timed,bash internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env-test.sh) $(call run-timed,bash hack/gitlab-runner-vm/executor/prepare_validation_test.sh) $(call run-timed,python3 skills/topissues/scripts/topissues_test.py) $(call run-timed,python3 skills/nextwork/scripts/nextwork_test.py) diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 1917bc0675..57574e1f42 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -185,6 +185,7 @@ export default defineConfig({ { text: "fullsend inference", link: "/cli/inference" }, { text: "fullsend mint", link: "/cli/mint" }, { text: "fullsend repos", link: "/cli/repos" }, + { text: "fullsend run", link: "/cli/run" }, ], }, ], @@ -195,12 +196,17 @@ export default defineConfig({ link: "/guides/getting-started/", items: [ { text: "Getting Inference", link: "/guides/getting-started/getting-inference" }, + { text: "Choose a Runtime", link: "/guides/getting-started/choosing-a-runtime" }, { text: "Configuring GitHub", link: "/guides/getting-started/configuring-github" }, { text: "Per-Org Mode", link: "/guides/getting-started/org-mode" }, { text: "Repo Management", link: "/guides/getting-started/repo-management" }, { text: "Operations", link: "/guides/getting-started/operations" }, ], }, + { + text: "Runtimes", + link: "/runtimes", + }, { text: "Agents", collapsed: true, @@ -244,7 +250,6 @@ export default defineConfig({ items: [ { text: "Vision", link: "/vision" }, { text: "Architecture", link: "/architecture" }, - { text: "Runtimes", link: "/runtimes" }, { text: "Glossary", link: "/glossary" }, ], }, diff --git a/docs/cli/README.md b/docs/cli/README.md index d049d7d99a..a5786b222f 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -24,7 +24,7 @@ Download the latest binary from [GitHub Releases](https://github.com/fullsend-ai | Command | Description | |---------|-------------| -| `fullsend run` | Execute an agent locally in a sandbox. See [running agents locally](../guides/user/running-agents-locally.md). | +| [`fullsend run`](run.md) | Execute an agent locally in a sandbox. See [running agents locally](../guides/user/running-agents-locally.md). | | `fullsend lock [agent-name]` | Pin remote dependencies to `lock.yaml` | | `fullsend scan` | Run security scanners on agent input/output | | `fullsend eval-measure` | Score wild-run traces into `eval-measurements.jsonl`. See [Eval measurements](../guides/infrastructure/eval-measurements.md). | diff --git a/docs/cli/repos.md b/docs/cli/repos.md index bed205fffc..e1097299dc 100644 --- a/docs/cli/repos.md +++ b/docs/cli/repos.md @@ -87,7 +87,7 @@ required in this case. This enables a greenfield setup without running Runs in three phases: -1. **Manifest add** — repos specified as positional arguments that are not already in the manifest are added (`--forge` is required when the target platform cannot be inferred). Per-repo overrides (`--inference-region`, `--fullsend-ref`, `--mint-url`, `--allowed-remote-resources`) are written to the manifest entry. +1. **Manifest add** — repos specified as positional arguments that are not already in the manifest are added (`--forge` is required when the target platform cannot be inferred). Per-repo overrides (`--inference-region`, `--fullsend-ref`, `--mint-url`, `--allowed-remote-resources`, `--runtime`) are written to the manifest entry. 2. **Provision** — repos in the manifest that are not yet provisioned are installed (scaffold files, variables, secrets). Repos with a guard variable set but other components missing are repaired automatically. 3. **Convergence** — repos that are already installed are checked for component drift (workflow, thin callers, variables, secrets — repaired automatically) and scaffold ref drift (upgraded automatically). @@ -118,6 +118,7 @@ When repos are specified as positional arguments, only those repos are processed | `--fullsend-ref` | | Per-repo fullsend workflow ref override | | `--mint-url` | | Per-repo mint URL override | | `--allowed-remote-resources` | | Per-repo allowed remote resources override | +| `--runtime` | | Agent runtime (`claude`, `pi`) recorded for repos this command adds; existing entries keep their `runtime` / `defaults.runtime` | | `--gitlab-bot-token` | | GitLab bot PAT for free-tier instances that don't support project access tokens (env: `FULLSEND_GITLAB_BOT_TOKEN`) | ### GitLab bot token @@ -255,6 +256,7 @@ fullsend repos set-default github.mint_url "" # removes the key | Key | Type | Description | |-----|------|-------------| | `defaults.allowed_remote_resources` | comma-separated URLs | HTTPS URLs agents may fetch at runtime | +| `defaults.runtime` | `claude` or `pi` | Agent runtime written as each repo's `runtime:` at install; a per-entry `runtime` overrides it (`none` stops the chain) | | `github.url` | URL | GitHub instance URL (default: `https://github.com`) | | `github.mint_url` | URL | Token mint service URL (defaults to `https://mint.fullsend.sh` in public mode) | | `github.mint_mode` | `public` or `private` | Controls the default mint URL: `public` defaults to `https://mint.fullsend.sh`; `private` requires an explicit `mint_url` (default: `public`) | diff --git a/docs/cli/run.md b/docs/cli/run.md new file mode 100644 index 0000000000..5a38af3d4b --- /dev/null +++ b/docs/cli/run.md @@ -0,0 +1,89 @@ +--- +sidebar_label: fullsend run +--- + +# fullsend run + +Execute an agent locally in a sandbox. `fullsend run` resolves the agent harness, provisions a sandbox container, and runs the agent to completion. + +## Usage + +```bash +fullsend run [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--fullsend-dir` | Path to the `.fullsend` configuration directory | +| `--runtime` | Override the agent runtime from `config.yaml` for this run (`claude`, `pi`, `dummy`); also `FULLSEND_RUNTIME` | +| `--model` | Override the harness/agent model for this run (alias, model id, or `provider/id` on pi); also `FULLSEND_MODEL` | +| `--effort` | Override the harness effort level for this run (`low`…`max`); also `FULLSEND_EFFORT` | +| `--output-dir` | Base directory for run output (default: `/tmp/fullsend`) | +| `--target-repo` | Path to the target repository | +| `--fullsend-binary` | Path to a Linux fullsend binary to copy into the sandbox | +| `--env-file` | Load environment variables from a dotenv file (repeatable) | +| `--no-post-script` | Skip post-script execution | +| `--keep-sandbox` | Skip sandbox deletion after the run | +| `--debug [filter]` | Enable agent runtime debug logging with optional category filter (e.g. `"api,hooks"`) | +| `--forge` | Forge platform to use (e.g. `"github"`, `"gitlab"`); auto-detected from CI env vars when omitted | +| `--offline` | Reject network fetches; only use cached remote resources | +| `--max-depth` | Maximum dependency depth for transitive resolution (0 disables) | + +## Plan block + +At startup, `fullsend run` prints a plan block summarizing the resolved configuration: + +``` +Agent: code +Role: code +Model: sonnet +Effort: high +Runtime: claude (from /path/to/.fullsend/config.yaml) +Image: fullsend-sandbox:latest +``` + +The **Runtime** line shows which runtime was selected and the config source it was read from. When no `config.yaml` exists, the source reads `default (config not found)`. + +## Runtime selection + +The runtime for a run is resolved once, in this order: `--runtime` flag, `FULLSEND_RUNTIME`, the per-repo `runtime:` in `config.yaml` / `.fullsend/config.yaml`, then the built-in `claude`. The same order applies to the model (`--model`, `FULLSEND_MODEL`, harness `model:`, agent frontmatter; `FULLSEND_PI_MODEL` is a lower-precedence alias on pi) and to effort (`--effort`, `FULLSEND_EFFORT`, harness `effort:`). `FULLSEND_FALLBACK_MODELS=a,b` becomes Claude Code's `--fallback-model`; pi ignores it with a warning. + +The plan block prints `Runtime: (from )` and, when an override applied, `Model: (from )`; stderr carries `runtime: selected "" from ` (and `model: requested "" from `) for scripts. An invalid override (unknown runtime, unknown effort level) fails before the sandbox is created. + +```bash +# try a repo's triage on pi with Gemini Flash, without touching its config +fullsend run triage --fullsend-dir . --target-repo ../repo \ + --runtime pi --model google-vertex/gemini-2.5-flash --effort medium +``` + +## Output artifacts + +Each run produces artifacts in the output directory: + +| File | Description | +|------|-------------| +| `metrics.json` | Behavioral metrics: tokens, cost, model, runtime, iterations | +| `transcripts/` | Agent conversation transcripts | +| `claude-debug.log` or `pi-debug.log` | Debug log (when `--debug` is set) | + +### metrics.json fields + +| Field | Description | +|-------|-------------| +| `runtime` | Runtime that executed the run (e.g. `claude`, `pi`) | +| `model` | Model the provider reported using | +| `requested_runtime` | Runtime selected for the run (config file, or a `--runtime`/`FULLSEND_RUNTIME` override) | +| `requested_model` | Model the harness/agent requested | +| `override_source` | Where `requested_model` came from (`--model flag`, `FULLSEND_MODEL`, `FULLSEND_PI_MODEL`, `harness`, `default`) | +| `runtime_source` | Where `requested_runtime` came from (`--runtime flag`, `FULLSEND_RUNTIME`, the config file path, or `default (config not found)`) | +| `total_cost_usd` | Total inference cost | +| `num_turns` | Number of conversation turns | +| `iterations` | Number of retry iterations | + +## Related + +- [Running Agents Locally](../guides/user/running-agents-locally.md) for a step-by-step walkthrough +- [Runtimes](../runtimes.md) for runtime selection and capabilities +- [CLI internals](../guides/dev/cli-internals.md) for the full command tree diff --git a/docs/contributing/documentation.md b/docs/contributing/documentation.md index 860e38f24a..96d449750c 100644 --- a/docs/contributing/documentation.md +++ b/docs/contributing/documentation.md @@ -83,7 +83,7 @@ The `admin` command group's `install`/`uninstall`/`analyze`/`enable`/`disable` s | Category | Files | |----------|-------| -| CLI reference | _(no dedicated page)_ | +| CLI reference | `docs/cli/run.md` | | Guides | `docs/guides/user/running-agents-locally.md`, `docs/guides/user/building-custom-agents.md`, `docs/guides/dev/cli-internals.md` | | ADRs | `docs/ADRs/0036-agent-execution-sandbox.md` | | Contributing | `docs/contributing/sandbox-topology.md` | diff --git a/docs/guides/getting-started/README.md b/docs/guides/getting-started/README.md index 7971bc2bf1..67baee2baa 100644 --- a/docs/guides/getting-started/README.md +++ b/docs/guides/getting-started/README.md @@ -8,5 +8,6 @@ This section contains easy and to the point guides to help you set up Fullsend. These are intended to be read in a certain order: 1. [Getting Inference](getting-inference.md) -2. [Configuring GitHub](configuring-github.md) -3. [Organization Mode](org-mode.md) +2. [Choose an agent runtime](choosing-a-runtime.md) +3. [Configuring GitHub](configuring-github.md) +4. [Organization Mode](org-mode.md) diff --git a/docs/guides/getting-started/choosing-a-runtime.md b/docs/guides/getting-started/choosing-a-runtime.md new file mode 100644 index 0000000000..0f4014b8af --- /dev/null +++ b/docs/guides/getting-started/choosing-a-runtime.md @@ -0,0 +1,38 @@ +--- +sidebar_label: Choose a Runtime +--- + +# Choose an agent runtime + +> **Claude Code is the stable default.** The fleet agents have run on Claude Code in production for a long time; it is what a new installation gets unless you ask for something else. **pi is in its enablement (experimental) phase** — it works end to end for `triage`, `prioritize`, `code` and `fix`, has no sub-agent tool yet (`review`/`retro` run in a single context), and its fleet pilot is still in progress. Unless you are taking part in that pilot, keep the default. + +This page explains what the choice means and where it is made. **You do not select anything on this page** — the selection happens in the next step, [Configuring GitHub](configuring-github.md), when `fullsend github setup` prompts for the runtime (press Enter for `claude`) or when you pass `--runtime`. + +Fullsend supports multiple agent runtimes. A runtime is the program that runs inside the sandbox and drives the model — it owns the tool-use loop, hook wiring, and transcript format. The runner (fullsend) owns everything outside: sandbox lifecycle, credentials, metrics, and the verdict. + +## Available runtimes + +| Runtime | Status | Description | When to use | +|---------|--------|-------------|-------------| +| `claude` | **Stable (default)** | Claude Code on Vertex AI | Every production deployment — mature, full sub-agent support for `review`/`retro` | +| `pi` | Experimental (enablement phase) | [Pi](https://github.com/earendil-works/pi) — Claude on Vertex by default; any provider pi supports by model name (e.g. Gemini on Vertex with the same credentials) | Opt-in pilots only; no sub-agent tool yet, so `review`/`retro` run single-context; see [Runtimes](../../runtimes.md) for known constraints | + +## When and how the runtime is selected + +1. **Next step — Configuring GitHub.** `fullsend github setup ` asks which runtime to use when run from a terminal; press Enter to keep `claude`. Passing `--runtime` skips the prompt. The setup PR it opens records the choice in `.fullsend/config.yaml` and describes how to change it. Nothing runs on this page — continue with [Configuring GitHub](configuring-github.md). +2. **Later — changing it.** Edit `runtime:` in the repo's `.fullsend/config.yaml` (the setup PR shows the key), or re-run `fullsend github setup --runtime `. Fleets managed through `repos.yaml` set `defaults.runtime` (or a per-entry `runtime`) — `fullsend repos set-default defaults.runtime pi` — and run `fullsend repos install`; see [fullsend repos](../../cli/repos.md). +3. **Per run — trying without changing the repo.** `fullsend run --runtime pi --model google-vertex/gemini-2.5-flash`, or the `FULLSEND_RUNTIME` / `FULLSEND_MODEL` / `FULLSEND_EFFORT` environment variables (flag beats environment beats config). In CI the same names work as repository variables. Reference: [fullsend run](../../cli/run.md) and [Runtimes — selecting and overriding](../../runtimes.md#selecting-and-overriding). + +## Where to see what ran + +After a run completes, the selected runtime and model appear in several places: + +- **Run plan block** — `Runtime: (from )` printed at the start of every `fullsend run` +- **Status comment** — the terminal status comment on the issue/PR includes a footer with runtime, model, effort, and cost +- **metrics.json** — `runtime`, `requested_runtime`, `runtime_source`, `requested_model`, and `override_source` fields record what was selected and why +- **stderr** — `runtime: selected "" from ` for script consumers + +## Next steps + +- [Configuring GitHub](configuring-github.md) to set up your repo +- [Runtimes](../../runtimes.md) for the full runtime reference, including model override precedence and the capability table diff --git a/docs/guides/getting-started/configuring-github.md b/docs/guides/getting-started/configuring-github.md index 1c3371f2cd..63cb1f017d 100644 --- a/docs/guides/getting-started/configuring-github.md +++ b/docs/guides/getting-started/configuring-github.md @@ -112,6 +112,8 @@ fetched content before committing. > because the preset provides its own configuration. `--config` is only > valid for per-repo mode. +This is where the agent runtime is selected: on a terminal, `fullsend github setup` asks once (press Enter to keep `claude`, the stable default); `--runtime` sets it explicitly. `pi` is experimental and meant for the enablement pilot — see [Choose a Runtime](choosing-a-runtime.md) for what the runtimes are and how to change the selection after setup. + ## Testing Fullsend After installing open a new issue or comment `/fs-triage` in an open issue. Then visit the diff --git a/docs/guides/user/running-agents-locally.md b/docs/guides/user/running-agents-locally.md index c2720aaef9..6b7106e60c 100644 --- a/docs/guides/user/running-agents-locally.md +++ b/docs/guides/user/running-agents-locally.md @@ -572,7 +572,9 @@ Stop reasons: toolUse=2, stop=1 | Variable | Description | |----------|-------------| -| `FULLSEND_PI_MODEL` | Override the model id (runner env) | +| `FULLSEND_MODEL` (or `fullsend run --model`) | Override the model for the run on any runtime; `FULLSEND_PI_MODEL` is kept as a pi-only alias | +| `FULLSEND_RUNTIME` (or `--runtime`) | Override the runtime selected by `config.yaml` | +| `FULLSEND_EFFORT` (or `--effort`) | Override the harness effort level | | `FULLSEND_PI_PROVIDER` | Override the inference provider (runner env) | | `FULLSEND_PI_BASH_ALLOWLIST` | Set to `enforce` to make the Bash first-token allowlist block instead of warn | diff --git a/docs/runtimes.md b/docs/runtimes.md index ab79fa9ec0..b802ee2053 100644 --- a/docs/runtimes.md +++ b/docs/runtimes.md @@ -135,7 +135,7 @@ Harness keys are runtime-neutral in the YAML but each runtime owns their transla | Harness key | Claude Code | OpenCode (stub) | Pi | Dummy | Notes for new runtimes | |-------------|-------------|-----------------|-----------|-------|------------------------| -| `model` | `--model` (identity; aliases like `opus` resolved by the CLI) | — | alias table `opus\|sonnet\|haiku` → pi 0.84.2 catalog ids (`claude-opus-4-6`, `claude-sonnet-4-6`, `claude-haiku-4-5`), bare ids get the provider prefix (`anthropic-vertex` by default), `provider/id` passes through; overrides: `FULLSEND_PI_PROVIDER`, `FULLSEND_PI_MODEL` (runner env); harness `model:` wins over the agent frontmatter `model:`; see [Pi-specific known constraints](#pi-specific-known-constraints-6464) | ignored | `validModelName` is `^[a-zA-Z0-9_.@-]+$` — no `/`. Runtimes with `provider/model` ids need an alias table or a follow-up regex change | +| `model` | `--model` (identity; aliases like `opus` resolved by the CLI) | — | alias table `opus\|sonnet\|haiku` → pi 0.84.2 catalog ids (`claude-opus-4-6`, `claude-sonnet-4-6`, `claude-haiku-4-5`), bare ids get the provider prefix (`anthropic-vertex` by default), `provider/id` passes through; overrides: `--model`/`FULLSEND_MODEL` resolved by the CLI (`FULLSEND_PI_MODEL` is a lower-precedence alias on pi), `FULLSEND_PI_PROVIDER` for the prefix of bare ids; harness `model:` wins over the agent frontmatter `model:`; see [Pi-specific known constraints](#pi-specific-known-constraints-6464) | ignored | `validModelName` is `^[a-zA-Z0-9_.@-]+$` — no `/`. Runtimes with `provider/model` ids need an alias table or a follow-up regex change | | `effort` | `--effort` (`low\|medium\|high\|xhigh\|max`, #6218) | — | `--thinking ` (pi levels `off\|minimal\|low\|medium\|high\|xhigh\|max` ⊇ harness levels); unset or unknown → `--thinking high`, matching Claude Code's default effort on Vertex/API-key (pi's own default is `medium`, so the fleet agents — which set no `effort:` — would otherwise reason lower on pi); pi maps the level onto Anthropic adaptive effort and clamps it for models without reasoning | ignored | Map to the runtime's reasoning knob or reject with a clear error | | `plugins` | Claude plugin marketplace layout (`bootstrapPlugins`) | — | unsupported — `Bootstrap` warns and skips each plugin (pi uses TypeScript extensions, not plugins) | ignored | Claude-specific format; warn and skip if unsupported | | Agent frontmatter `tools:` (`Bash(gh,jq)` syntax, ADR 0027) | Native Claude permission syntax | — | `Bash/Read/Write/Edit/Grep/Glob/LS` → `--tools bash,read,write,edit,grep,find,ls` (strict pi allowlist); `Skill` maps to no tool but adds `read` (pi's skills are prompt-driven — the system prompt tells the model to read `SKILL.md`, and that section is only emitted when `read` is active; `read` is also added whenever the harness ships skills); other names warn and drop; `Bash(a,b)` becomes a first-token allowlist checked by the `fullsend-hooks.js` extension on every simple command — advisory by default (logged), matching Claude Code where it is steering rather than enforcement (ADR 0027); `FULLSEND_PI_BASH_ALLOWLIST=enforce` in the runner environment makes it block. Enforce mode is a first-token check, not a shell parser: it splits on `;`, `\|`/`\|&`, `&&`, `\|\|`, newlines and a backgrounding `&` (fd redirections such as `2>&1` are not separators) and checks each side; it refuses command substitution, subshells/groups, paths to binaries (unless the path itself is allowlisted), every `VAR=value` prefix (loader variables like `PATH=`/`LD_*`, but also program-specific ones like `GH_PAGER=` that make an allowlisted program spawn a command) and `eval`/`exec`/`sh`/`bash`/`source`/`command`/`env`/`xargs` wrappers; heredoc body lines are judged as if they were commands (in practice refused); redirections (`> /dev/tcp/…`) and an allowlisted program's own exec features (`gh extension exec`, `git -c core.pager=…`, `find -exec`) are not checked — egress is the sandbox's and the SSRF hook's job | ignored | Enforce via `--tools`/allowlist plus a hook adapter; Claude tool names differ in case from most runtimes (#608) | @@ -268,7 +268,7 @@ The `dummy` runtime executes a YAML script of operations inside the real sandbox | Credentials | same WIF `external_account` + refreshed OIDC token path; `ANTHROPIC_*` unset for the Vertex provider | | Unattended | no approval prompts; missing credential exits 1; stdin closed; bounded retries | | Artifacts | `output.jsonl`, `transcripts/-_.jsonl`, `metrics.json` with `runtime: pi`, `pi-debug.log` with `--debug`; `analyze-transcript` reads them | -| Knobs | `FULLSEND_PI_MODEL`, `FULLSEND_PI_PROVIDER`, `FULLSEND_PI_BASH_ALLOWLIST=enforce` | +| Knobs | `--runtime`/`--model`/`--effort` or `FULLSEND_RUNTIME`/`FULLSEND_MODEL`/`FULLSEND_EFFORT` (resolved once by the CLI; `FULLSEND_PI_MODEL` kept as an alias), `FULLSEND_PI_PROVIDER` (prefix for bare ids), `FULLSEND_PI_BASH_ALLOWLIST=enforce`; in CI the same names as repository variables, plain or role-prefixed | | Not yet | fleet lifecycle run on Vertex, sub-agents, Bedrock/Azure providers, `plugins:` | One iteration, end to end — the amber decision is what makes "hooks enabled" enforceable, since pi silently skips a missing `-e` extension: @@ -307,9 +307,66 @@ flowchart TB - **Agent definition translation** — the Claude-style agent `.md` is parsed by `Bootstrap`: body → `APPEND_SYSTEM.md` (pi's default prompt and tool guidance are kept; `SYSTEM.md` would replace them — a deliberate difference from Claude Code, whose `--agent` makes the body *the* system prompt; the lifecycle run should confirm the fleet prompts tolerate pi's preamble, otherwise switch to `--system-prompt`), frontmatter `tools:` → `--tools` (pi enforces this strictly, Claude Code ≥ 2.1.119 enforces it unreliably) + an advisory Bash allowlist, `model:` → fallback for the harness `model:`, `description` → header line. `metrics.json`/`InitEvent` carry the bare model id (`claude-opus-4-6`), as for Claude Code; the provider is `gen_ai.system`'s job. Everything `Run` and the hook extension need is in `fullsend-manifest.json` because `Bootstrap` and `Run` are separate calls with no shared process state. - **Hook adapter contract** — `fullsend-hooks.js` sends the scripts `{tool_name, tool_input, tool_result, tool_response}` with Claude tool names (`bash→Bash`, `read→Read`, `write→Write`, `edit→Edit`, `grep→Grep`, `find→Glob`, `ls→LS`; `path` mirrored to `file_path`) and reads back either the v1 `tool_result` or the v2 `hookSpecificOutput.updatedToolOutput` (#6357), so the same extension works before and after the PostToolUse chain lands. PreToolUse groups run in `HookPlan` order and stop at the first block; a script that cannot be spawned blocks; PostToolUse blocks withhold the result and mark it `isError`. An unreadable manifest, or one without a hook plan, blocks every tool call; because pi silently skips a missing `-e` path, `Run` checks — before sourcing the agent-writable `.env`, with `command -p sha256sum` / `command -p cut` so nothing in the shell environment can stand in for them — that the adapter exists and matches the embedded copy's SHA-256 and that the manifest exists, failing closed (exit 97) otherwise, refuses to start at all when security is enabled but the manifest carries no hook plan, and decides whether to load the adapter from the runner's security signal rather than the manifest. The manifest and the hook scripts themselves stay agent-writable between iterations — the same residue Claude Code has with `claude-config/hooks.json` and its scripts (both are written once at `Bootstrap`). Edit inputs keep pi's `edits[]` shape, with `path` mirrored to `file_path` and the first `oldText`/`newText` pair mirrored to `old_string`/`new_string`; no shipped script reads the latter. pi fires `tool_result` for failed calls too, so — unlike Claude Code's `PostToolUse` — errored tool output is sanitized as well. - **Exit code** — `Run` returns 1 when pi exited 0 but the stream's single `ResultEvent` reports an error (model error, incomplete stream), so the runner's exit-0 override and this agree; `ParseTranscriptFile` gives the same verdict from the tee'd `output.jsonl`. -- **Not yet exercised** — `runtime: pi` is selectable, but no fleet lifecycle run on Vertex has been recorded yet: the Vertex model ids and the copied `compat` flags have not been exercised against Vertex (smoke an adaptive and a non-adaptive model first; override with `FULLSEND_PI_MODEL` if an id is rejected); parser fixtures are hand-authored to the v0.84.2 wire docs (re-record with `internal/runtime/testdata/pi/regen.sh` once a run exists); `extension_error` events are not mapped; the behaviour scenario `features/runtime/pi.feature` (a real haiku run on Vertex of a minimal tool-using agent, asserting `metrics.json` `runtime: pi`, a `toolCall` in the pi session transcript and token usage) is gated on `BEHAVIOUR_CAPABILITIES=runtime-pi` until `fullsend-sandbox:latest` carries `PI_VERSION`, and `features/triage/triage.feature` asserts the runtime selected from the repo config on every run. Pilot on a disposable org with `triage`/`prioritize` (no sub-agent assumptions) before `code`/`fix`; `review`/`retro` rely on Claude sub-agent rosters and are not supported: pi v0.84.2 has no sub-agent tool or `agents/*.md` concept in core — only the bundled example extension (`examples/extensions/subagent/`, spawns `pi -p --mode json` children without our hook adapter, Vertex provider, `--no-approve` or session dir) and the SDK route (`createAgentSession()` per child; parent extensions do not fire for children) — so a fullsend-owned sub-agent extension with the full child flag set is a follow-up tracked on #6464. +- **Not yet exercised** — `runtime: pi` is selectable, but no fleet lifecycle run on Vertex has been recorded yet: the Vertex model ids and the copied `compat` flags have not been exercised against Vertex (smoke an adaptive and a non-adaptive model first; override with `--model`/`FULLSEND_MODEL` if an id is rejected); parser fixtures are hand-authored to the v0.84.2 wire docs (re-record with `internal/runtime/testdata/pi/regen.sh` once a run exists); `extension_error` events are not mapped; the behaviour scenario `features/runtime/pi.feature` (a real haiku run on Vertex of a minimal tool-using agent, asserting `metrics.json` `runtime: pi`, a `toolCall` in the pi session transcript and token usage) is gated on `BEHAVIOUR_CAPABILITIES=runtime-pi` until `fullsend-sandbox:latest` carries `PI_VERSION`, and `features/triage/triage.feature` asserts the runtime selected from the repo config on every run. Pilot on a disposable org with `triage`/`prioritize` (no sub-agent assumptions) before `code`/`fix`; `review`/`retro` rely on Claude sub-agent rosters and are not supported: pi v0.84.2 has no sub-agent tool or `agents/*.md` concept in core — only the bundled example extension (`examples/extensions/subagent/`, spawns `pi -p --mode json` children without our hook adapter, Vertex provider, `--no-approve` or session dir) and the SDK route (`createAgentSession()` per child; parent extensions do not fire for children) — so a fullsend-owned sub-agent extension with the full child flag set is a follow-up tracked on #6527 (runtime parity backlog); until then `Bootstrap` appends a runtime note telling the agent no sub-agent tool exists and to execute sub-agent definitions itself, in order. - **Other clouds** — pi ships native `amazon-bedrock` (SDK default credential chain, incl. `AWS_WEB_IDENTITY_TOKEN_FILE`) and `azure-openai-responses` (`api-key` only, no Entra ID) providers; neither is wired into `Run`'s alias table, credential hygiene or the runner's OIDC refresh yet, and the egress profile allows only Anthropic + Google hosts. Follow-up tracked against #6464. +## Selecting and overriding + +### Precedence + +The runtime and model are resolved in the following order (first non-empty value wins): + +Overrides follow the usual CLI convention — **flag > environment variable > config file / harness > built-in default** — and are resolved once by `fullsend run`, validated the same way the config/harness value would be, printed with their source, and recorded in `metrics.json`. Runtimes never read the override variables themselves. + +**Runtime:** + +1. `fullsend run --runtime ` +2. `FULLSEND_RUNTIME` +3. Per-repo `runtime:` in `.fullsend/config.yaml` (written by `fullsend github setup --runtime` / its interactive prompt, or by `fullsend repos install` from `repos.yaml`'s `runtime` / `defaults.runtime`) +4. Built-in default: `claude` + +**Model:** + +1. `fullsend run --model ` +2. `FULLSEND_MODEL` (any runtime); `FULLSEND_PI_MODEL` is kept as a lower-precedence alias on pi runs +3. Harness `model:` field +4. Agent frontmatter `model:` field +5. Runtime default (Claude Code: provider default; pi: `opus` via alias table) + +Values are aliases (`opus`, `sonnet`, `haiku`, …), a model id, or — on pi — `provider/id` (e.g. `google-vertex/gemini-2.5-flash`); Claude Code accepts its own aliases natively, pi resolves them through its alias table and applies `FULLSEND_PI_PROVIDER` to bare ids. Gemini on Vertex needs nothing beyond the model name: pi's built-in `google-vertex` provider uses the same `GOOGLE_APPLICATION_CREDENTIALS` and project as Claude-on-Vertex, and the pi run exports `GOOGLE_CLOUD_LOCATION` from `CLOUD_ML_REGION` for it. + +**Effort:** `fullsend run --effort` > `FULLSEND_EFFORT` > harness `effort:` > runtime default (Claude Code's own default; pi `--thinking high`). + +**Fallback models:** `FULLSEND_FALLBACK_MODELS=a,b` — Claude Code receives it as `--fallback-model a,b` (tried in order when the primary model is overloaded or retired); pi reports it as unsupported and ignores it (a fullsend extension for pi fallback chains is tracked in #6527). + +In CI, the `FULLSEND_*` variables are runner-process environment: the dispatch workflow forwards repository variables of the same name (plain, or role-prefixed such as `TRIAGE_FULLSEND_MODEL`) into the run, so a repo can switch one role's model without a pull request; harness `env.runner` does **not** reach the `fullsend` process. + +### Where the selection appears + +| Surface | What it shows | +|---------|---------------| +| **Run plan block** | `Runtime: (from )` next to Model and Effort | +| **stderr** | `runtime: selected "" from ` for script consumers | +| **Status comment** | Footer on the terminal comment: `Runtime: · Model: · Effort: · Cost: $` (arrow only when requested differs from reported; unknown fields omitted) | +| **`::notice::` annotation** | Same format as the status comment footer | +| **OTel span** | `fullsend.runtime` attribute on the agent span, next to `gen_ai.request.model` | +| **metrics.json** | `runtime`, `requested_runtime`, `runtime_source`, `requested_model`, `override_source` | + +The `requested_model` field records the model handed to the runtime after the per-run overrides were applied, and `override_source` says where it came from (`--model flag`, `FULLSEND_MODEL`, `FULLSEND_PI_MODEL`, `harness`, `default`) so a silent override is visible after the fact; `requested_runtime` likewise records the selected runtime (the plan block and stderr line show its source: the flag, `FULLSEND_RUNTIME`, or the config path). The run plan prints `Model: (from )` and `Effort: … (from …)` whenever a per-run override applied. + +### Runtime capability table + +| Capability | Claude Code | Pi | +|------------|-------------|-----| +| Start-time model selection | `--model` (aliases `opus`/`sonnet`/`haiku`/`fable` or a model id) | `--model provider/id`; fullsend alias table (`opus`/`sonnet`/`haiku` → catalog ids), bare ids prefixed with `FULLSEND_PI_PROVIDER` (default `anthropic-vertex`) | +| Effort / thinking | `--effort` (`low`..`max`) | `--thinking` (superset of effort levels; `high` when unset) | +| Fallback model chain (`FULLSEND_FALLBACK_MODELS`) | `--fallback-model a,b`, tried in order when the primary is overloaded or retired | Not supported yet — warned and ignored (extension tracked in #6527) | +| Mid-run model switch | Not supported in print mode | Possible from an extension (`pi.setModel`); not wired by fullsend yet (#6527) | +| Cross-provider | Anthropic models only (Vertex AI here) | Any provider pi supports by model name; Claude-on-Vertex (vendored extension) and Gemini-on-Vertex (built-in `google-vertex`) share the fleet's WIF credentials and egress; Bedrock/Azure need Track D (#6464) | +| Sub-agents | Native (`Agent` tool) | Not yet supported — `Bootstrap` appends a runtime note so skills execute sub-agent definitions themselves, in order (#6527) | + +> **Note:** `review` and `retro` on pi currently run in a single context without per-persona models and are not yet exercised on large PRs. + ## Related docs - [cli-internals.md](guides/dev/cli-internals.md) — sandbox constants, key sandbox operations diff --git a/internal/cli/admin.go b/internal/cli/admin.go index 27f9601b6f..b98d753814 100644 --- a/internal/cli/admin.go +++ b/internal/cli/admin.go @@ -1245,6 +1245,7 @@ func (a *gcfProvisionerAdapter) DeleteWIFProvider(ctx context.Context, repo stri type scaffoldOptions struct { direct bool // push directly to the default branch instead of creating a PR signOffTrailer string // e.g. "Signed-off-by: Name "; appended to the commit message when non-empty + runtime string // runtime written to .fullsend/config.yaml ("" = default claude); described in the PR body } // applyPerRepoScaffold commits scaffold files to the repo's default branch @@ -1264,6 +1265,7 @@ func applyPerRepoScaffold(ctx context.Context, client forge.Client, printer *ui. // BuildScaffoldPRMetadata will use the guard variable to distinguish // fresh installs from upgrades without version information. meta := repos.BuildScaffoldPRMetadata(ctx, client, owner, repo, "") + meta.PRBody += repos.RuntimeSection(opts.runtime) if opts.signOffTrailer != "" { meta.CommitMsg += "\n\n" + opts.signOffTrailer } diff --git a/internal/cli/github.go b/internal/cli/github.go index 512c7e4192..efa893ae20 100644 --- a/internal/cli/github.go +++ b/internal/cli/github.go @@ -188,7 +188,7 @@ values (mint URL, WIF provider, project ID) are provided as flags.`, cmd.Flags().BoolVar(&cfg.enrollNone, "enroll-none", false, "skip repository enrollment without prompting") cmd.Flags().BoolVar(&cfg.dryRun, "dry-run", false, "print actions without making changes") cmd.Flags().BoolVar(&cfg.direct, "direct", false, "push scaffold files directly to the default branch instead of creating a PR") - cmd.Flags().StringVar(&cfg.runtime, "runtime", "", "agent runtime for per-repo config (claude, pi or dummy)") + cmd.Flags().StringVar(&cfg.runtime, "runtime", "", "agent runtime for per-repo config (claude or pi; dummy is for behaviour-test installs only). Prompted on a terminal when omitted") addVendorFlags(cmd, &cfg.vendor, &cfg.fullsendBinary, &cfg.fullsendSource) cmd.Flags().StringVar(&cfg.configPreset, "config", "", "local file path or HTTPS URL to a vendor preset (committed as .fullsend/config.base.yaml)") cmd.Flags().StringVar(&cfg.configHash, "config-hash", "", "SHA-256 hex digest to validate the preset content") @@ -290,6 +290,19 @@ func runGitHubSetupPerRepo(ctx context.Context, client forge.Client, printer *ui } } + // Runtime: --runtime wins; otherwise ask once on an interactive + // terminal (Enter keeps claude). Presets carry their own value. + if cfg.runtime == "" && presetData == nil && !cfg.dryRun { + choice, err := promptRuntime(printer, os.Stdin, stdinIsInteractive()) + if err != nil { + return err + } + cfg.runtime = choice + } + if cfg.runtime == "pi" { + printer.StepWarn("runtime pi needs a sandbox image that carries pi (fullsend-sandbox/fullsend-code built from fullsend main after #6467); harnesses pinning an older image will fail at preflight") + } + // --- Build config files --- var cfgYAML []byte if presetData == nil { @@ -480,7 +493,7 @@ func runGitHubSetupPerRepo(ctx context.Context, client forge.Client, printer *ui } } - if err := applyPerRepoScaffold(ctx, client, printer, owner, repo, files, repoVars, repoSecrets, scaffoldOptions{direct: cfg.direct, signOffTrailer: signOffTrailer}); err != nil { + if err := applyPerRepoScaffold(ctx, client, printer, owner, repo, files, repoVars, repoSecrets, scaffoldOptions{direct: cfg.direct, signOffTrailer: signOffTrailer, runtime: cfg.runtime}); err != nil { return err } diff --git a/internal/cli/prescript_run_test.go b/internal/cli/prescript_run_test.go index 27ba9a3ee0..b132e0ad47 100644 --- a/internal/cli/prescript_run_test.go +++ b/internal/cli/prescript_run_test.go @@ -93,7 +93,7 @@ func TestRunAgent_PreScriptSkip_ReturnsBeforeSandboxCreation(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} err := runAgent(context.Background(), "code", dir, "", t.TempDir(), "", nil, false, "", "", rFlags, - statusOpts{}, ui.New(io.Discard), false) + statusOpts{}, ui.New(io.Discard), false, runOverrideFlags{}) require.NoError(t, err) } @@ -110,7 +110,7 @@ func TestRunAgent_PreScriptNoSkip_ProceedsToSandboxAndRelaysFalse(t *testing.T) rFlags := resolveFlags{maxDepth: 10, maxResources: 50} err := runAgent(context.Background(), "code", dir, "", t.TempDir(), "", nil, false, "", "", rFlags, - statusOpts{}, ui.New(io.Discard), false) + statusOpts{}, ui.New(io.Discard), false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "creating sandbox") @@ -131,7 +131,7 @@ func TestRunAgent_NoPreScript_StillRelaysSkippedFalse(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} err := runAgent(context.Background(), "code", dir, "", t.TempDir(), "", nil, false, "", "", rFlags, - statusOpts{}, ui.New(io.Discard), false) + statusOpts{}, ui.New(io.Discard), false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "creating sandbox") @@ -152,7 +152,7 @@ func TestRunAgent_PreScriptSkip_RelaysSkippedTrue(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} require.NoError(t, runAgent(context.Background(), "code", dir, "", t.TempDir(), "", nil, false, "", "", - rFlags, statusOpts{}, ui.New(io.Discard), false)) + rFlags, statusOpts{}, ui.New(io.Discard), false, runOverrideFlags{})) data, err := os.ReadFile(out) require.NoError(t, err) @@ -170,7 +170,7 @@ func TestRunAgent_PreScriptRelayFailureIsHardError(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} err := runAgent(context.Background(), "code", dir, "", t.TempDir(), "", nil, false, "", "", rFlags, - statusOpts{}, ui.New(io.Discard), false) + statusOpts{}, ui.New(io.Discard), false, runOverrideFlags{}) require.ErrorContains(t, err, "relaying pre-script outputs") } @@ -334,7 +334,7 @@ func TestRunAgent_PreScriptExit78_RelaysSkippedTrue(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} require.NoError(t, runAgent(context.Background(), "code", dir, "", t.TempDir(), "", nil, false, "", "", - rFlags, statusOpts{}, ui.New(io.Discard), false)) + rFlags, statusOpts{}, ui.New(io.Discard), false, runOverrideFlags{})) data, err := os.ReadFile(out) require.NoError(t, err) diff --git a/internal/cli/repos.go b/internal/cli/repos.go index c87f8e7725..959b13c32d 100644 --- a/internal/cli/repos.go +++ b/internal/cli/repos.go @@ -432,6 +432,7 @@ type reposInstallConfig struct { fullsendRef string mintURL string allowedRemoteResources []string + runtime string // Test overrides testClient forge.Client @@ -481,6 +482,7 @@ GCP infrastructure (WIF, mint) must be provisioned separately via cmd.Flags().StringVar(&opts.fullsendRef, "fullsend-ref", "", "per-repo fullsend workflow ref override") cmd.Flags().StringVar(&opts.mintURL, "mint-url", "", "per-repo mint URL override") cmd.Flags().StringSliceVar(&opts.allowedRemoteResources, "allowed-remote-resources", nil, "per-repo allowed remote resources override") + cmd.Flags().StringVar(&opts.runtime, "runtime", "", "agent runtime written to the per-repo config for repos added by this command (claude, pi); repos already in the manifest keep their entry/defaults.runtime") cmd.Flags().StringVar(&opts.gitlabBotToken, "gitlab-bot-token", "", "GitLab bot PAT for free-tier instances that don't support project access tokens") return cmd @@ -607,6 +609,11 @@ func runReposInstall(ctx context.Context, opts *reposInstallConfig) error { if forgeName != repos.ForgeGitHub && opts.mintURL != "" { printer.StepWarn(fmt.Sprintf("--mint-url is only used with GitHub repos; ignored for %s", forgeName)) } + if opts.runtime != "" { + if err := validateRuntimeName(opts.runtime); err != nil { + return fmt.Errorf("--runtime: %w", err) + } + } entries := make([]repos.RepoEntry, len(notInManifest)) for i, r := range notInManifest { @@ -623,6 +630,9 @@ func runReposInstall(ctx context.Context, opts *reposInstallConfig) error { if len(opts.allowedRemoteResources) > 0 { entry.AllowedRemoteResources = opts.allowedRemoteResources } + if opts.runtime != "" && opts.runtime != manifest.Defaults.Runtime { + entry.Runtime = opts.runtime + } entries[i] = entry } diff --git a/internal/cli/run.go b/internal/cli/run.go index 36cba5d001..1f609588c5 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -162,6 +162,20 @@ type aggregateMetrics struct { // Runtime is the backend that ran the iterations (claude, pi, dummy), // so artifacts record which runtime a per-repo `runtime:` selected. Runtime string `json:"runtime,omitempty"` + // RequestedRuntime is the runtime selected for the run (config file or a + // --runtime/FULLSEND_RUNTIME override); Runtime is what actually ran. + RequestedRuntime string `json:"requested_runtime,omitempty"` + // RuntimeSource is where RequestedRuntime came from: "--runtime flag", + // "FULLSEND_RUNTIME", the config file path, or "default (config not found)". + RuntimeSource string `json:"runtime_source,omitempty"` + // RequestedModel is the model handed to the runtime after the per-run + // overrides (--model, FULLSEND_MODEL, FULLSEND_PI_MODEL on pi) were + // applied; Model is what the provider reported. + RequestedModel string `json:"requested_model,omitempty"` + // OverrideSource records where RequestedModel came from ("--model flag", + // "FULLSEND_MODEL", "FULLSEND_PI_MODEL", "harness", "default") so a + // silent override is visible after the fact. + OverrideSource string `json:"override_source,omitempty"` } func writeMetricsJSON(dir string, m aggregateMetrics) error { @@ -258,6 +272,7 @@ func newRunCmd() *cobra.Command { var forgeFlag string var rFlags resolveFlags var sOpts statusOpts + var oFlags runOverrideFlags cmd := &cobra.Command{ Use: "run ", @@ -267,7 +282,7 @@ func newRunCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { agentName := args[0] printer := ui.New(os.Stdout) - return runAgent(cmd.Context(), agentName, fullsendDir, outputBase, targetRepo, fullsendBinary, envFiles, noPostScript, debugFilter, forgeFlag, rFlags, sOpts, printer, keepSandbox) + return runAgent(cmd.Context(), agentName, fullsendDir, outputBase, targetRepo, fullsendBinary, envFiles, noPostScript, debugFilter, forgeFlag, rFlags, sOpts, printer, keepSandbox, oFlags) }, } @@ -288,13 +303,16 @@ func newRunCmd() *cobra.Command { cmd.Flags().StringVar(&sOpts.statusRepo, "status-repo", "", "repository (owner/repo) for status comments") cmd.Flags().IntVar(&sOpts.statusNum, "status-number", 0, "issue/PR number for status comments") cmd.Flags().StringVar(&sOpts.mintURL, "mint-url", "", "mint service URL for on-demand status tokens (default: $FULLSEND_MINT_URL)") + cmd.Flags().StringVar(&oFlags.runtime, "runtime", "", "override the agent runtime from config.yaml for this run (claude, pi or dummy; also $FULLSEND_RUNTIME)") + cmd.Flags().StringVar(&oFlags.model, "model", "", "override the harness/agent model for this run (alias such as opus/sonnet/haiku, a model id, or provider/id on pi; also $FULLSEND_MODEL)") + cmd.Flags().StringVar(&oFlags.effort, "effort", "", "override the harness effort level for this run (low, medium, high, xhigh, max; also $FULLSEND_EFFORT)") _ = cmd.MarkFlagRequired("fullsend-dir") _ = cmd.MarkFlagRequired("target-repo") return cmd } -func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRepo, fullsendBinary string, envFiles []string, noPostScript bool, debug string, forgeFlag string, rFlags resolveFlags, sOpts statusOpts, printer *ui.Printer, keepSandbox bool) (runErr error) { +func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRepo, fullsendBinary string, envFiles []string, noPostScript bool, debug string, forgeFlag string, rFlags resolveFlags, sOpts statusOpts, printer *ui.Printer, keepSandbox bool, oFlags runOverrideFlags) (runErr error) { printer.Banner(Version()) printer.Blank() printer.Header("Running agent: " + agentName) @@ -659,6 +677,53 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // (runner_env + env.runner), not just the declared runner_env entries. h.RunnerEnv = effectiveRunnerEnv + // Resolve the per-run overrides (flag > env) and the runtime early so + // both appear in the plan block. The full backend is used later (step + // 5b) for sandbox setup. Overrides are resolved once, here; runtimes + // never read FULLSEND_* themselves (#6526). + overrides, err := resolveRunOverrides(oFlags, os.Getenv, "") + if err != nil { + printer.StepFail(err.Error()) + return err + } + if overrides.runtime == "" { + // The pi-only FULLSEND_PI_MODEL alias depends on which runtime the + // config selects; resolve the config runtime first, then re-run. + if b, _, e := backendFromConfigFile(orgConfigPath); e == nil { + overrides, err = resolveRunOverrides(oFlags, os.Getenv, b.Runtime.Name()) + if err != nil { + printer.StepFail(err.Error()) + return err + } + } + } + runtimeBackend, runtimeConfigSource, runtimeErr := resolveBackend(overrides, orgConfigPath) + if runtimeErr != nil { + switch { + case errors.Is(runtimeErr, errParsingConfigRuntime): + printer.StepFail("Failed to parse config.yaml") + case errors.Is(runtimeErr, errResolvingRuntime): + printer.StepFail("Failed to resolve runtime") + default: + printer.StepFail("Failed to load config.yaml") + } + return runtimeErr + } + + // Apply the model/effort overrides to the composed harness so every + // consumer (plan, runtime, metrics, status comment) sees one value. + if overrides.model != "" { + h.Model = overrides.model + } + if overrides.effort != "" { + if !harness.ValidEffort(overrides.effort) { + err := fmt.Errorf("%s: invalid effort %q: must be one of %s", overrides.effortSource, overrides.effort, strings.Join(harness.ValidEffortLevels(), ", ")) + printer.StepFail(err.Error()) + return err + } + h.Effort = overrides.effort + } + // Print plan. printer.KeyValue("Agent", h.Agent) if h.Role != "" { @@ -671,11 +736,15 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep printer.KeyValue("Policy", h.Policy) } if h.Model != "" { - printer.KeyValue("Model", h.Model) + printer.KeyValue("Model", withSource(h.Model, overrides.modelSource)) } if h.Effort != "" { - printer.KeyValue("Effort", h.Effort) + printer.KeyValue("Effort", withSource(h.Effort, overrides.effortSource)) + } + if len(overrides.fallbackModels) > 0 { + printer.KeyValue("Fallback models", withSource(strings.Join(overrides.fallbackModels, ", "), overrides.fallbackSource)) } + printer.KeyValue("Runtime", fmt.Sprintf("%s (from %s)", runtimeBackend.Runtime.Name(), runtimeConfigSource)) if h.Image != "" { printer.KeyValue("Image", h.Image) } @@ -726,6 +795,11 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep var runSkipped bool var runSkipReason string + // aggMetrics accumulates behavioral metrics across retry iterations. + // Declared here so the status-notification defer (below) can read the + // final values for the completion comment footer. + var aggMetrics aggregateMetrics + // 1c. Set up status notifications (comments on the issue/PR). // Lives in the CLI layer (not harness or post-script) so it wraps the // entire run lifecycle including sandbox setup, validation loop, and @@ -753,6 +827,9 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep status = "skipped" detail = runSkipReason } + // Set RunInfo for the completion footer. aggMetrics + // is fully populated by now (after all iterations). + notifier.SetRunInfo(runInfoFor(aggMetrics, h.Effort)) dCtx, dCancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) defer dCancel() if err := notifier.PostCompletionWithDetail(dCtx, description, status, detail); err != nil { @@ -939,7 +1016,6 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep var lastExitCode int var transcriptErrorOverride bool var runCount int - var aggMetrics aggregateMetrics tracer, tracingCleanup := telemetry.Setup(runDir, Version()) tid := resolveTraceIdentity(ctx, tracer, os.Getenv("TRACEPARENT"), os.Getenv("TRACESTATE"), []attribute.KeyValue{ stringAttr("fullsend.agent", agentName), @@ -1194,26 +1270,20 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep repoName := filepath.Base(hostRepositoryDir) remoteRepositoryDir := fmt.Sprintf("%s/%s", sandbox.SandboxWorkspace, repoName) - // 5b. Resolve the agent runtime. Done before the fetch service starts so - // runtime-owned paths (skill destination) come from the runtime, not from - // Claude-specific constants. - var backend agentruntime.Backend - orgConfigPath = filepath.Join(absFullsendDir, "config.yaml") - backend, configSource, backendErr := backendFromConfigFile(orgConfigPath) - if backendErr != nil { - switch { - case errors.Is(backendErr, errParsingConfigRuntime): - printer.StepFail("Failed to parse config.yaml") - case errors.Is(backendErr, errResolvingRuntime): - printer.StepFail("Failed to resolve runtime") - default: - printer.StepFail("Failed to load config.yaml") - } - return backendErr - } + // 5b. Resolve the agent runtime. Already resolved before the plan block + // for display; reuse the result here. The stderr line stays for scripts. + backend := runtimeBackend + configSource := runtimeConfigSource fmt.Fprintf(os.Stderr, "runtime: selected %q from %s\n", backend.Runtime.Name(), configSource) + if overrides.modelSource != "" { + fmt.Fprintf(os.Stderr, "model: requested %q from %s\n", h.Model, overrides.modelSource) + } rt := backend.Runtime aggMetrics.Runtime = rt.Name() + aggMetrics.RequestedRuntime = rt.Name() + aggMetrics.RuntimeSource = configSource + aggMetrics.RequestedModel = h.Model + aggMetrics.OverrideSource = modelOverrideSource(overrides, h.Model) tx := backend.Transcripts // 6. Start runtime fetch service (Phase 4, ADR-0038). @@ -1562,6 +1632,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep AgentBaseName: agentBaseName, Model: h.Model, Effort: h.Effort, + FallbackModels: overrides.fallbackModels, RepoDir: remoteRepositoryDir, FullsendDir: absFullsendDir, PluginDirs: pluginDirs, @@ -1577,7 +1648,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep aggregateRunMetrics(&aggMetrics, &metrics, iteration) if runErr != nil { - finalizeAgentSpan(agentSpan, runErr, iteration, exitCode, rt.System(), &metrics, "") + finalizeAgentSpan(agentSpan, runErr, iteration, exitCode, rt.System(), rt.Name(), &metrics, "") printer.StepFail("Agent execution failed") // Record the real exit code (rt.Run returns -1 when the agent never // started) so the telemetry summary reports the failure faithfully @@ -1614,7 +1685,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep } } - finalizeAgentSpan(agentSpan, nil, iteration, exitCode, rt.System(), &metrics, transcriptErrMsg) + finalizeAgentSpan(agentSpan, nil, iteration, exitCode, rt.System(), rt.Name(), &metrics, transcriptErrMsg) printer.Blank() // Non-zero exit is a warning, not a failure — the validation loop is the success gate. @@ -1769,6 +1840,9 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep if err := writeMetricsJSON(runDir, aggMetrics); err != nil { printer.StepWarn("Failed to write metrics.json: " + err.Error()) } + // Same runtime/model/effort/cost line as the status-comment footer, as a + // workflow annotation — emitted whether or not status comments are on. + emitRunInfoNotice(os.Stderr, os.Getenv("GITHUB_ACTIONS") == "true", runInfoFor(aggMetrics, h.Effort)) // 9e-bis. Surface transcript errors in workflow logs (GitHub Actions). // Parse transcript JSONL files and emit ::error:: annotations so operators @@ -2599,12 +2673,13 @@ func rootSpanEndAttrs(agg aggregateMetrics, runCount int) []attribute.KeyValue { } } -func agentSpanEndAttrs(iteration, exitCode int, system string, m *agentruntime.RunMetrics) []attribute.KeyValue { +func agentSpanEndAttrs(iteration, exitCode int, system, runtimeName string, m *agentruntime.RunMetrics) []attribute.KeyValue { attrs := []attribute.KeyValue{ attribute.Int("iteration", iteration), attribute.Int("exit_code", exitCode), stringAttr("gen_ai.system", system), stringAttr("gen_ai.request.model", m.Model), + stringAttr("fullsend.runtime", runtimeName), attribute.Int("gen_ai.usage.input_tokens", m.InputTokens), attribute.Int("gen_ai.usage.output_tokens", m.OutputTokens), attribute.Int("gen_ai.usage.cache_creation.input_tokens", m.CacheCreationInputTokens), @@ -2808,8 +2883,8 @@ func transcriptErrorMessage(te agentruntime.TranscriptError) string { // agent span and ends it. transcriptErr is non-empty when the transcript // reported a failure the process exit code did not (#2786): exit_code keeps // the raw process exit and fullsend.transcript_error marks the override. -func finalizeAgentSpan(span trace.Span, runErr error, iteration, exitCode int, system string, m *agentruntime.RunMetrics, transcriptErr string) { - span.SetAttributes(agentSpanEndAttrs(iteration, exitCode, system, m)...) +func finalizeAgentSpan(span trace.Span, runErr error, iteration, exitCode int, system, runtimeName string, m *agentruntime.RunMetrics, transcriptErr string) { + span.SetAttributes(agentSpanEndAttrs(iteration, exitCode, system, runtimeName, m)...) switch { case runErr != nil: recordSanitizedError(span, runErr) @@ -4499,3 +4574,36 @@ func checkProviderProfileIntegrity(providers []resolve.ResolvedProvider, profile } return "", nil } + +// withSource appends the override source to a plan value when the value came +// from a per-run override rather than the config/harness. +func withSource(value, source string) string { + if source == "" { + return value + } + return fmt.Sprintf("%s (from %s)", value, source) +} + +// runInfoFor builds the status-comment/annotation footer input from the +// aggregated metrics and the effective effort. +func runInfoFor(m aggregateMetrics, effort string) statuscomment.RunInfo { + return statuscomment.RunInfo{ + Runtime: m.Runtime, + RequestedModel: m.RequestedModel, + ReportedModel: m.Model, + Effort: effort, + CostUSD: m.TotalCostUSD, + } +} + +// emitRunInfoNotice writes the run-info footer as a GitHub Actions +// `::notice::` annotation when running in CI; a no-op elsewhere or when +// nothing is known. +func emitRunInfoNotice(w io.Writer, inCI bool, info statuscomment.RunInfo) { + if !inCI { + return + } + if footer := statuscomment.BuildRunInfoFooter(&info); footer != "" { + fmt.Fprintf(w, "::notice::%s\n", footer) + } +} diff --git a/internal/cli/run_overrides.go b/internal/cli/run_overrides.go new file mode 100644 index 0000000000..e5e79ec6d8 --- /dev/null +++ b/internal/cli/run_overrides.go @@ -0,0 +1,142 @@ +package cli + +import ( + "fmt" + "strings" + + "github.com/fullsend-ai/fullsend/internal/config" + agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" +) + +// Runtime-neutral override environment variables. They are overrides of the +// same values the config file and the harness carry; the CLI resolves them +// once (flag > env > config/harness > default), prints the source, records it +// in metrics.json, and hands the result to the runtime — runtimes never read +// these themselves (#6526). +const ( + envRuntime = "FULLSEND_RUNTIME" + envModel = "FULLSEND_MODEL" + envEffort = "FULLSEND_EFFORT" + envFallbackModels = "FULLSEND_FALLBACK_MODELS" + // envPiModel is the pre-#6526 pi-only model override, kept as an alias of + // FULLSEND_MODEL for pi runs. FULLSEND_PI_PROVIDER stays pi-only (it is + // the provider prefix for bare ids, not a model choice). + envPiModel = "FULLSEND_PI_MODEL" + + sourceFlagRuntime = "--runtime flag" + sourceFlagModel = "--model flag" + sourceFlagEffort = "--effort flag" + sourceHarness = "harness" + sourceDefault = "default" +) + +// runOverrideFlags are the per-run override flags of `fullsend run`. +type runOverrideFlags struct { + runtime string + model string + effort string +} + +// runOverrides is the resolved per-run override set. Empty *Source fields +// mean "not overridden" — the config file (runtime) or the composed harness +// (model, effort) stays in charge. +type runOverrides struct { + runtime string + runtimeSource string + + model string + modelSource string + + effort string + effortSource string + + fallbackModels []string + fallbackSource string +} + +// resolveRunOverrides applies the precedence flag > env for each override. +// runtimeName is the runtime that will run (after the runtime override, if +// any) and only gates the pi-specific FULLSEND_PI_MODEL alias. +func resolveRunOverrides(flags runOverrideFlags, getenv func(string) string, runtimeName string) (runOverrides, error) { + var o runOverrides + env := func(name string) string { return strings.TrimSpace(getenv(name)) } + + switch { + case strings.TrimSpace(flags.runtime) != "": + o.runtime, o.runtimeSource = strings.TrimSpace(flags.runtime), sourceFlagRuntime + case env(envRuntime) != "": + o.runtime, o.runtimeSource = env(envRuntime), envRuntime + } + if o.runtime != "" { + if err := validateRuntimeName(o.runtime); err != nil { + return runOverrides{}, fmt.Errorf("%s: %w", o.runtimeSource, err) + } + runtimeName = o.runtime + } + + switch { + case strings.TrimSpace(flags.model) != "": + o.model, o.modelSource = strings.TrimSpace(flags.model), sourceFlagModel + case env(envModel) != "": + o.model, o.modelSource = env(envModel), envModel + case runtimeName == "pi" && env(envPiModel) != "": + o.model, o.modelSource = env(envPiModel), envPiModel + } + + switch { + case strings.TrimSpace(flags.effort) != "": + o.effort, o.effortSource = strings.TrimSpace(flags.effort), sourceFlagEffort + case env(envEffort) != "": + o.effort, o.effortSource = env(envEffort), envEffort + } + + if v := env(envFallbackModels); v != "" { + for _, m := range strings.Split(v, ",") { + if m = strings.TrimSpace(m); m != "" { + o.fallbackModels = append(o.fallbackModels, m) + } + } + if len(o.fallbackModels) > 0 { + o.fallbackSource = envFallbackModels + } + } + return o, nil +} + +// validateRuntimeName mirrors the config validation so a flag/env override +// cannot select a runtime the config file could not. +func validateRuntimeName(name string) error { + for _, v := range config.ValidRuntimes() { + if name == v { + return nil + } + } + return fmt.Errorf("invalid runtime %q: must be one of %s", name, strings.Join(config.ValidRuntimes(), ", ")) +} + +// resolveBackend returns the runtime backend for the run and a human-readable +// source: the override (flag/env) when set, else the config file path (or the +// built-in default when no config exists). +func resolveBackend(o runOverrides, configPath string) (agentruntime.Backend, string, error) { + if o.runtime == "" { + return backendFromConfigFile(configPath) + } + backend, err := agentruntime.Resolve(o.runtime) + if err != nil { + return agentruntime.Backend{}, o.runtimeSource, fmt.Errorf("%w: %w", errResolvingRuntime, err) + } + return backend, o.runtimeSource, nil +} + +// modelOverrideSource is the metrics.json override_source value for the +// effective model: the override source when one applied, "harness" when the +// composed harness/agent supplied the model, "default" otherwise. +func modelOverrideSource(o runOverrides, effectiveModel string) string { + if o.modelSource != "" { + return o.modelSource + } + if effectiveModel != "" { + return sourceHarness + } + return sourceDefault +} diff --git a/internal/cli/run_overrides_test.go b/internal/cli/run_overrides_test.go new file mode 100644 index 0000000000..9d5e2ecafc --- /dev/null +++ b/internal/cli/run_overrides_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func envMap(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +func TestResolveRunOverrides_Precedence(t *testing.T) { + t.Parallel() + cases := []struct { + name string + flags runOverrideFlags + env map[string]string + runtime string + want runOverrides + }{ + {name: "nothing set", want: runOverrides{}}, + { + name: "flags beat env", + flags: runOverrideFlags{runtime: "pi", model: "sonnet", effort: "low"}, + env: map[string]string{envRuntime: "claude", envModel: "haiku", envEffort: "high", envPiModel: "x"}, + want: runOverrides{ + runtime: "pi", runtimeSource: sourceFlagRuntime, + model: "sonnet", modelSource: sourceFlagModel, + effort: "low", effortSource: sourceFlagEffort, + }, + }, + { + name: "env applies when flags absent", + env: map[string]string{envRuntime: " pi ", envModel: "google-vertex/gemini-2.5-flash", envEffort: "medium", envFallbackModels: "sonnet, haiku,"}, + want: runOverrides{ + runtime: "pi", runtimeSource: envRuntime, + model: "google-vertex/gemini-2.5-flash", modelSource: envModel, + effort: "medium", effortSource: envEffort, + fallbackModels: []string{"sonnet", "haiku"}, fallbackSource: envFallbackModels, + }, + }, + { + name: "FULLSEND_PI_MODEL is an alias on pi", + env: map[string]string{envPiModel: "claude-opus-4-8"}, + runtime: "pi", + want: runOverrides{model: "claude-opus-4-8", modelSource: envPiModel}, + }, + { + name: "FULLSEND_PI_MODEL is ignored on claude", + env: map[string]string{envPiModel: "claude-opus-4-8"}, + runtime: "claude", + want: runOverrides{}, + }, + { + name: "FULLSEND_MODEL beats FULLSEND_PI_MODEL on pi", + env: map[string]string{envModel: "haiku", envPiModel: "opus", envRuntime: "pi"}, + want: runOverrides{runtime: "pi", runtimeSource: envRuntime, model: "haiku", modelSource: envModel}, + }, + { + name: "runtime override gates the pi alias", + env: map[string]string{envPiModel: "opus", envRuntime: "pi"}, + // config said claude, env switches to pi: the alias applies. + runtime: "claude", + want: runOverrides{runtime: "pi", runtimeSource: envRuntime, model: "opus", modelSource: envPiModel}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := resolveRunOverrides(tc.flags, envMap(tc.env), tc.runtime) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestResolveRunOverrides_InvalidRuntime(t *testing.T) { + t.Parallel() + _, err := resolveRunOverrides(runOverrideFlags{runtime: "opencode"}, envMap(nil), "") + require.Error(t, err) + assert.Contains(t, err.Error(), "--runtime flag") + assert.Contains(t, err.Error(), `invalid runtime "opencode"`) + + _, err = resolveRunOverrides(runOverrideFlags{}, envMap(map[string]string{envRuntime: "nope"}), "") + require.Error(t, err) + assert.Contains(t, err.Error(), envRuntime) +} + +func TestResolveBackend_OverrideWinsOverConfig(t *testing.T) { + dir := t.TempDir() + cfg := dir + "/config.yaml" + require.NoError(t, os.WriteFile(cfg, []byte("# fullsend per-repo configuration\nversion: \"1\"\nruntime: claude\n"), 0o644)) + + backend, source, err := resolveBackend(runOverrides{}, cfg) + require.NoError(t, err) + assert.Equal(t, "claude", backend.Runtime.Name()) + assert.Equal(t, cfg, source) + + backend, source, err = resolveBackend(runOverrides{runtime: "pi", runtimeSource: sourceFlagRuntime}, cfg) + require.NoError(t, err) + assert.Equal(t, "pi", backend.Runtime.Name()) + assert.Equal(t, sourceFlagRuntime, source) +} + +func TestModelOverrideSource(t *testing.T) { + t.Parallel() + assert.Equal(t, sourceDefault, modelOverrideSource(runOverrides{}, "")) + assert.Equal(t, sourceHarness, modelOverrideSource(runOverrides{}, "opus")) + assert.Equal(t, envModel, modelOverrideSource(runOverrides{model: "haiku", modelSource: envModel}, "haiku")) + assert.Equal(t, envPiModel, modelOverrideSource(runOverrides{model: "x", modelSource: envPiModel}, "x")) +} + +func TestWithSource(t *testing.T) { + t.Parallel() + assert.Equal(t, "opus", withSource("opus", "")) + assert.Equal(t, "haiku (from FULLSEND_MODEL)", withSource("haiku", envModel)) +} + +func TestEmitRunInfoNotice(t *testing.T) { + t.Parallel() + info := runInfoFor(aggregateMetrics{Runtime: "pi", RequestedModel: "haiku", Model: "claude-haiku-4-5", TotalCostUSD: 0.42}, "medium") + + var out strings.Builder + emitRunInfoNotice(&out, false, info) + assert.Empty(t, out.String(), "no annotation outside CI") + + emitRunInfoNotice(&out, true, info) + assert.Equal(t, "::notice::Runtime: pi · Model: haiku → claude-haiku-4-5 · Effort: medium · Cost: $0.42\n", out.String()) + + out.Reset() + emitRunInfoNotice(&out, true, runInfoFor(aggregateMetrics{}, "")) + assert.Empty(t, out.String(), "nothing known, nothing emitted") +} diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 7147fa54e4..8506edbe04 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -194,7 +194,7 @@ func TestRunAgent_HarnessLoadPipeline(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -224,7 +224,7 @@ func TestRunAgent_YMLFallback(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -237,7 +237,7 @@ func TestRunAgent_HarnessNotFound(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "nonexistent", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "nonexistent", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "no config and agents-repo fallback unavailable") } @@ -269,7 +269,7 @@ func TestRunAgent_HarnessLoadWithOrgConfig(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -299,7 +299,7 @@ func TestRunAgent_PerRepoConfig(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -562,7 +562,7 @@ func TestRunAgent_MalformedOrgConfig(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "no config and agents-repo fallback unavailable") } @@ -589,7 +589,7 @@ func TestRunAgent_MalformedOrgConfigWithURLRefs(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "no config and agents-repo fallback unavailable") } @@ -611,7 +611,7 @@ func TestRunAgent_URLRefsNoOrgConfig(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "no config and agents-repo fallback unavailable") } @@ -653,7 +653,7 @@ func TestRunAgent_WithURLBase(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -709,7 +709,7 @@ openshell: rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) // The test will fail after the orchestration block (e.g. during // bootstrapCommon or pre-script setup), but it must NOT fail at // the gateway check or provider/profile steps. @@ -745,7 +745,7 @@ func TestRunAgent_URLBaseNoAllowlist(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "not in allowed_remote_resources") } @@ -774,7 +774,7 @@ func TestRunAgent_URLBaseMalformedOrgConfig(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "no config and agents-repo fallback unavailable") } @@ -886,7 +886,7 @@ func TestRunAgent_ConfigAgentLocalPath(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "custom", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "custom", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -922,7 +922,7 @@ func TestRunAgent_ConfigAgentURL(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "triage", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "triage", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -956,7 +956,7 @@ func TestRunAgent_ConfigAgentOverridesScaffold(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -978,7 +978,7 @@ func TestRunAgent_AgentNotInConfig(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "not in config and agents-repo fallback unavailable") } @@ -998,7 +998,7 @@ func TestRunAgent_UnknownAgentName(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "nonexistent", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "nonexistent", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "not in config and agents-repo fallback unavailable") } @@ -3521,7 +3521,7 @@ func TestRunAgent_PreflightCheck_Passing(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) // Must pass the preflight guard and reach the openshell check. assert.Contains(t, err.Error(), "openshell") @@ -3536,7 +3536,7 @@ func TestRunAgent_PreflightCheck_Failing(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "preflight_check failed") } @@ -3550,7 +3550,7 @@ func TestRunAgent_PreflightCheck_NoCheckConfigured(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -3582,7 +3582,7 @@ func TestRunAgent_PreflightCheck_NilValidationLoop(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } @@ -3603,7 +3603,7 @@ func TestRunAgent_PreflightCheck_Timeout(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(io.Discard) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "timed out") } @@ -4353,7 +4353,7 @@ func TestRunAgent_ErrorOnMissingRole(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(&buf) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "invalid harness: role field is required") @@ -4998,7 +4998,7 @@ func TestRunAgent_FallsBackToFULLSEND_MINT_URL(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(&buf) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "openshell") @@ -5041,7 +5041,7 @@ func TestRunAgent_WarnsWhenNoMintURL(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(&buf) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, buf.String(), "skipping token minting") @@ -5083,7 +5083,7 @@ func TestRunAgent_MintTokenError(t *testing.T) { rFlags := resolveFlags{maxDepth: 10, maxResources: 50} printer := ui.New(&buf) repoDir := t.TempDir() - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, statusOpts{}, printer, false, runOverrideFlags{}) require.Error(t, err) assert.Contains(t, err.Error(), "agent token minting failed") @@ -5134,7 +5134,7 @@ func TestRunAgent_StatusNotifierSetup(t *testing.T) { statusNum: 42, mintURL: "https://mint.example.com", } - err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, sOpts, printer, false) + err := runAgent(context.Background(), "code", dir, "", repoDir, "", nil, false, "", "", rFlags, sOpts, printer, false, runOverrideFlags{}) // Will error downstream (openshell not available), but status notifier setup should succeed require.Error(t, err) diff --git a/internal/cli/runtime_prompt.go b/internal/cli/runtime_prompt.go new file mode 100644 index 0000000000..041a809232 --- /dev/null +++ b/internal/cli/runtime_prompt.go @@ -0,0 +1,76 @@ +package cli + +import ( + "bufio" + "fmt" + "io" + "os" + "slices" + "strings" + + "golang.org/x/term" + + "github.com/fullsend-ai/fullsend/internal/config" + "github.com/fullsend-ai/fullsend/internal/ui" +) + +// defaultRuntimeChoice is what setup selects when the user does not choose. +const defaultRuntimeChoice = "claude" + +// promptRuntime asks which agent runtime the per-repo config should select +// when --runtime was not given. It only prompts on an interactive terminal; +// otherwise (CI, pipes, --dry-run callers that pass interactive=false) it +// returns the default so setup never blocks. Enter or EOF keep the default. +// The choice is written to `runtime:` in .fullsend/config.yaml — the same +// key `fullsend github setup --runtime` and a per-run `--runtime` override. +func promptRuntime(printer *ui.Printer, in io.Reader, interactive bool) (string, error) { + if !interactive { + return "", nil + } + printer.Header("Agent Runtime") + printer.Blank() + printer.StepInfo("Choose the agent runtime for this repository:") + printer.StepInfo(" [claude] Claude Code — stable default, recommended; all fleet agents, concurrent sub-agents") + printer.StepInfo(" [pi] pi — experimental (enablement phase); any provider pi supports, no sub-agent tool yet") + printer.StepInfo(" Keep the default unless you are taking part in the pi pilot. Change later with `runtime:`") + printer.StepInfo(" in .fullsend/config.yaml or `fullsend github setup --runtime`; see docs/runtimes.md.") + printer.Blank() + + reader := bufio.NewReader(in) + for { + printer.StepInfo(fmt.Sprintf("Enter runtime [%s]: ", defaultRuntimeChoice)) + line, err := reader.ReadString('\n') + if err != nil && err != io.EOF { + return "", fmt.Errorf("reading runtime choice: %w", err) + } + choice := strings.ToLower(strings.TrimSpace(line)) + if choice == "" { + return "", nil // keep the default (not written to the overlay) + } + if slices.Contains(userRuntimeChoices(), choice) { + return choice, nil + } + printer.StepWarn(fmt.Sprintf("Invalid runtime: %q (expected one of %s)", choice, strings.Join(userRuntimeChoices(), ", "))) + if err == io.EOF { + return "", nil + } + } +} + +// userRuntimeChoices lists the runtimes offered to a person: every valid +// runtime except dummy, which exists for behaviour-test installs and is only +// ever selected with an explicit --runtime dummy. +func userRuntimeChoices() []string { + var out []string + for _, r := range config.ValidRuntimes() { + if r != "dummy" { + out = append(out, r) + } + } + return out +} + +// stdinIsInteractive reports whether stdin is a terminal. +func stdinIsInteractive() bool { + return term.IsTerminal(int(os.Stdin.Fd())) +} diff --git a/internal/cli/runtime_prompt_test.go b/internal/cli/runtime_prompt_test.go new file mode 100644 index 0000000000..8bdf26e3d8 --- /dev/null +++ b/internal/cli/runtime_prompt_test.go @@ -0,0 +1,50 @@ +package cli + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/fullsend-ai/fullsend/internal/ui" +) + +func TestPromptRuntime(t *testing.T) { + t.Parallel() + cases := []struct { + name string + input string + interactive bool + want string + warns bool + }{ + {name: "non-interactive never prompts", input: "pi\n", interactive: false, want: ""}, + {name: "enter keeps the default (unset)", input: "\n", interactive: true, want: ""}, + {name: "EOF keeps the default", input: "", interactive: true, want: ""}, + {name: "pi", input: " PI \n", interactive: true, want: "pi"}, + {name: "claude is written explicitly when chosen", input: "claude\n", interactive: true, want: "claude"}, + {name: "invalid then valid", input: "opencode\npi\n", interactive: true, want: "pi", warns: true}, + {name: "invalid then EOF keeps the default", input: "opencode\n", interactive: true, want: "", warns: true}, + {name: "dummy is not a human choice", input: "dummy\npi\n", interactive: true, want: "pi", warns: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var out bytes.Buffer + got, err := promptRuntime(ui.New(&out), strings.NewReader(tc.input), tc.interactive) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + if tc.interactive { + assert.Contains(t, out.String(), "Agent Runtime") + } else { + assert.Empty(t, out.String(), "non-interactive must print nothing") + } + if tc.warns { + assert.Contains(t, out.String(), "Invalid runtime") + assert.Contains(t, out.String(), "(expected one of claude, pi)", "dummy is not offered to people") + } + }) + } +} diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index 07833f450d..6240d25830 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -348,11 +348,12 @@ func TestAgentSpanEndAttrs(t *testing.T) { m.TotalCostUSD = 0.335349 m.ToolCalls.Store(11) - a := agentSpanEndAttrs(2, 0, "anthropic", &m) + a := agentSpanEndAttrs(2, 0, "anthropic", "claude", &m) assert.Contains(t, a, attribute.Int("iteration", 2)) assert.Contains(t, a, attribute.Int("exit_code", 0)) assert.Contains(t, a, attribute.String("gen_ai.system", "anthropic")) assert.Contains(t, a, attribute.String("gen_ai.request.model", "claude-opus-4-6")) + assert.Contains(t, a, attribute.String("fullsend.runtime", "claude")) assert.Contains(t, a, attribute.Int("gen_ai.usage.input_tokens", 11)) assert.Contains(t, a, attribute.Int("gen_ai.usage.output_tokens", 1505)) assert.Contains(t, a, attribute.Int("gen_ai.usage.cache_creation.input_tokens", 38832)) @@ -372,7 +373,7 @@ func TestAgentSpanEndAttrs_WithReasoningTokens(t *testing.T) { m.OutputTokens = 50 m.ReasoningTokens = 42 - a := agentSpanEndAttrs(1, 0, "anthropic", &m) + a := agentSpanEndAttrs(1, 0, "anthropic", "claude", &m) assert.Contains(t, a, attribute.Int("gen_ai.usage.reasoning_tokens", 42), "reasoning_tokens attribute should be present when non-zero") } @@ -597,7 +598,7 @@ func TestFinalizeAgentSpan(t *testing.T) { tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)) _, span := tp.Tracer("test").Start(context.Background(), "agent") m := &agentruntime.RunMetrics{Model: "claude-opus-4-6"} - finalizeAgentSpan(span, runErr, 1, exitCode, "gcp.vertex_ai", m, transcriptErr) + finalizeAgentSpan(span, runErr, 1, exitCode, "gcp.vertex_ai", "claude", m, transcriptErr) ended := rec.Ended() require.Len(t, ended, 1, "span must be ended exactly once") return tracetest.SpanStubFromReadOnlySpan(ended[0]) @@ -675,7 +676,7 @@ func TestFinalizeAgentSpan(t *testing.T) { tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)) _, span := tp.Tracer("test").Start(context.Background(), "agent") m := &agentruntime.RunMetrics{Model: "claude-\xff\xfeopus"} - finalizeAgentSpan(span, nil, 1, 0, "gcp.vertex_ai", m, "") + finalizeAgentSpan(span, nil, 1, 0, "gcp.vertex_ai", "claude", m, "") ended := rec.Ended() require.Len(t, ended, 1) s := tracetest.SpanStubFromReadOnlySpan(ended[0]) diff --git a/internal/config/config.go b/internal/config/config.go index 5dfb43dd46..2771a6db8f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -564,6 +564,10 @@ const perRepoConfigHeader = `# fullsend per-repo configuration # # This file configures fullsend for per-repo installation mode. # See ADR 0033 for details. +# +# The "runtime" key selects which agent runtime runs the agents, claude +# (default when unset) or pi. For one run, the 'fullsend run --runtime' +# flag wins, then FULLSEND_RUNTIME, then this file. See docs/runtimes.md. ` // NewPerRepoConfig creates a new perRepoConfig with the given roles. diff --git a/internal/harness/harness.go b/internal/harness/harness.go index 9f0394ed2e..426c438d76 100644 --- a/internal/harness/harness.go +++ b/internal/harness/harness.go @@ -39,6 +39,14 @@ func validEffortLevel(level string) bool { return slices.Contains(validEffortLevels, level) } +// ValidEffort reports whether level is an accepted effort value; exported so +// per-run overrides (--effort, FULLSEND_EFFORT) are validated the same way +// as the harness field. +func ValidEffort(level string) bool { return validEffortLevel(level) } + +// ValidEffortLevels returns the accepted effort values, in documentation order. +func ValidEffortLevels() []string { return slices.Clone(validEffortLevels) } + // ValidPluginBasename reports whether name matches the allowed plugin name pattern. func ValidPluginBasename(name string) bool { return validPluginName.MatchString(name) diff --git a/internal/repos/batch_install.go b/internal/repos/batch_install.go index 993f3cb05d..8937583296 100644 --- a/internal/repos/batch_install.go +++ b/internal/repos/batch_install.go @@ -447,6 +447,7 @@ func BatchInstall(ctx context.Context, cfg BatchInstallConfig, RunnerTags: gitlabRunnerTags(cfg.Manifest), Direct: cfg.Direct, ReuseSecrets: dr.secretsExist, + Runtime: dr.resolved.Runtime, } // When the manifest pins to a different version than the diff --git a/internal/repos/manifest.go b/internal/repos/manifest.go index d4231e29bb..a16b8c3ed3 100644 --- a/internal/repos/manifest.go +++ b/internal/repos/manifest.go @@ -21,6 +21,8 @@ import ( "github.com/fullsend-ai/fullsend/internal/forge" "github.com/fullsend-ai/fullsend/internal/netutil" "gopkg.in/yaml.v3" + + "github.com/fullsend-ai/fullsend/internal/config" ) const maxManifestBytes = 1 << 20 // 1 MB @@ -89,12 +91,18 @@ type RepoEntry struct { MintURL string `yaml:"mint_url,omitempty"` MintMode string `yaml:"mint_mode,omitempty"` AllowedRemoteResources []string `yaml:"allowed_remote_resources,omitempty"` + // Runtime is the agent runtime written as the repo's `runtime:` at + // install time (claude, pi); empty inherits defaults.runtime, and an + // empty resolved value keeps the code default (claude). + Runtime string `yaml:"runtime,omitempty"` } // DefaultsConfig holds default field values applied to every repo // across all platforms. type DefaultsConfig struct { AllowedRemoteResources []string `yaml:"allowed_remote_resources,omitempty"` + // Runtime is the default agent runtime for every repo (claude, pi). + Runtime string `yaml:"runtime,omitempty"` } // DefaultGitHubURL is the default forge URL for GitHub.com. @@ -122,6 +130,9 @@ type ResolvedConfig struct { MintMode string FullsendRef string AllowedRemoteResources []string + // Runtime is the resolved agent runtime (entry, then defaults); empty + // means the code default. + Runtime string } func parseManifestBytes(data []byte, m *Manifest) error { @@ -329,6 +340,23 @@ func (m *Manifest) Validate() error { return fmt.Errorf("unsupported manifest version %d (expected 1)", m.Version) } + if err := validateRuntimeValue("defaults.runtime", m.Defaults.Runtime); err != nil { + return err + } + for _, p := range []struct { + name string + cfg *PlatformConfig + }{{"github", m.GitHub}, {"gitlab", m.GitLab}} { + if p.cfg == nil { + continue + } + for _, e := range p.cfg.Repos { + if err := validateRuntimeValue(fmt.Sprintf("%s.repos[%s].runtime", p.name, e.Name), e.Runtime); err != nil { + return err + } + } + } + // Track all repo names across platforms for cross-platform duplicate detection. allSeen := make(map[string]bool) @@ -729,6 +757,9 @@ func (m *Manifest) resolveWithEntry(owner, repo, forgeName string, platform *Pla } else { cfg.AllowedRemoteResources = m.Defaults.AllowedRemoteResources } + // Runtime: per-repo overrides the global default; "none" stops the + // chain like the other string fields. + cfg.Runtime = resolveField(entry.Runtime, m.Defaults.Runtime, "") // Source infrastructure config from the platform-level section, // with per-repo overrides via the string fallback chain. @@ -874,3 +905,18 @@ func IsNumeric(s string) bool { func (m *Manifest) Marshal() ([]byte, error) { return yaml.Marshal(m) } + +// validateRuntimeValue accepts an empty value (inherit), the "none" sentinel +// (stop the chain; code default) or a runtime the per-repo config would +// accept, so a manifest cannot install a runtime config.yaml would reject. +func validateRuntimeValue(key, value string) error { + if value == "" || value == NoneSentinel { + return nil + } + for _, v := range config.ValidRuntimes() { + if value == v { + return nil + } + } + return fmt.Errorf("%s %q is not a valid runtime; valid runtimes: %s", key, value, strings.Join(config.ValidRuntimes(), ", ")) +} diff --git a/internal/repos/manifest_edit.go b/internal/repos/manifest_edit.go index 265db554c1..485d22e8fb 100644 --- a/internal/repos/manifest_edit.go +++ b/internal/repos/manifest_edit.go @@ -303,6 +303,7 @@ func writeManifest(path string, m *Manifest) error { // `repos set-default`. Order matches the help text. var ValidDefaultKeys = []string{ "defaults.allowed_remote_resources", + "defaults.runtime", "github.url", "github.mint_url", "github.mint_mode", @@ -355,6 +356,8 @@ func SetDefault(manifestPath, key, value string) error { // Apply. switch key { + case "defaults.runtime": + m.Defaults.Runtime = value case "defaults.allowed_remote_resources": if value == "" { m.Defaults.AllowedRemoteResources = nil @@ -439,6 +442,10 @@ func validateDefaultValue(key, value string) error { if !IsValidRef(value) { return fmt.Errorf("%s %q contains invalid characters; only alphanumeric, dot, underscore, and hyphen are allowed", key, value) } + case "defaults.runtime": + if err := validateRuntimeValue(key, value); err != nil { + return err + } case "defaults.allowed_remote_resources": for _, raw := range strings.Split(value, ",") { v := strings.TrimSpace(raw) diff --git a/internal/repos/manifest_edit_test.go b/internal/repos/manifest_edit_test.go index 8a84e7263c..6044321ba5 100644 --- a/internal/repos/manifest_edit_test.go +++ b/internal/repos/manifest_edit_test.go @@ -8,6 +8,9 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/fullsend-ai/fullsend/internal/forge" ) @@ -820,3 +823,23 @@ func TestSetDefault_InvalidKey(t *testing.T) { t.Errorf("expected invalid key error, got: %v", err) } } + +func TestSetDefault_Runtime(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "repos.yaml") + require.NoError(t, os.WriteFile(path, []byte("version: 1\ngithub:\n repos:\n - name: acme/a\n"), 0o644)) + + require.NoError(t, SetDefault(path, "defaults.runtime", "pi")) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(data), "runtime: pi") + + err = SetDefault(path, "defaults.runtime", "opencode") + require.Error(t, err) + assert.Contains(t, err.Error(), "not a valid runtime") + + require.NoError(t, SetDefault(path, "defaults.runtime", ""), "empty clears the default") + data, err = os.ReadFile(path) + require.NoError(t, err) + assert.NotContains(t, string(data), "runtime:") +} diff --git a/internal/repos/manifest_test.go b/internal/repos/manifest_test.go index 4e64843c26..9cd7923708 100644 --- a/internal/repos/manifest_test.go +++ b/internal/repos/manifest_test.go @@ -1842,3 +1842,35 @@ func TestIsValidGCPProjectID(t *testing.T) { assert.False(t, IsValidGCPProjectID("a-project-id-that-is-way-too-long-for-gcp")) assert.False(t, IsValidGCPProjectID("my-project-")) } + +func TestManifest_RuntimeResolvesAndValidates(t *testing.T) { + t.Parallel() + m := &Manifest{ + Version: 1, + Defaults: DefaultsConfig{Runtime: "pi"}, + GitHub: &PlatformConfig{Repos: []RepoEntry{ + {Name: "acme/a"}, + {Name: "acme/b", Runtime: "claude"}, + {Name: "acme/c", Runtime: NoneSentinel}, + }}, + } + require.NoError(t, m.Validate()) + + rc, ok := m.ResolveConfig("acme", "a") + require.True(t, ok) + assert.Equal(t, "pi", rc.Runtime, "entry inherits defaults.runtime") + rc, _ = m.ResolveConfig("acme", "b") + assert.Equal(t, "claude", rc.Runtime, "entry overrides the default") + rc, _ = m.ResolveConfig("acme", "c") + assert.Equal(t, "", rc.Runtime, "none stops the chain: code default") + + bad := &Manifest{Version: 1, Defaults: DefaultsConfig{Runtime: "opencode"}} + err := bad.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), `defaults.runtime "opencode" is not a valid runtime`) + + bad = &Manifest{Version: 1, GitHub: &PlatformConfig{Repos: []RepoEntry{{Name: "acme/x", Runtime: "nope"}}}} + err = bad.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), `github.repos[acme/x].runtime "nope"`) +} diff --git a/internal/repos/scaffold_metadata.go b/internal/repos/scaffold_metadata.go index dcf3a10dff..295be82aba 100644 --- a/internal/repos/scaffold_metadata.go +++ b/internal/repos/scaffold_metadata.go @@ -143,3 +143,17 @@ func detectExistingVersion(ctx context.Context, client forge.Client, } return "" } + +// RuntimeSection renders the "Runtime" section appended to the scaffold PR +// body so the reviewer of a setup PR sees which agent runtime the repo will +// use and how to change it. An empty runtime means the default. +func RuntimeSection(runtime string) string { + if runtime == "" { + runtime = "claude" + } + return "\n\n## Runtime\n\n" + + fmt.Sprintf("Agents in this repository run on **%s**", runtime) + + " (`runtime:` in `.fullsend/config.yaml`). To change it later, edit that key, " + + "re-run `fullsend github setup --runtime `, or override a " + + "single run with `fullsend run --runtime`. See https://github.com/fullsend-ai/fullsend/blob/main/docs/runtimes.md." +} diff --git a/internal/repos/scaffold_metadata_test.go b/internal/repos/scaffold_metadata_test.go index cf80eed6a6..e835287bd0 100644 --- a/internal/repos/scaffold_metadata_test.go +++ b/internal/repos/scaffold_metadata_test.go @@ -2,6 +2,7 @@ package repos import ( "context" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -178,3 +179,14 @@ func TestDetectExistingVersion(t *testing.T) { assert.Equal(t, "v1.0.0-alpha-1", v) }) } + +func TestRuntimeSection(t *testing.T) { + t.Parallel() + def := RuntimeSection("") + assert.Contains(t, def, "## Runtime") + assert.Contains(t, def, "run on **claude**") + assert.Contains(t, RuntimeSection("pi"), "run on **pi**") + assert.Contains(t, def, "`runtime:` in `.fullsend/config.yaml`") + assert.Contains(t, def, "fullsend run --runtime") + assert.True(t, strings.HasPrefix(def, "\n\n"), "section must be appended after the body with a paragraph break") +} diff --git a/internal/runtime/claude.go b/internal/runtime/claude.go index 9a80202ef1..6c43a691a1 100644 --- a/internal/runtime/claude.go +++ b/internal/runtime/claude.go @@ -329,6 +329,12 @@ func buildRunCommand(params RunParams) string { parts = append(parts, fmt.Sprintf("--effort '%s'", strings.ReplaceAll(params.Effort, "'", "'\\''"))) } + if len(params.FallbackModels) > 0 { + // Claude Code accepts a comma-separated chain tried in order when the + // primary model is overloaded or retired. + parts = append(parts, fmt.Sprintf("--fallback-model '%s'", strings.ReplaceAll(strings.Join(params.FallbackModels, ","), "'", "'\\''"))) + } + for _, pd := range params.PluginDirs { parts = append(parts, fmt.Sprintf("--plugin-dir '%s'", strings.ReplaceAll(pd, "'", "'\\''"))) } diff --git a/internal/runtime/pi_bootstrap.go b/internal/runtime/pi_bootstrap.go index f915eaaa16..017767397d 100644 --- a/internal/runtime/pi_bootstrap.go +++ b/internal/runtime/pi_bootstrap.go @@ -212,9 +212,20 @@ func piAppendSystem(agentName string, def *piAgentDef) []byte { } b.WriteString(def.Body) b.WriteString("\n") + b.WriteString(piNoSubagentNote) return []byte(b.String()) } +// piNoSubagentNote makes the absence of a sub-agent tool explicit so skills +// written for Claude Code's Agent tool (pr-review, retro) take their +// single-context path deliberately instead of recording a failed dispatch. +// A fullsend-owned Agent tool for pi is tracked on #6527. +const piNoSubagentNote = "\n## Runtime note\n\n" + + "This agent runs on the pi runtime (FULLSEND_RUNTIME=pi). No sub-agent tool " + + "(Agent/Task) is available. When a skill says to dispatch sub-agents, execute each " + + "sub-agent definition yourself, in the listed order, with the same context package, " + + "and treat each output as that sub-agent's result.\n" + // piSettingsJSON is the locked-down global settings for the sandbox run. // defaultProjectTrust "never" means a repo-owned .pi/ (settings, extensions, // SYSTEM.md) is never loaded in non-interactive modes; skills as slash diff --git a/internal/runtime/pi_bootstrap_test.go b/internal/runtime/pi_bootstrap_test.go index d32af2e6f3..f973c3e0cb 100644 --- a/internal/runtime/pi_bootstrap_test.go +++ b/internal/runtime/pi_bootstrap_test.go @@ -102,6 +102,8 @@ func TestPiRuntimeBootstrap_WritesConfigAndManifest(t *testing.T) { cfg := PiRuntime{}.ConfigDir() appendSystem := string(storedUpload(t, store, cfg+"/APPEND_SYSTEM.md")) + assert.Contains(t, appendSystem, "## Runtime note", "the no-sub-agent note is appended so skills take their single-context path deliberately") + assert.Contains(t, appendSystem, "No sub-agent tool (Agent/Task) is available") assert.True(t, strings.HasPrefix(appendSystem, "# Agent: triage\n\nInspect an issue.\n\nYou are the triage agent."), appendSystem) var settings map[string]any @@ -172,7 +174,7 @@ func TestPiRuntimeBootstrap_PreflightFailure(t *testing.T) { } func TestPiRuntimeRun_StreamsFixtureAndReportsMetrics(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") t.Setenv(piProviderEnv, "") work := t.TempDir() store := filepath.Join(work, "store") @@ -219,7 +221,7 @@ func TestPiRuntimeRun_StreamsFixtureAndReportsMetrics(t *testing.T) { } func TestPiRuntimeRun_ExitZeroWithStreamErrorReturnsOne(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") work := t.TempDir() store := filepath.Join(work, "store") fixture, err := filepath.Abs(filepath.Join("testdata", "pi", "error_run.ndjson")) @@ -239,7 +241,7 @@ func TestPiRuntimeRun_ExitZeroWithStreamErrorReturnsOne(t *testing.T) { } func TestPiRuntimeRun_MissingHookAdapterFailsClosed(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") work := t.TempDir() store := filepath.Join(work, "store") // Bootstrap with security on (so the manifest carries a hook plan), @@ -274,7 +276,7 @@ exit 0 } func TestPiRuntimeRun_SecurityOnButManifestWithoutHooksFailsFast(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") work := t.TempDir() store := filepath.Join(work, "store") logPath := filepath.Join(work, "openshell.log") diff --git a/internal/runtime/pi_run.go b/internal/runtime/pi_run.go index 8daaad6b31..0cfa66d10c 100644 --- a/internal/runtime/pi_run.go +++ b/internal/runtime/pi_run.go @@ -24,10 +24,14 @@ import ( const ( piDefaultProvider = "anthropic-vertex" piDefaultModel = "opus" - // piModelEnv replaces the whole model argument (e.g. "anthropic/claude-opus-4-6"). - piModelEnv = "FULLSEND_PI_MODEL" // piProviderEnv replaces the provider prefix applied to bare model ids. + // The model itself is resolved once by the CLI (--model, FULLSEND_MODEL, + // or the FULLSEND_PI_MODEL alias on pi; #6526) and arrives in + // RunParams.Model — the runtime does not read a model env var. piProviderEnv = "FULLSEND_PI_PROVIDER" + // piRuntimeEnv tells skills running inside the sandbox which runtime + // they are on, so a skill can take a runtime-specific path deliberately. + piRuntimeEnv = "FULLSEND_RUNTIME" ) var piModelAliases = map[string]string{ @@ -36,20 +40,15 @@ var piModelAliases = map[string]string{ "haiku": "claude-haiku-4-5", } -// translatePiModel resolves the harness/agent model into pi's --model value. +// translatePiModel resolves the harness/agent model (already overridden by +// the CLI when --model/FULLSEND_MODEL/FULLSEND_PI_MODEL apply) into pi's +// --model value: aliases map to catalog ids, bare ids get the provider +// prefix, provider/id passes through. func translatePiModel(model string) string { provider := strings.TrimSpace(os.Getenv(piProviderEnv)) if provider == "" { provider = piDefaultProvider } - if v := strings.TrimSpace(os.Getenv(piModelEnv)); v != "" { - // A bare override still needs a provider, or the Vertex extension - // and the credential hygiene would both be skipped. - if strings.Contains(v, "/") { - return v - } - return provider + "/" + v - } model = strings.TrimSpace(model) if model == "" { model = piDefaultModel @@ -134,6 +133,13 @@ func buildPiRunCommand(params RunParams, m *piManifest) string { parts = append(parts, "&& . "+shellQuote(envFile), "&& export "+piManifestEnv+"="+shellQuote(r.piManifestPath()), + "&& export "+piRuntimeEnv+"=pi", + // pi's built-in google-vertex (Gemini) provider resolves credentials + // from GOOGLE_APPLICATION_CREDENTIALS + GOOGLE_CLOUD_PROJECT + + // GOOGLE_CLOUD_LOCATION, all required; the fleet exports the region + // as CLOUD_ML_REGION (what the Anthropic-on-Vertex extension reads), + // so mirror it and Gemini on Vertex is just a model name. + `&& export GOOGLE_CLOUD_LOCATION="${GOOGLE_CLOUD_LOCATION:-$CLOUD_ML_REGION}"`, ) // pi matches the provider prefix case-insensitively, so the gate must // too or "Anthropic-Vertex/..." would run on Vertex with the unset @@ -243,6 +249,11 @@ func (r PiRuntime) Run(ctx context.Context, params RunParams, printer *ui.Printe if _, ok := piThinkingFor(params.Effort); !ok { printer.StepWarn(fmt.Sprintf("effort %q is not a pi thinking level; running at --thinking %s", sanitizeOutput(params.Effort), piDefaultThinking)) } + if len(params.FallbackModels) > 0 { + // pi has no built-in fallback chain; a fullsend extension for it is + // tracked on #6527. Say so rather than silently dropping the list. + printer.StepWarn(fmt.Sprintf("fallback models %s are not supported on pi yet and are ignored", sanitizeOutput(strings.Join(params.FallbackModels, ",")))) + } cmd := buildPiRunCommand(params, m) stdout, execCmd, cancel, err := sandbox.ExecStreamReader(ctx, params.SandboxName, cmd, params.Timeout, os.Stderr) diff --git a/internal/runtime/pi_run_test.go b/internal/runtime/pi_run_test.go index 549e438db8..2bae0095da 100644 --- a/internal/runtime/pi_run_test.go +++ b/internal/runtime/pi_run_test.go @@ -18,7 +18,7 @@ import ( ) func TestTranslatePiModel(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") t.Setenv(piProviderEnv, "") assert.Equal(t, "anthropic-vertex/claude-opus-4-6", translatePiModel("opus")) assert.Equal(t, "anthropic-vertex/claude-sonnet-4-6", translatePiModel("sonnet")) @@ -30,12 +30,14 @@ func TestTranslatePiModel(t *testing.T) { t.Setenv(piProviderEnv, "anthropic") assert.Equal(t, "anthropic/claude-opus-4-6", translatePiModel("opus")) - t.Setenv(piModelEnv, "google-vertex/gemini-2.5-pro") - assert.Equal(t, "google-vertex/gemini-2.5-pro", translatePiModel("opus"), "FULLSEND_PI_MODEL overrides everything") - + // The model override is resolved by the CLI (--model, FULLSEND_MODEL, + // FULLSEND_PI_MODEL) and arrives as the model argument; the runtime no + // longer reads FULLSEND_PI_MODEL itself. + t.Setenv("FULLSEND_PI_MODEL", "google-vertex/gemini-2.5-pro") + assert.Equal(t, "anthropic/claude-opus-4-6", translatePiModel("opus"), "runtime ignores FULLSEND_PI_MODEL") + assert.Equal(t, "google-vertex/gemini-2.5-pro", translatePiModel("google-vertex/gemini-2.5-pro")) t.Setenv(piProviderEnv, "") - t.Setenv(piModelEnv, "claude-opus-4-8") - assert.Equal(t, "anthropic-vertex/claude-opus-4-8", translatePiModel("opus"), "a bare override still gets the provider prefix") + assert.Equal(t, "anthropic-vertex/claude-opus-4-8", translatePiModel("claude-opus-4-8"), "a bare override still gets the provider prefix") } func TestPiThinkingFor(t *testing.T) { @@ -63,7 +65,7 @@ func piTestParams() RunParams { } func TestBuildPiRunCommand_Basic(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") t.Setenv(piProviderEnv, "") m := &piManifest{AgentName: "triage", Model: "opus", Tools: []string{"bash"}, BashAllowlist: []string{"gh"}, Hooks: &piHooksManifest{}} params := piTestParams() @@ -71,7 +73,11 @@ func TestBuildPiRunCommand_Basic(t *testing.T) { cmd := buildPiRunCommand(params, m) // The guard runs before the agent-writable .env is sourced. - assert.True(t, strings.HasPrefix(cmd, `cd '/sandbox/workspace/repo' && `+piHooksGuard("/sandbox/pi-config/fullsend-hooks.js", "/sandbox/pi-config/fullsend-manifest.json")+` && . '/sandbox/workspace/.env' && export FULLSEND_PI_MANIFEST='/sandbox/pi-config/fullsend-manifest.json' && unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL ANTHROPIC_VERTEX_BASE_URL && export GOOGLE_CLOUD_PROJECT="${ANTHROPIC_VERTEX_PROJECT_ID:-$GOOGLE_CLOUD_PROJECT}" && pi --print --mode json`), cmd) + assert.True(t, strings.HasPrefix(cmd, `cd '/sandbox/workspace/repo' && `+piHooksGuard("/sandbox/pi-config/fullsend-hooks.js", "/sandbox/pi-config/fullsend-manifest.json")+` && . '/sandbox/workspace/.env' && export FULLSEND_PI_MANIFEST='/sandbox/pi-config/fullsend-manifest.json' && export FULLSEND_RUNTIME=pi && export GOOGLE_CLOUD_LOCATION="${GOOGLE_CLOUD_LOCATION:-$CLOUD_ML_REGION}" && unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL ANTHROPIC_VERTEX_BASE_URL && export GOOGLE_CLOUD_PROJECT="${ANTHROPIC_VERTEX_PROJECT_ID:-$GOOGLE_CLOUD_PROJECT}" && pi --print --mode json`), cmd) + // Gemini on Vertex needs GOOGLE_CLOUD_LOCATION; the fleet exports the + // region as CLOUD_ML_REGION, so it is mirrored after .env is sourced. + assert.Contains(t, cmd, `&& export GOOGLE_CLOUD_LOCATION="${GOOGLE_CLOUD_LOCATION:-$CLOUD_ML_REGION}"`) + assert.Contains(t, cmd, "&& export FULLSEND_RUNTIME=pi") for _, want := range []string{ "--no-approve", "--no-extensions", "--no-prompt-templates", "--no-themes", "--session-dir '/sandbox/pi-config/sessions'", @@ -97,7 +103,7 @@ func TestBuildPiRunCommand_Basic(t *testing.T) { assert.Contains(t, cmd, `&& export GOOGLE_CLOUD_PROJECT="${ANTHROPIC_VERTEX_PROJECT_ID:-$GOOGLE_CLOUD_PROJECT}"`) // pi resolves the provider prefix case-insensitively; so must the gate. - t.Setenv(piModelEnv, "Anthropic-Vertex/claude-opus-4-6") + params.Model = "Anthropic-Vertex/claude-opus-4-6" cmd = buildPiRunCommand(params, m) assert.Contains(t, cmd, "&& unset ANTHROPIC_API_KEY") assert.Contains(t, cmd, "--model 'Anthropic-Vertex/claude-opus-4-6'") @@ -168,7 +174,7 @@ func TestPiHooksGuard(t *testing.T) { } func TestBuildPiRunCommand_DirectProviderKeepsAnthropicEnv(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") t.Setenv(piProviderEnv, "anthropic") cmd := buildPiRunCommand(piTestParams(), &piManifest{}) assert.Contains(t, cmd, "--model 'anthropic/claude-opus-4-6'") @@ -178,7 +184,7 @@ func TestBuildPiRunCommand_DirectProviderKeepsAnthropicEnv(t *testing.T) { } func TestBuildPiRunCommand_HarnessOverridesAndFlags(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") t.Setenv(piProviderEnv, "") params := piTestParams() params.Model = "sonnet" @@ -199,7 +205,7 @@ func TestBuildPiRunCommand_HarnessOverridesAndFlags(t *testing.T) { } func TestBuildPiRunCommand_EmptyToolRestriction(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") m := &piManifest{Tools: []string{}} cmd := buildPiRunCommand(piTestParams(), m) assert.Contains(t, cmd, "--no-builtin-tools") @@ -207,7 +213,7 @@ func TestBuildPiRunCommand_EmptyToolRestriction(t *testing.T) { } func TestBuildPiRunCommand_QuotesRepoDirAndModel(t *testing.T) { - t.Setenv(piModelEnv, "") + t.Setenv("FULLSEND_PI_MODEL", "") params := piTestParams() params.RepoDir = "/sandbox/workspace/it's" params.Model = "anthropic/claude'x" diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 3c4cd06988..3b4daab2cc 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -34,10 +34,14 @@ type RunParams struct { AgentBaseName string Model string Effort string - RepoDir string - FullsendDir string - PluginDirs []string - Debug string + // FallbackModels is the ordered overload/retirement fallback chain + // (FULLSEND_FALLBACK_MODELS). Claude Code passes it as --fallback-model; + // runtimes without the capability ignore it with a warning. + FallbackModels []string + RepoDir string + FullsendDir string + PluginDirs []string + Debug string // HooksSettingsPath, if set, is passed as --settings so Claude Code // loads the runner's hook wiring regardless of its working directory. HooksSettingsPath string diff --git a/internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env-test.sh b/internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env-test.sh new file mode 100644 index 0000000000..35aba384e0 --- /dev/null +++ b/internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env-test.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Tests for setup-agent-env.sh: prefix stripping and the FULLSEND_* override +# passthrough from repository variables. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script="${here}/setup-agent-env.sh" +fail=0 +check() { # name expected actual + if [[ "$2" == "$3" ]]; then echo "PASS: $1"; else echo "FAIL: $1 — expected [$2], got [$3]"; fail=1; fi +} + +run() { # env assignments... ; prints GITHUB_ENV content + local envfile; envfile="$(mktemp)" + ( export GITHUB_ENV="${envfile}"; env "$@" bash "${script}" >/dev/null ) + cat "${envfile}"; rm -f "${envfile}" +} + +# 1. Prefix stripping still works. +out="$(run AGENT_PREFIX=TRIAGE_ TRIAGE_TARGET_REPO_DIR=target-repo)" +check "prefix stripped" "1" "$(grep -c '^TARGET_REPO_DIR<<' <<<"${out}")" + +# 2. No FULLSEND_REPO_VARS: nothing extra exported. +check "no vars no overrides" "0" "$(grep -c '^FULLSEND_' <<<"${out}" || true)" + +# 3. Plain and role-prefixed variables; role wins. +vars='{"FULLSEND_MODEL":"opus","TRIAGE_FULLSEND_MODEL":"google-vertex/gemini-2.5-flash","FULLSEND_EFFORT":"medium","OTHER":"x"}' +out="$(run AGENT_PREFIX=TRIAGE_ FULLSEND_REPO_VARS="${vars}")" +check "role-prefixed wins" "FULLSEND_MODEL=google-vertex/gemini-2.5-flash" "$(grep '^FULLSEND_MODEL=' <<<"${out}")" +check "plain applies" "FULLSEND_EFFORT=medium" "$(grep '^FULLSEND_EFFORT=' <<<"${out}")" +check "other vars ignored" "0" "$(grep -c '^OTHER' <<<"${out}" || true)" + +# 4. Another role sees the plain value. +out="$(run AGENT_PREFIX=CODE_ FULLSEND_REPO_VARS="${vars}")" +check "other role gets plain" "FULLSEND_MODEL=opus" "$(grep '^FULLSEND_MODEL=' <<<"${out}")" + +# 5. Unsafe values are skipped (newline, shell metacharacters). +vars='{"FULLSEND_MODEL":"opus; rm -rf /","FULLSEND_RUNTIME":"pi\nx","FULLSEND_PI_PROVIDER":"anthropic-vertex"}' +out="$(run AGENT_PREFIX=TRIAGE_ FULLSEND_REPO_VARS="${vars}")" +check "metachars skipped" "0" "$(grep -c '^FULLSEND_MODEL=' <<<"${out}" || true)" +check "newline skipped" "0" "$(grep -c '^FULLSEND_RUNTIME=' <<<"${out}" || true)" +check "safe value kept" "FULLSEND_PI_PROVIDER=anthropic-vertex" "$(grep '^FULLSEND_PI_PROVIDER=' <<<"${out}")" + +# 6a. The legacy pi-only name is still forwarded (CLI treats it as an alias). +vars='{"FULLSEND_PI_MODEL":"claude-opus-4-8"}' +out="$(run AGENT_PREFIX=TRIAGE_ FULLSEND_REPO_VARS="${vars}")" +check "legacy pi model forwarded" "FULLSEND_PI_MODEL=claude-opus-4-8" "$(grep '^FULLSEND_PI_MODEL=' <<<"${out}")" + +# 6. Fallback chain keeps commas. +vars='{"FULLSEND_FALLBACK_MODELS":"sonnet,haiku"}' +out="$(run AGENT_PREFIX=TRIAGE_ FULLSEND_REPO_VARS="${vars}")" +check "fallback chain" "FULLSEND_FALLBACK_MODELS=sonnet,haiku" "$(grep '^FULLSEND_FALLBACK_MODELS=' <<<"${out}")" + +exit "${fail}" diff --git a/internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env.sh b/internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env.sh index b66834221e..e6523fbd91 100644 --- a/internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env.sh +++ b/internal/scaffold/fullsend-repo/.github/scripts/setup-agent-env.sh @@ -3,6 +3,20 @@ # using the name with the prefix stripped (multiline-safe). AGENT_PREFIX must end # with '_' (e.g. TRIAGE_). The workflow step should set AGENT_PREFIX and any # AGENT_PREFIX* variables (e.g. secrets mapped under prefixed names). +# +# Per-run override passthrough: when FULLSEND_REPO_VARS holds the caller +# repository's Actions variables as JSON (`${{ toJSON(vars) }}`), the +# allowlisted FULLSEND_* override variables are exported too, so a repo can +# switch a role's runtime/model/effort with a repository variable instead of +# a pull request. A role-prefixed variable (TRIAGE_FULLSEND_MODEL) wins over +# the plain one (FULLSEND_MODEL). Values must be single-line and limited to +# the characters a model id / runtime name can contain; anything else is +# skipped with a warning. fullsend validates the values themselves. +# The whole variable map is passed (not individual keys) because the +# custom-harness matrix job only knows its role at runtime and GitHub +# expressions cannot upper-case it to build the prefixed name; the map holds +# the caller repository's own non-secret Actions variables, which the +# workflow can already read, and only the allowlisted keys leave this script. set -euo pipefail @@ -23,3 +37,21 @@ while IFS= read -r name; do ;; esac done < <(compgen -e | sort -u) + +# Override passthrough from repository variables (optional). +if [[ -n "${FULLSEND_REPO_VARS:-}" ]]; then + # FULLSEND_PI_MODEL is the pre-#6526 pi-only name, honoured by the CLI as a + # lower-precedence alias of FULLSEND_MODEL on pi runs. + override_keys=(FULLSEND_RUNTIME FULLSEND_MODEL FULLSEND_EFFORT FULLSEND_FALLBACK_MODELS FULLSEND_PI_PROVIDER FULLSEND_PI_MODEL) + for key in "${override_keys[@]}"; do + # Role-prefixed first, then plain. jq -r yields "" when absent. + value="$(printf '%s' "${FULLSEND_REPO_VARS}" | jq -r --arg k "${AGENT_PREFIX}${key}" --arg p "${key}" '(.[$k] // .[$p] // "") | tostring')" + [[ -n "${value}" ]] || continue + if [[ ! "${value}" =~ ^[A-Za-z0-9._/@:,-]+$ ]]; then + echo "::warning::ignoring repository variable ${key}: value contains characters outside [A-Za-z0-9._/@:,-]" + continue + fi + printf '%s=%s\n' "${key}" "${value}" >> "${GITHUB_ENV}" + echo "${key}=${value} (repository variable)" + done +fi diff --git a/internal/scaffold/scaffold_test.go b/internal/scaffold/scaffold_test.go index 086619af71..4dbfec4b98 100644 --- a/internal/scaffold/scaffold_test.go +++ b/internal/scaffold/scaffold_test.go @@ -749,6 +749,11 @@ func TestSetupAgentEnvContent(t *testing.T) { s := string(content) assert.Contains(t, s, "AGENT_PREFIX") assert.Contains(t, s, "GITHUB_ENV") + // Per-run override passthrough from repository variables (#6526). + assert.Contains(t, s, "FULLSEND_REPO_VARS") + for _, key := range []string{"FULLSEND_RUNTIME", "FULLSEND_MODEL", "FULLSEND_EFFORT", "FULLSEND_FALLBACK_MODELS", "FULLSEND_PI_PROVIDER", "FULLSEND_PI_MODEL"} { + assert.Contains(t, s, key) + } } func TestRepoMaintenanceWorkflowContent(t *testing.T) { diff --git a/internal/statuscomment/statuscomment.go b/internal/statuscomment/statuscomment.go index 3e654ba044..3e124fde2c 100644 --- a/internal/statuscomment/statuscomment.go +++ b/internal/statuscomment/statuscomment.go @@ -55,6 +55,17 @@ var now = time.Now // API operation so the underlying token is never stale. type ClientFactory func(ctx context.Context) (forge.Client, error) +// RunInfo holds optional runtime/model metadata surfaced in the terminal +// status comment footer. All fields are optional — omitted fields are +// excluded from the rendered line. +type RunInfo struct { + Runtime string // e.g. "claude", "pi" + RequestedModel string // model the harness/agent asked for + ReportedModel string // model the provider actually used + Effort string // e.g. "medium", "high" + CostUSD float64 // total cost; zero means unknown +} + // Notifier manages status comment lifecycle for a single agent run. type Notifier struct { client forge.Client @@ -70,6 +81,7 @@ type Notifier struct { startTime time.Time now func() time.Time warnf func(string, ...any) + runInfo *RunInfo } // New creates a Notifier. The runID is embedded in the HTML marker comment @@ -97,6 +109,12 @@ func (n *Notifier) SetWarnFunc(f func(string, ...any)) { n.warnf = f } +// SetRunInfo sets optional runtime/model metadata rendered in the +// terminal status comment footer. +func (n *Notifier) SetRunInfo(info RunInfo) { + n.runInfo = &info +} + // SetClientFactory sets a factory that mints a fresh forge.Client before // each API operation. When set, the static client passed to New is only // used if the factory is nil. @@ -313,9 +331,45 @@ func (n *Notifier) buildCompletionBody(description, status, detail string, compl b.WriteString("\n\n") b.WriteString(line2) } + + if footer := BuildRunInfoFooter(n.runInfo); footer != "" { + b.WriteString("\n\n") + b.WriteString(footer) + } return b.String() } +// BuildRunInfoFooter renders the optional runtime/model/effort/cost line +// for the terminal status comment. Unknown fields are omitted. +func BuildRunInfoFooter(info *RunInfo) string { + if info == nil { + return "" + } + var parts []string + if info.Runtime != "" { + parts = append(parts, "Runtime: "+info.Runtime) + } + if info.RequestedModel != "" { + if info.ReportedModel != "" && info.ReportedModel != info.RequestedModel { + parts = append(parts, fmt.Sprintf("Model: %s → %s", info.RequestedModel, info.ReportedModel)) + } else { + parts = append(parts, "Model: "+info.RequestedModel) + } + } else if info.ReportedModel != "" { + parts = append(parts, "Model: "+info.ReportedModel) + } + if info.Effort != "" { + parts = append(parts, "Effort: "+info.Effort) + } + if info.CostUSD > 0 { + parts = append(parts, fmt.Sprintf("Cost: $%.2f", info.CostUSD)) + } + if len(parts) == 0 { + return "" + } + return strings.Join(parts, " · ") +} + func (n *Notifier) buildSecondLine() string { var parts []string if short := shortSHA(n.sha); short != "" { diff --git a/internal/statuscomment/statuscomment_test.go b/internal/statuscomment/statuscomment_test.go index 2cf1e792df..dac8d1bd71 100644 --- a/internal/statuscomment/statuscomment_test.go +++ b/internal/statuscomment/statuscomment_test.go @@ -1567,3 +1567,104 @@ func TestParagraphBreak_BetweenStatusAndMetadata(t *testing.T) { "interrupted body should use paragraph break before metadata line") }) } + +func TestBuildRunInfoFooter_DifferentModels(t *testing.T) { + // When requested != reported, show arrow format. + info := &RunInfo{ + Runtime: "pi", + RequestedModel: "haiku", + ReportedModel: "gemini-2.5-pro", + Effort: "medium", + CostUSD: 0.42, + } + footer := BuildRunInfoFooter(info) + assert.Equal(t, "Runtime: pi · Model: haiku → gemini-2.5-pro · Effort: medium · Cost: $0.42", footer) +} + +func TestBuildRunInfoFooter_SameModel(t *testing.T) { + // When requested == reported, show single value. + info := &RunInfo{ + Runtime: "claude", + RequestedModel: "sonnet", + ReportedModel: "sonnet", + Effort: "high", + CostUSD: 1.23, + } + footer := BuildRunInfoFooter(info) + assert.Equal(t, "Runtime: claude · Model: sonnet · Effort: high · Cost: $1.23", footer) +} + +func TestBuildRunInfoFooter_UnknownFieldsOmitted(t *testing.T) { + // Unknown fields omitted. + info := &RunInfo{ + Runtime: "pi", + RequestedModel: "haiku", + } + footer := BuildRunInfoFooter(info) + assert.Equal(t, "Runtime: pi · Model: haiku", footer) +} + +func TestBuildRunInfoFooter_NilReturnsEmpty(t *testing.T) { + assert.Equal(t, "", BuildRunInfoFooter(nil)) +} + +func TestBuildRunInfoFooter_AllFieldsEmpty(t *testing.T) { + info := &RunInfo{} + assert.Equal(t, "", BuildRunInfoFooter(info)) +} + +func TestCompletionBody_IncludesRunInfoFooter(t *testing.T) { + fc := forge.NewFakeClient() + cfg := config.StatusNotificationConfig{ + Comment: config.CommentNotificationConfig{Start: "enabled", Completion: "enabled"}, + } + n := newTestNotifier(fc, cfg) + + err := n.PostStart(context.Background(), "Code") + require.NoError(t, err) + + n.SetRunInfo(RunInfo{ + Runtime: "claude", + RequestedModel: "sonnet", + ReportedModel: "sonnet", + Effort: "high", + CostUSD: 1.50, + }) + + completionTime := fixedTime().Add(7 * time.Minute) + n.now = func() time.Time { return completionTime } + + err = n.PostCompletion(context.Background(), "Code", "success") + require.NoError(t, err) + + require.Len(t, fc.UpdatedComments, 1) + body := fc.UpdatedComments[0].Body + assert.Contains(t, body, "Runtime: claude · Model: sonnet · Effort: high · Cost: $1.50") +} + +func TestCompletionBody_RunInfoFooterWithModelDiff(t *testing.T) { + fc := forge.NewFakeClient() + cfg := config.StatusNotificationConfig{ + Comment: config.CommentNotificationConfig{Start: "enabled", Completion: "enabled"}, + } + n := newTestNotifier(fc, cfg) + + err := n.PostStart(context.Background(), "Code") + require.NoError(t, err) + + n.SetRunInfo(RunInfo{ + Runtime: "pi", + RequestedModel: "haiku", + ReportedModel: "gemini-2.5-pro", + Effort: "medium", + CostUSD: 0.42, + }) + + n.now = func() time.Time { return fixedTime().Add(7 * time.Minute) } + err = n.PostCompletion(context.Background(), "Code", "success") + require.NoError(t, err) + + require.Len(t, fc.UpdatedComments, 1) + body := fc.UpdatedComments[0].Body + assert.Contains(t, body, "Runtime: pi · Model: haiku → gemini-2.5-pro · Effort: medium · Cost: $0.42") +}