From 5d4d1610ee483fe761a769398a5834366c006113 Mon Sep 17 00:00:00 2001 From: Marta Anon Date: Tue, 14 Apr 2026 01:52:54 +0200 Subject: [PATCH] Add fullsend run agent command and experiment for test it Co-Authored-By: Claude Opus 4.6 Signed-off-by: Marta Anon --- .../.fullsend/agents/hello-world.md | 14 + .../.fullsend/env/gcp-vertex.env | 4 + .../.fullsend/harness/hello-world.yaml | 24 + .../.fullsend/policies/hello-world.yaml | 28 + .../.fullsend/scripts/validate-output.sh | 45 ++ .../skills/hello-world-summary/SKILL.md | 15 + experiments/runner-hello-world/HOW_TO.md | 134 +++++ .../runner-hello-world/HOW_TO_LOCAL.md | 122 ++++ experiments/runner-hello-world/README.md | 88 +++ .../experiment/Containerfile | 31 + .../experiment/run-experiment.sh | 93 +++ .../experiment/tools/hello-world-bin | 15 + .../experiment/workflow/hello-world.yml | 56 ++ .../runner-hello-world/vertex-auth-flow.md | 229 +++++++ internal/cli/root.go | 1 + internal/cli/run.go | 562 ++++++++++++++++++ internal/cli/run_test.go | 87 +++ internal/harness/harness.go | 296 +++++++++ internal/harness/harness_test.go | 343 +++++++++++ internal/sandbox/sandbox.go | 415 +++++++++++++ internal/sandbox/sandbox_test.go | 131 ++++ 21 files changed, 2733 insertions(+) create mode 100644 experiments/runner-hello-world/.fullsend/agents/hello-world.md create mode 100644 experiments/runner-hello-world/.fullsend/env/gcp-vertex.env create mode 100644 experiments/runner-hello-world/.fullsend/harness/hello-world.yaml create mode 100644 experiments/runner-hello-world/.fullsend/policies/hello-world.yaml create mode 100755 experiments/runner-hello-world/.fullsend/scripts/validate-output.sh create mode 100644 experiments/runner-hello-world/.fullsend/skills/hello-world-summary/SKILL.md create mode 100644 experiments/runner-hello-world/HOW_TO.md create mode 100644 experiments/runner-hello-world/HOW_TO_LOCAL.md create mode 100644 experiments/runner-hello-world/README.md create mode 100644 experiments/runner-hello-world/experiment/Containerfile create mode 100755 experiments/runner-hello-world/experiment/run-experiment.sh create mode 100755 experiments/runner-hello-world/experiment/tools/hello-world-bin create mode 100644 experiments/runner-hello-world/experiment/workflow/hello-world.yml create mode 100644 experiments/runner-hello-world/vertex-auth-flow.md create mode 100644 internal/cli/run.go create mode 100644 internal/cli/run_test.go create mode 100644 internal/harness/harness.go create mode 100644 internal/harness/harness_test.go create mode 100644 internal/sandbox/sandbox.go create mode 100644 internal/sandbox/sandbox_test.go diff --git a/experiments/runner-hello-world/.fullsend/agents/hello-world.md b/experiments/runner-hello-world/.fullsend/agents/hello-world.md new file mode 100644 index 0000000000..77472f3b23 --- /dev/null +++ b/experiments/runner-hello-world/.fullsend/agents/hello-world.md @@ -0,0 +1,14 @@ +--- +name: hello-world +description: A minimal agent that runs a tool and summarizes the repository code. +skills: + - hello-world-summary +tools: Bash(hello-world-bin) +model: sonnet +--- + +You are a minimal test agent. Your job is to: + +1. Run the `hello-world-bin` tool — this writes `output/hello-world.md` +2. Explore the repository in the current path +3. Use the `hello-world-summary` skill to write a summary of the repository to `output/summary.md` diff --git a/experiments/runner-hello-world/.fullsend/env/gcp-vertex.env b/experiments/runner-hello-world/.fullsend/env/gcp-vertex.env new file mode 100644 index 0000000000..9277d498cf --- /dev/null +++ b/experiments/runner-hello-world/.fullsend/env/gcp-vertex.env @@ -0,0 +1,4 @@ +export CLAUDE_CODE_USE_VERTEX=1 +export ANTHROPIC_VERTEX_PROJECT_ID=${ANTHROPIC_VERTEX_PROJECT_ID} +export CLOUD_ML_REGION=${CLOUD_ML_REGION} +export GOOGLE_APPLICATION_CREDENTIALS=/tmp/workspace/.gcp-credentials.json diff --git a/experiments/runner-hello-world/.fullsend/harness/hello-world.yaml b/experiments/runner-hello-world/.fullsend/harness/hello-world.yaml new file mode 100644 index 0000000000..f8bd9a5b88 --- /dev/null +++ b/experiments/runner-hello-world/.fullsend/harness/hello-world.yaml @@ -0,0 +1,24 @@ +# harness/hello-world.yaml +agent: agents/hello-world.md +model: sonnet +image: quay.io/manonru/fullsend-exp:latest +policy: policies/hello-world.yaml + +host_files: + - src: env/gcp-vertex.env + dest: /tmp/workspace/.env.d/gcp-vertex.env + expand: true + - src: ${GOOGLE_APPLICATION_CREDENTIALS} + dest: /tmp/workspace/.gcp-credentials.json + +skills: + - skills/hello-world-summary + +validation_loop: + script: scripts/validate-output.sh + max_iterations: 3 + +runner_env: + VALIDATION_EXPECTED_FAILURES: "1" + +timeout_minutes: 5 diff --git a/experiments/runner-hello-world/.fullsend/policies/hello-world.yaml b/experiments/runner-hello-world/.fullsend/policies/hello-world.yaml new file mode 100644 index 0000000000..a886b58d9a --- /dev/null +++ b/experiments/runner-hello-world/.fullsend/policies/hello-world.yaml @@ -0,0 +1,28 @@ +version: 1 + +# Minimal policy for the hello-world agent. +# Only allows outbound access to Google Cloud APIs (Vertex AI authentication). + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + vertex_ai: + name: vertex-ai + endpoints: + - host: "*.googleapis.com" + port: 443 + protocol: tcp + enforcement: enforce + access: allow + binaries: + - path: "**/curl" + - path: "**/claude" + - path: "**/node" diff --git a/experiments/runner-hello-world/.fullsend/scripts/validate-output.sh b/experiments/runner-hello-world/.fullsend/scripts/validate-output.sh new file mode 100755 index 0000000000..d52433374f --- /dev/null +++ b/experiments/runner-hello-world/.fullsend/scripts/validate-output.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +HELLO_FILE="output/hello-world.md" +SUMMARY_FILE="output/summary.md" + +# Counter-based failure for testing retry logic. +# VALIDATION_EXPECTED_FAILURES controls how many times to fail before passing. +# Counter file lives in FULLSEND_RUN_DIR so it persists across iterations. +EXPECTED_FAILURES="${VALIDATION_EXPECTED_FAILURES:-0}" +COUNTER_FILE="${FULLSEND_RUN_DIR:-.}/.validation-counter" + +if [ "$EXPECTED_FAILURES" -gt 0 ]; then + COUNT=0 + if [ -f "$COUNTER_FILE" ]; then + COUNT=$(cat "$COUNTER_FILE") + fi + COUNT=$((COUNT + 1)) + echo "$COUNT" > "$COUNTER_FILE" + + if [ "$COUNT" -le "$EXPECTED_FAILURES" ]; then + echo "FAIL: deliberate failure $COUNT of $EXPECTED_FAILURES (testing retry)" + exit 1 + fi +fi + +# Validate hello-world.md exists and contains expected output. +if [ ! -f "$HELLO_FILE" ]; then + echo "FAIL: $HELLO_FILE not found" + exit 1 +fi + +if ! grep -q "Hello world from repo" "$HELLO_FILE"; then + echo "FAIL: $HELLO_FILE missing expected 'Hello world from repo' line" + exit 1 +fi + +# Validate summary.md exists. +if [ ! -f "$SUMMARY_FILE" ]; then + echo "FAIL: $SUMMARY_FILE not found" + exit 1 +fi + +echo "PASS: output validated" +exit 0 diff --git a/experiments/runner-hello-world/.fullsend/skills/hello-world-summary/SKILL.md b/experiments/runner-hello-world/.fullsend/skills/hello-world-summary/SKILL.md new file mode 100644 index 0000000000..bf37656e63 --- /dev/null +++ b/experiments/runner-hello-world/.fullsend/skills/hello-world-summary/SKILL.md @@ -0,0 +1,15 @@ +--- +name: hello-world-summary +description: Explore the repository and produce a summary of its code +allowed-tools: Read, Glob, Grep +--- + +Explore the repository in the current directory and produce a summary. + +The summary should include: +- The repository name +- A brief description of what the repository contains +- The main languages and frameworks used +- A list of the top-level directories and their purpose + +Write the summary to `$FULLSEND_OUTPUT_DIR/summary.md` (the `FULLSEND_OUTPUT_DIR` environment variable points to the output directory). diff --git a/experiments/runner-hello-world/HOW_TO.md b/experiments/runner-hello-world/HOW_TO.md new file mode 100644 index 0000000000..573735dfe9 --- /dev/null +++ b/experiments/runner-hello-world/HOW_TO.md @@ -0,0 +1,134 @@ +# How to run the experiment + +## Two-repo model + +This experiment uses a two-repo setup that mirrors the production layout: + +- **`.fullsend` repo** (`test-fullsend/.fullsend`): Contains the harness definition, agents, skills, env files, policies, scripts, and the GitHub Actions workflow. This is where the workflow runs. +- **Target repo** (`test-fullsend/test-repo`): The codebase the agent analyzes. Checked out by the workflow and passed to the CLI via `--target-repo`. + +The `run-experiment.sh` script syncs the `.fullsend/` directory and workflow to the `.fullsend` repo, then triggers the workflow with the target repo as an input. + +## Requirements + +### Local (to run the experiment) + +- **Go toolchain** (1.23+) +- **gh CLI** authenticated with access to the fullsend fork and both repos in the target org +- **podman** (for building and pushing the container image) +- **rsync** +- A local clone of the `.fullsend` repo (see below) + +### Repos + +The default setup uses the `test-fullsend` org with two repos: + +- `test-fullsend/.fullsend` — harness and workflow (secrets configured here) +- `test-fullsend/test-repo` — target codebase for the agent + +The `.fullsend` repo needs **GitHub secrets** configured (see [Setting up GCP secrets](#setting-up-gcp-secrets) below) and a **GitHub release** on your fullsend fork (used to distribute the binary to the runner). + +### Setting up GCP secrets + +The experiment uses Claude Code via Vertex AI, which requires a GCP project with the Vertex AI API enabled and a service account key. + +**If you already use Claude Code via Vertex AI locally** (i.e. `ANTHROPIC_VERTEX_PROJECT_ID` and `CLOUD_ML_REGION` are set in your environment), you already have a GCP project with the Vertex AI API enabled — skip step 1 and reuse your project ID and region for the secrets in step 4: + +```bash +gh secret set GCP_PROJECT --repo your-org/.fullsend --body "${ANTHROPIC_VERTEX_PROJECT_ID}" +gh secret set GCP_REGION --repo your-org/.fullsend --body "${CLOUD_ML_REGION}" +``` + +You still need a service account key for CI (steps 2-4), since the workflow can't use interactive authentication like `gcloud auth application-default login`. + +**If you need to set up Vertex AI from scratch**, follow all steps below: + +1. **Create or select a GCP project** with the [Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com) enabled. + +2. **Create a service account** with the `Vertex AI User` role: + ```bash + gcloud iam service-accounts create fullsend-runner \ + --display-name="Fullsend Runner" \ + --project=${ANTHROPIC_VERTEX_PROJECT_ID} + + gcloud projects add-iam-policy-binding ${ANTHROPIC_VERTEX_PROJECT_ID} \ + --member="serviceAccount:fullsend-runner@${ANTHROPIC_VERTEX_PROJECT_ID}.iam.gserviceaccount.com" \ + --role="roles/aiplatform.user" \ + --condition=None + ``` + +3. **Create and download a JSON key**: + ```bash + gcloud iam service-accounts keys create /tmp/sa-key.json \ + --iam-account=fullsend-runner@${ANTHROPIC_VERTEX_PROJECT_ID}.iam.gserviceaccount.com + ``` + +4. **Set the secrets on the `.fullsend` repo** (where the workflow runs): + ```bash + gh secret set GCP_SA_KEY --repo your-org/.fullsend < /tmp/sa-key.json + gh secret set GCP_PROJECT --repo your-org/.fullsend --body "${ANTHROPIC_VERTEX_PROJECT_ID}" + gh secret set GCP_REGION --repo your-org/.fullsend --body "${CLOUD_ML_REGION}" + ``` + +5. **Delete the local key file** (it's now stored as a GitHub secret): + ```bash + rm /tmp/sa-key.json + ``` + +Available regions for Claude on Vertex AI include `us-east5`, `europe-west1`, and `asia-southeast1`. Check the [Vertex AI documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#regions) for the latest list. + +### GitHub Actions runner + +The workflow installs these automatically: + +- **fullsend** binary (from a GitHub release) +- **OpenShell** CLI + +Claude Code and experiment tool binaries are pre-installed in the container image (`quay.io/manonru/fullsend-exp`), which the sandbox is created from via `--from`. + +## Quick start (using defaults) + +```bash +# Clone the .fullsend repo (one-time setup) +git clone git@github.com:test-fullsend/.fullsend.git /tmp/dot-fullsend + +# Run the experiment (builds image, pushes to quay.io, syncs, triggers workflow) +./experiments/runner-hello-world/experiment/run-experiment.sh +``` + +The script will print the workflow run URL. You can watch it with: + +```bash +gh run watch --repo test-fullsend/.fullsend +``` + +## Using a different org + +To run against your own org, edit the variables at the top of `experiment/run-experiment.sh`: + +```bash +FULLSEND_REPO="/tmp/your-dot-fullsend" # Local clone of your .fullsend repo +RELEASE_REPO="your-user/fullsend" # Where to upload the fullsend binary +RELEASE_TAG="runner-hello-world-dev" # Release tag name +WORKFLOW_REPO="your-org/.fullsend" # Where to trigger the workflow +WORKFLOW_FILE="hello-world.yml" # Workflow file name +TARGET_REPO="your-org/your-target-repo" # Target repo for the agent +IMAGE_REPO="quay.io/your-user/your-image" # Container image registry +``` + +Then update the workflow file (`experiment/workflow/hello-world.yml`) to point the fullsend install step at your release: + +```yaml +- name: Install fullsend + run: | + curl -LsSf https://github.com/your-user/fullsend/releases/download/runner-hello-world-dev/fullsend_dev_linux_amd64.tar.gz -o /tmp/fullsend.tar.gz + sudo tar xzf /tmp/fullsend.tar.gz -C /usr/local/bin/ +``` + +Steps: + +1. Create two repos in your org: `.fullsend` and a target repo with some content +2. Create a GitHub release on your fullsend fork: `gh release create runner-hello-world-dev --repo your-user/fullsend --title "Dev" --notes "Dev build"` +3. Set the required secrets on the `.fullsend` repo (see [Setting up GCP secrets](#setting-up-gcp-secrets)) +4. Clone the `.fullsend` repo locally: `git clone git@github.com:your-org/.fullsend.git /tmp/your-dot-fullsend` +5. Run `./experiments/runner-hello-world/experiment/run-experiment.sh` diff --git a/experiments/runner-hello-world/HOW_TO_LOCAL.md b/experiments/runner-hello-world/HOW_TO_LOCAL.md new file mode 100644 index 0000000000..e6a23449d6 --- /dev/null +++ b/experiments/runner-hello-world/HOW_TO_LOCAL.md @@ -0,0 +1,122 @@ +# How to run `fullsend run` locally + +This guide explains how to run the `fullsend run` CLI command on your local machine against a target repository. + +## Prerequisites + +- **Go toolchain** (1.23+) +- **OpenShell** installed and in PATH +- **Docker** (required by OpenShell — Podman is not supported) +- **GCP credentials** for Vertex AI (if using the hello-world experiment) + +## Install Docker + +OpenShell requires Docker. Podman is not supported. Install Docker CE and make sure the daemon is running: + +```bash +# Fedora +sudo dnf config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo +sudo dnf install -y docker-ce docker-ce-cli containerd.io + +# Start the daemon +sudo systemctl start docker +sudo systemctl enable docker + +# Allow your user to run docker without sudo +sudo usermod -aG docker $USER +newgrp docker + +# Verify +docker info +``` + +## Install OpenShell + +```bash +curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/v0.0.30/install.sh | sh +openshell --version +``` + +## Build fullsend + +From the repository root: + +```bash +go build -o ~/.local/bin/fullsend ./cmd/fullsend/ +fullsend --version +``` + +## Required environment variables + +The hello-world experiment requires these environment variables on the host: + +| Variable | Purpose | +|----------|---------| +| `ANTHROPIC_VERTEX_PROJECT_ID` | GCP project with Vertex AI API enabled | +| `CLOUD_ML_REGION` | GCP region (e.g. `us-east5`, `europe-west1`) | +| `GOOGLE_APPLICATION_CREDENTIALS` | Path to GCP service account key JSON | + +If you already use Claude Code via Vertex AI locally, `ANTHROPIC_VERTEX_PROJECT_ID` and `CLOUD_ML_REGION` are likely already set. For `GOOGLE_APPLICATION_CREDENTIALS`, you can use a service account key file or the Application Default Credentials from gcloud: + +```bash +gcloud auth application-default login +export GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json +``` + +## Pre-pull the sandbox image + +The first sandbox creation will time out if the image hasn't been pulled yet. Pull it in advance: + +```bash +docker pull quay.io/manonru/fullsend-exp:latest +``` + +## Running the agent + +```bash +fullsend run hello-world \ + --fullsend-dir /path/to/experiments/runner-hello-world/.fullsend \ + --target-repo /path/to/your-repo \ + --output-dir /tmp/fullsend-output +``` + +### CLI flags + +| Flag | Required | Default | Purpose | +|------|----------|---------|---------| +| `--fullsend-dir` | yes | | Base directory containing the `.fullsend` layout | +| `--target-repo` | yes | | Path to the target repository | +| `--output-dir` | no | `/tmp/fullsend` | Base directory for run output (per-invocation subdirectories are created under it) | + +## Inspecting results + +After a run, output is organized under the `--output-dir` directory: + +``` +/tmp/fullsend-output/ + agent-hello-world--/ + iteration-1/ + output/ # Files the agent produced (hello-world.md, summary.md) + transcripts/ # Claude transcript .jsonl files + iteration-2/ # Only if validation retried + output/ + transcripts/ +``` + +The CLI prints the full run directory path at the end: + +``` +Run directory /tmp/fullsend-output/agent-hello-world-12345-1713200000 +Agent exit code 0 +Agent runs 1 +Validation passed +``` + +## Troubleshooting + +- **"openshell not found in PATH"**: Install OpenShell (see above). +- **"sandbox not ready after 1m0s"**: The sandbox image hasn't been pulled yet. Run `docker pull quay.io/manonru/fullsend-exp:latest` first (see above). +- **"Docker socket exists but the daemon is not responding"**: Run `sudo systemctl start docker`. +- **"permission denied while trying to connect to the Docker daemon"**: Your user is not in the `docker` group. Run `sudo usermod -aG docker $USER` and start a new shell session. +- **"host variable X is not set"**: A required environment variable is missing. Check the table above. +- **Validation failures**: Check `iteration-N/output/` for the agent's output files. The validation script (`scripts/validate-output.sh`) runs on the host and checks for expected files. diff --git a/experiments/runner-hello-world/README.md b/experiments/runner-hello-world/README.md new file mode 100644 index 0000000000..68ac289053 --- /dev/null +++ b/experiments/runner-hello-world/README.md @@ -0,0 +1,88 @@ +# Runner Hello World Experiment + +A minimal end-to-end test of the `fullsend run` CLI. It provisions an OpenShell sandbox from a pre-built container image, runs a Claude Code agent inside it, extracts output, and validates the result with a retry loop. + +## What it does + +1. Reads the harness definition (`harness/hello-world.yaml`) +2. Creates an OpenShell sandbox from a pre-built container image (`quay.io/manonru/fullsend-exp`) and applies a network policy (Vertex AI access) +3. Bootstraps the sandbox with the agent definition, skills, env vars, and GCP credentials +4. Copies the target repository into the sandbox +5. Runs a Claude Code agent that executes a hello-world tool and summarizes the repository +6. Extracts output files and runs validation (with configurable retries) +7. Extracts transcripts and prints results + +## Key design decisions + +- **Container image over SCP**: Tool binaries (`hello-world-bin`) and Claude Code are pre-installed in the container image rather than copied via SCP at runtime. This makes sandboxes self-contained and faster to provision. +- **OpenShell base image**: The container image extends `ghcr.io/nvidia/openshell-community/sandboxes/base:latest`, which is required by OpenShell for custom sandbox images. +- **Two output files**: The agent produces `output/hello-world.md` (from the tool binary) and `output/summary.md` (from the skill). Validation checks both. +- **Vertex AI via service account key**: GCP credentials are injected as a host file with env-var expansion, not via OpenShell providers (see [vertex-auth-flow.md](vertex-auth-flow.md) for details). + +## What's not tested + +The following CLI features are implemented but not exercised by this experiment: + +- **Pre/post scripts** (`pre_script`, `post_script`): Host-side scripts that run before sandbox creation and after sandbox cleanup. No scripts are configured in this harness. +- **Providers**: OpenShell credential providers. This experiment uses host_files for GCP credentials instead. +- **API servers**: Host-side REST proxy servers. Not used in this experiment. +- **Agent input**: Additional input files copied to the sandbox. Not used. +- **Feedback mode**: `validation_loop.feedback_mode` for feeding validation output back to the agent. Not configured. + +## Directory layout + +``` +experiments/runner-hello-world/ + README.md # This file + HOW_TO.md # How to reproduce the experiment (CI) + HOW_TO_LOCAL.md # How to run fullsend locally + vertex-auth-flow.md # Vertex AI auth design notes + .fullsend/ # Production layout (synced to test repo) + agents/hello-world.md # Agent definition + env/ + gcp-vertex.env # Sandbox env: Vertex AI credentials + repo.env # Sandbox env: repo-specific vars + harness/hello-world.yaml # Harness: wires agent, skills, image, policy + policies/hello-world.yaml # Network policy (allows googleapis.com) + scripts/ + validate-output.sh # Validation script (runs on host) + skills/hello-world-summary/ + SKILL.md # Skill: explore repo and write summary + experiment/ # Experiment-only files (not synced to test repo) + run-experiment.sh # Build + deploy + trigger script + Containerfile # Container image (Claude Code + tool binaries) + tools/hello-world-bin # Shell script: writes output (baked into image) + workflow/hello-world.yml # GitHub Actions workflow (copied to test repo) +``` + +The `.fullsend/` directory mirrors the layout expected in production: harness definitions, agents, skills, env files, policies, and scripts. The `experiment/` directory contains files only needed to build and run this experiment (container image, run script, workflow). + +## Creating new agents + +When creating a new agent for fullsend, prefer baking tool binaries and dependencies into the sandbox container image rather than copying them at runtime via SCP. This makes sandboxes self-contained and reproducible. + +If your agent needs tools that are scripts rather than compiled binaries, you can deliver them via `host_files` in the harness YAML: + +```yaml +host_files: + - src: scripts/my-tool.sh + dest: /tmp/workspace/bin/my-tool.sh +``` + +If a script is specifically crafted for use by a skill, bundle it inside the skill directory (e.g. `skills/my-skill/scripts/run.sh`) rather than using `host_files`. Fullsend copies the entire skill directory recursively into the sandbox, including `scripts/`, `references/`, and `assets/` subdirectories, following the [agentskills.io specification](https://agentskills.io/specification). + +### Sandbox environment variables + +Fullsend sets the following environment variables inside the sandbox: + +| Variable | Value | Purpose | +|----------|-------|---------| +| `FULLSEND_OUTPUT_DIR` | `/tmp/workspace/output` | Directory where the agent should write output files. Extracted to the host after each iteration. Cleared between iterations in a validation loop. | +| `FULLSEND_TARGET_REPO_DIR` | `/tmp/workspace/` | Path to the target repository inside the sandbox. The agent starts with this as its working directory. | +| `CLAUDE_CONFIG_DIR` | `/tmp/claude-config` | Claude configuration directory. Agent and skill definitions are placed here automatically. | + +## How to reproduce + +See [HOW_TO.md](HOW_TO.md) for prerequisites, GCP setup, and step-by-step instructions to run the experiment via CI. + +See [HOW_TO_LOCAL.md](HOW_TO_LOCAL.md) for instructions to run `fullsend run` locally. diff --git a/experiments/runner-hello-world/experiment/Containerfile b/experiments/runner-hello-world/experiment/Containerfile new file mode 100644 index 0000000000..0b7fb4bd76 --- /dev/null +++ b/experiments/runner-hello-world/experiment/Containerfile @@ -0,0 +1,31 @@ +# Containerfile — sandbox image for the runner-hello-world experiment. +# +# Contains Claude Code and the hello-world-bin tool pre-installed. +# Must be based on the OpenShell base image (the sandbox supervisor is +# side-loaded at runtime). +# +# Build: +# podman build -t quay.io/manonru/fullsend-exp:v1 \ +# -f experiments/runner-hello-world/Containerfile \ +# experiments/runner-hello-world/ + +FROM ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:f0d1ec92bf219759c1e863c0fa6b1f432ec827331e79ab9e31dab91a3b86b413 + +# The base image runs as the sandbox user (998). Switch to root for +# package installation and switch back at the end. +USER root + +# Install Node.js (required for Claude Code) and rsync (for safe repo write-back). +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs rsync \ + && rm -rf /var/lib/apt/lists/* + +# Install Claude Code. +RUN npm install -g @anthropic-ai/claude-code + +# Copy experiment tool binaries. +COPY tools/hello-world-bin /usr/local/bin/hello-world-bin +RUN chmod +x /usr/local/bin/hello-world-bin + +# Switch back to the sandbox user. +USER sandbox diff --git a/experiments/runner-hello-world/experiment/run-experiment.sh b/experiments/runner-hello-world/experiment/run-experiment.sh new file mode 100755 index 0000000000..68a250fe3f --- /dev/null +++ b/experiments/runner-hello-world/experiment/run-experiment.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# run-experiment.sh — Build fullsend, sync experiment files to the .fullsend repo, +# upload the binary to a GitHub release, and trigger the workflow. +# +# Usage: ./experiments/runner-hello-world/experiment/run-experiment.sh +# +# Prerequisites: +# - gh CLI authenticated +# - Go toolchain installed +# - podman (for building and pushing container images) +# - /tmp/dot-fullsend is a clone of test-fullsend/.fullsend + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +RUNNER_DIR="${REPO_ROOT}/experiments/runner-hello-world" +EXPERIMENT_DIR="${RUNNER_DIR}/experiment" +FULLSEND_DIR="${RUNNER_DIR}/.fullsend" +FULLSEND_REPO="/tmp/dot-fullsend" +RELEASE_REPO="maruiz93/fullsend" +RELEASE_TAG="runner-hello-world-dev" +WORKFLOW_REPO="test-fullsend/.fullsend" +WORKFLOW_FILE="hello-world.yml" +TARGET_REPO="test-fullsend/test-repo" +IMAGE_REPO="quay.io/manonru/fullsend-exp" +IMAGE_TAG="$(git -C "${REPO_ROOT}" rev-parse --short HEAD)" + +echo "==> Building fullsend (linux/amd64)..." +GOOS=linux GOARCH=amd64 go build -o /tmp/fullsend_build/fullsend "${REPO_ROOT}/cmd/fullsend/" +echo " Built: /tmp/fullsend_build/fullsend" + +echo "==> Creating tarball..." +tar czf /tmp/fullsend_build/fullsend_dev_linux_amd64.tar.gz -C /tmp/fullsend_build fullsend +echo " Created: /tmp/fullsend_build/fullsend_dev_linux_amd64.tar.gz" + +echo "==> Building container image..." +podman build -t "${IMAGE_REPO}:${IMAGE_TAG}" \ + -f "${EXPERIMENT_DIR}/Containerfile" "${EXPERIMENT_DIR}/" +echo " Built: ${IMAGE_REPO}:${IMAGE_TAG}" + +echo "==> Pushing container image..." +podman tag "${IMAGE_REPO}:${IMAGE_TAG}" "${IMAGE_REPO}:latest" +podman push "${IMAGE_REPO}:${IMAGE_TAG}" +podman push "${IMAGE_REPO}:latest" +echo " Pushed: ${IMAGE_REPO}:${IMAGE_TAG} and :latest" + +echo "==> Syncing .fullsend files to ${FULLSEND_REPO}..." +rsync -av --delete \ + --exclude='.git' \ + --exclude='.github' \ + "${FULLSEND_DIR}/" "${FULLSEND_REPO}/" + +# Sync workflow file to .github/workflows/ +mkdir -p "${FULLSEND_REPO}/.github/workflows" +cp "${EXPERIMENT_DIR}/workflow/hello-world.yml" "${FULLSEND_REPO}/.github/workflows/hello-world.yml" +echo " Synced .fullsend files and workflow" + +echo "==> Pushing experiment changes to .fullsend repo..." +cd "${FULLSEND_REPO}" + +# Safety: verify the clone's remote matches the expected workflow repo. +ACTUAL_REMOTE=$(git remote get-url origin 2>/dev/null || true) +if [[ "$ACTUAL_REMOTE" != *"${WORKFLOW_REPO}"* ]]; then + echo "ERROR: ${FULLSEND_REPO} remote (${ACTUAL_REMOTE}) does not match expected repo (${WORKFLOW_REPO})" + exit 1 +fi + +git add -A +if git diff --cached --quiet; then + echo " No changes to push" +else + git commit -m "Update hello-world experiment files" + git push + echo " Pushed" +fi + +echo "==> Uploading binary to release ${RELEASE_TAG}..." +gh release upload "${RELEASE_TAG}" \ + /tmp/fullsend_build/fullsend_dev_linux_amd64.tar.gz \ + --clobber --repo "${RELEASE_REPO}" +echo " Uploaded" + +echo "==> Triggering workflow ${WORKFLOW_FILE} (target: ${TARGET_REPO})..." +RUN_URL=$(gh workflow run "${WORKFLOW_FILE}" --repo "${WORKFLOW_REPO}" \ + -f target-repo="${TARGET_REPO}" 2>&1) +echo " ${RUN_URL}" + +# Give GitHub a moment to register the run, then fetch the URL. +sleep 3 +RUN_ID=$(gh run list --repo "${WORKFLOW_REPO}" --workflow "${WORKFLOW_FILE}" --limit 1 --json databaseId --jq '.[0].databaseId') +echo "" +echo "==> Workflow run: https://github.com/${WORKFLOW_REPO}/actions/runs/${RUN_ID}" +echo " Watch with: gh run watch ${RUN_ID} --repo ${WORKFLOW_REPO}" diff --git a/experiments/runner-hello-world/experiment/tools/hello-world-bin b/experiments/runner-hello-world/experiment/tools/hello-world-bin new file mode 100755 index 0000000000..e84c9c633e --- /dev/null +++ b/experiments/runner-hello-world/experiment/tools/hello-world-bin @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +OUTPUT_DIR="${FULLSEND_OUTPUT_DIR:-output}" +mkdir -p "$OUTPUT_DIR" + +cat > "$OUTPUT_DIR/hello-world.md" <", + Short: "Run an agent", + Long: "Execute an agent by name: read its harness YAML, set up the sandbox, and run the agent.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + agentName := args[0] + printer := ui.New(os.Stdout) + return runAgent(agentName, fullsendDir, outputBase, targetRepo, printer) + }, + } + + cmd.Flags().StringVar(&fullsendDir, "fullsend-dir", "", "base directory containing the .fullsend layout") + cmd.Flags().StringVar(&outputBase, "output-dir", "", "base directory for run output (default: /tmp/fullsend)") + cmd.Flags().StringVar(&targetRepo, "target-repo", "", "path to the target repository") + _ = cmd.MarkFlagRequired("fullsend-dir") + _ = cmd.MarkFlagRequired("target-repo") + + return cmd +} + +func runAgent(agentName, fullsendDir, outputBase, targetRepo string, printer *ui.Printer) error { + printer.Banner() + printer.Blank() + printer.Header("Running agent: " + agentName) + printer.Blank() + + // 1. Resolve and load harness. + harnessPath := filepath.Join(fullsendDir, "harness", agentName+".yaml") + printer.StepStart("Loading harness: " + harnessPath) + + h, err := harness.Load(harnessPath) + if err != nil { + printer.StepFail("Failed to load harness") + return fmt.Errorf("loading harness: %w", err) + } + + absFullsendDir, err := filepath.Abs(fullsendDir) + if err != nil { + return fmt.Errorf("resolving fullsend dir: %w", err) + } + if err := h.ResolveRelativeTo(absFullsendDir); err != nil { + printer.StepFail("Path validation failed") + return fmt.Errorf("resolving paths: %w", err) + } + + if err := h.ValidateRunnerEnv(); err != nil { + printer.StepFail("Environment validation failed") + return fmt.Errorf("validating env: %w", err) + } + for k, v := range h.RunnerEnv { + h.RunnerEnv[k] = os.ExpandEnv(v) + } + if err := h.ValidateFilesExist(); err != nil { + printer.StepFail("File validation failed") + return fmt.Errorf("validating files: %w", err) + } + printer.StepDone("Harness loaded") + + // Print plan. + printer.KeyValue("Agent", h.Agent) + if h.Policy != "" { + printer.KeyValue("Policy", h.Policy) + } + if h.Model != "" { + printer.KeyValue("Model", h.Model) + } + if h.Image != "" { + printer.KeyValue("Image", h.Image) + } + if len(h.Providers) > 0 { + printer.KeyValue("Providers", strings.Join(h.Providers, ", ")) + } + if len(h.Skills) > 0 { + printer.KeyValue("Skills", strings.Join(h.Skills, ", ")) + } + if h.AgentInput != "" { + printer.KeyValue("Agent input", h.AgentInput) + } + if h.PreScript != "" { + printer.KeyValue("Pre-script", h.PreScript) + } + if h.PostScript != "" { + printer.KeyValue("Post-script", h.PostScript) + } + if h.TimeoutMinutes > 0 { + printer.KeyValue("Timeout", fmt.Sprintf("%d minutes", h.TimeoutMinutes)) + } + printer.Blank() + + // 2. Check openshell availability. + printer.StepStart("Checking openshell availability") + if err := sandbox.EnsureAvailable(); err != nil { + printer.StepFail("openshell not available") + return fmt.Errorf("openshell is required: %w", err) + } + printer.StepDone("openshell available") + + // 2a. Ensure a gateway is running. + printer.StepStart("Ensuring gateway") + if err := sandbox.EnsureGateway(); err != nil { + printer.StepFail("Failed to start gateway") + return fmt.Errorf("starting gateway: %w", err) + } + printer.StepDone("Gateway ready") + + // 2b. Ensure providers exist on the gateway (if any declared). + if len(h.Providers) > 0 { + providersDir := filepath.Join(absFullsendDir, "providers") + providerDefs, err := harness.LoadProviderDefs(providersDir) + if err != nil { + printer.StepFail("Failed to load provider definitions") + return fmt.Errorf("loading provider definitions: %w", err) + } + for _, pd := range providerDefs { + printer.StepStart("Ensuring provider: " + pd.Name) + if err := sandbox.EnsureProvider(pd.Name, pd.Type, pd.Credentials, pd.Config); err != nil { + printer.StepFail("Failed to create provider " + pd.Name) + return fmt.Errorf("ensuring provider %q: %w", pd.Name, err) + } + printer.StepDone("Provider ready: " + pd.Name) + } + } + + // 2c. Run pre-script on the host (if configured). + if h.PreScript != "" { + printer.StepStart("Running pre-script: " + h.PreScript) + preCmd := exec.Command(h.PreScript) + preCmd.Env = append(os.Environ(), envToList(h.RunnerEnv)...) + preCmd.Stdout = os.Stdout + preCmd.Stderr = os.Stderr + if err := preCmd.Run(); err != nil { + printer.StepFail("Pre-script failed") + return fmt.Errorf("running pre-script: %w", err) + } + printer.StepDone("Pre-script completed") + } + + // 3. Create sandbox. + sandboxName := fmt.Sprintf("agent-%s-%d-%d", agentName, os.Getpid(), time.Now().Unix()) + printer.StepStart("Creating sandbox: " + sandboxName) + + if err := sandbox.Create(sandboxName, h.Providers, h.Image, h.Policy); err != nil { + printer.StepFail("Failed to create sandbox") + return fmt.Errorf("creating sandbox: %w", err) + } + if outputBase == "" { + outputBase = filepath.Join(os.TempDir(), "fullsend") + } + runDir := filepath.Join(outputBase, sandboxName) + + // Post-script runs after sandbox cleanup (defers are LIFO). + if h.PostScript != "" { + defer func() { + printer.StepStart("Running post-script: " + h.PostScript) + postCmd := exec.Command(h.PostScript) + postCmd.Dir = runDir + postCmd.Env = append(os.Environ(), envToList(h.RunnerEnv)...) + postCmd.Stdout = os.Stdout + postCmd.Stderr = os.Stderr + if err := postCmd.Run(); err != nil { + printer.StepWarn("Post-script failed: " + err.Error()) + } else { + printer.StepDone("Post-script completed") + } + }() + } + defer func() { + printer.StepStart("Cleaning up sandbox") + if err := sandbox.Delete(sandboxName); err != nil { + printer.StepWarn("Sandbox cleanup failed: " + err.Error()) + } else { + printer.StepDone("Sandbox deleted") + } + }() + printer.StepDone("Sandbox created") + + // 4. Get SSH config. + sshConfig, err := sandbox.GetSSHConfig(sandboxName) + if err != nil { + printer.StepFail("Failed to get SSH config") + return err + } + + sshConfigFile, err := os.CreateTemp("", "openshell-ssh-*.config") + if err != nil { + return fmt.Errorf("creating SSH config temp file: %w", err) + } + sshConfigPath := sshConfigFile.Name() + if _, err := sshConfigFile.WriteString(sshConfig); err != nil { + sshConfigFile.Close() + os.Remove(sshConfigPath) + return fmt.Errorf("writing SSH config: %w", err) + } + sshConfigFile.Close() + defer os.Remove(sshConfigPath) + + // 6. Resolve target repo path (needed by bootstrap for env vars). + repoSrc, err := filepath.Abs(targetRepo) + if err != nil { + return fmt.Errorf("resolving target repo path: %w", err) + } + repoName := filepath.Base(repoSrc) + repoDir := fmt.Sprintf("%s/%s", sandbox.SandboxWorkspace, repoName) + + // 7. Bootstrap sandbox. + printer.StepStart("Bootstrapping sandbox") + if err := bootstrapSandbox(sshConfigPath, sandboxName, repoDir, h); err != nil { + printer.StepFail("Failed to bootstrap sandbox") + return err + } + printer.StepDone("Sandbox bootstrapped") + + // 8. Make project code available (copy repo root into a named subdirectory). + printer.StepStart("Copying project code into sandbox") + mkRepoCmd := fmt.Sprintf("mkdir -p %s", repoDir) + if _, _, _, err := sandbox.SSH(sshConfigPath, sandboxName, mkRepoCmd, 10*time.Second); err != nil { + return fmt.Errorf("creating repo dir in sandbox: %w", err) + } + if err := sandbox.SCP(sshConfigPath, sandboxName, repoSrc+"/.", repoDir+"/"); err != nil { + printer.StepFail("Failed to copy project code") + return fmt.Errorf("copying project code: %w", err) + } + printer.StepDone("Project code copied to " + repoName + "/") + + // 8b. Copy agent-input files (if configured). + if h.AgentInput != "" { + printer.StepStart("Copying agent-input files into sandbox") + remoteInput := fmt.Sprintf("%s/agent-input", sandbox.SandboxWorkspace) + mkInputCmd := fmt.Sprintf("mkdir -p %s", remoteInput) + if _, _, _, err := sandbox.SSH(sshConfigPath, sandboxName, mkInputCmd, 10*time.Second); err != nil { + return fmt.Errorf("creating agent-input dir in sandbox: %w", err) + } + if err := sandbox.SCP(sshConfigPath, sandboxName, h.AgentInput+"/.", remoteInput+"/"); err != nil { + printer.StepFail("Failed to copy agent-input files") + return fmt.Errorf("copying agent-input files: %w", err) + } + printer.StepDone("Agent-input files copied") + } + + // 9. Run agent with validation loop. + agentBaseName := strings.TrimSuffix(filepath.Base(h.Agent), ".md") + claudeCmd := buildClaudeCommand(agentBaseName, h.Model, repoDir) + + timeout := time.Duration(h.TimeoutMinutes) * time.Minute + if timeout == 0 { + timeout = 30 * time.Minute + } + + maxIterations := 1 + if h.ValidationLoop != nil && h.ValidationLoop.MaxIterations > 0 { + maxIterations = h.ValidationLoop.MaxIterations + } + + if err := os.MkdirAll(runDir, 0o755); err != nil { + return fmt.Errorf("creating run directory: %w", err) + } + + var lastExitCode int + var validationPassed bool + var runCount int + + for iteration := 1; iteration <= maxIterations; iteration++ { + runCount = iteration + + // Each iteration gets its own subdirectory for output and transcripts. + iterDir := filepath.Join(runDir, fmt.Sprintf("iteration-%d", iteration)) + iterOutputDir := filepath.Join(iterDir, "output") + iterTranscriptDir := filepath.Join(iterDir, "transcripts") + if err := os.MkdirAll(iterDir, 0o755); err != nil { + return fmt.Errorf("creating iteration directory: %w", err) + } + + if maxIterations > 1 { + printer.Blank() + printer.Header(fmt.Sprintf("Iteration %d of %d", iteration, maxIterations)) + } + + // Clear sandbox-side output and transcripts so the next iteration starts fresh. + if iteration > 1 { + clearCmd := fmt.Sprintf("rm -rf %s/output/* %s/*.jsonl", + sandbox.SandboxWorkspace, sandbox.SandboxClaudeConfig) + if _, _, _, clearErr := sandbox.SSH(sshConfigPath, sandboxName, clearCmd, 10*time.Second); clearErr != nil { + printer.StepWarn("Failed to clear sandbox output: " + clearErr.Error()) + } + } + + // 9a. Run agent. + printer.StepStart("Running agent") + printer.Blank() + + exitCode, runErr := sandbox.SSHStream(sshConfigPath, sandboxName, claudeCmd, timeout, os.Stdout, os.Stderr) + if runErr != nil { + printer.StepFail("Agent execution failed") + return fmt.Errorf("running agent (iteration %d): %w", iteration, runErr) + } + lastExitCode = exitCode + + printer.Blank() + // Non-zero exit is a warning, not a failure — the validation loop is the success gate. + if exitCode == 0 { + printer.StepDone(fmt.Sprintf("Agent exited with code %d", exitCode)) + } else { + printer.StepWarn(fmt.Sprintf("Agent exited with code %d", exitCode)) + } + + // 9b. Extract output files. + printer.StepStart("Extracting output files") + remoteSrc := fmt.Sprintf("%s/output", sandbox.SandboxWorkspace) + extracted, extractErr := sandbox.ExtractOutputFiles(sshConfigPath, sandboxName, remoteSrc, iterOutputDir) + if extractErr != nil { + printer.StepWarn("Failed to extract output files: " + extractErr.Error()) + } else if len(extracted) == 0 { + printer.StepInfo("No output files found") + } else { + for _, f := range extracted { + printer.StepInfo(f) + } + printer.StepDone(fmt.Sprintf("Extracted %d output file(s)", len(extracted))) + } + + // 9c. Extract transcripts for this iteration. + printer.StepStart("Extracting transcripts") + if err := sandbox.ExtractTranscripts(sshConfigPath, sandboxName, agentName, iterTranscriptDir); err != nil { + printer.StepWarn("Failed to extract transcripts: " + err.Error()) + } else { + printer.StepDone("Transcripts extracted") + } + + // 9d. Extract target repo back to host. Uses rsync with --no-links + // and --exclude .git/hooks/ to prevent sandbox escape via symlinks + // or injected git hooks. + printer.StepStart("Extracting target repo") + if err := sandbox.RsyncFrom(sshConfigPath, sandboxName, repoDir, repoSrc); err != nil { + printer.StepWarn("Failed to extract target repo: " + err.Error()) + } else { + printer.StepDone("Target repo extracted to " + repoSrc) + } + + // 9e. Run validation. + if h.ValidationLoop == nil { + break + } + + printer.StepStart("Running validation: " + h.ValidationLoop.Script) + valCmd := exec.Command(h.ValidationLoop.Script) + valCmd.Dir = iterDir + valCmd.Env = append(os.Environ(), + append(envToList(h.RunnerEnv), + fmt.Sprintf("TARGET_REPO_DIR=%s", repoSrc), + fmt.Sprintf("FULLSEND_RUN_DIR=%s", runDir), + )..., + ) + valOut, valErr := valCmd.CombinedOutput() + + if valErr == nil { + printer.StepDone("Validation passed: " + strings.TrimSpace(string(valOut))) + validationPassed = true + break + } + + printer.StepFail("Validation failed: " + strings.TrimSpace(string(valOut))) + if iteration < maxIterations { + printer.StepInfo(fmt.Sprintf("Will retry (%d iterations remaining)", maxIterations-iteration)) + } + } + + // 10. Print results. + printer.Blank() + printer.Header("Results") + printer.KeyValue("Run directory", runDir) + printer.KeyValue("Agent exit code", fmt.Sprintf("%d", lastExitCode)) + printer.KeyValue("Agent runs", fmt.Sprintf("%d", runCount)) + if h.ValidationLoop != nil { + if validationPassed { + printer.KeyValue("Validation", "passed") + } else { + printer.KeyValue("Validation", "failed") + } + } + printer.Blank() + + if h.ValidationLoop != nil && !validationPassed { + return fmt.Errorf("validation failed after %d iteration(s)", runCount) + } + + return nil +} + +func bootstrapSandbox(sshConfigPath, sandboxName, repoDir string, h *harness.Harness) error { + // Create workspace structure and Claude config dir for transcripts. + // Agent and skill definitions go in CLAUDE_CONFIG_DIR so `claude --agent` + // finds them regardless of the repo's own .claude/ directory. When + // CLAUDE_CONFIG_DIR is set, Claude uses it instead of ~/.claude/. + mkdirCmd := fmt.Sprintf("mkdir -p %s/agents %s/skills %s/bin %s/.env.d %s", + sandbox.SandboxClaudeConfig, sandbox.SandboxClaudeConfig, sandbox.SandboxWorkspace, sandbox.SandboxWorkspace, sandbox.SandboxClaudeConfig) + if _, _, _, err := sandbox.SSH(sshConfigPath, sandboxName, mkdirCmd, 10*time.Second); err != nil { + return fmt.Errorf("creating workspace dirs: %w", err) + } + + // Copy agent definition to $CLAUDE_CONFIG_DIR/agents/. + if err := sandbox.SCP(sshConfigPath, sandboxName, h.Agent, + fmt.Sprintf("%s/agents/", sandbox.SandboxClaudeConfig)); err != nil { + return fmt.Errorf("copying agent definition: %w", err) + } + + // Copy skills (SCP -r copies the entire directory tree, including any + // scripts/, references/, and assets/ bundled with the skill per the + // agentskills.io specification). + for _, skillPath := range h.Skills { + if err := sandbox.SCP(sshConfigPath, sandboxName, skillPath, + fmt.Sprintf("%s/skills/", sandbox.SandboxClaudeConfig)); err != nil { + return fmt.Errorf("copying skill %q: %w", skillPath, err) + } + } + + // Write .env file (infrastructure vars) and copy host files. + if err := bootstrapEnv(sshConfigPath, sandboxName, repoDir, h); err != nil { + return fmt.Errorf("bootstrapping environment: %w", err) + } + + return nil +} + +// bootstrapEnv writes environment variables to a .env file in the sandbox and +// copies host files. +// +// The .env file contains infrastructure vars (PATH, CLAUDE_CONFIG_DIR) and +// sources all env files from .env.d/. Application-specific env vars (e.g. +// Vertex AI credentials) are delivered as expanded env files via host_files +// with expand: true. +// +// host_files entries copy files from the host into the sandbox at specified +// destination paths. Src values may contain ${VAR} references expanded from +// the host environment. When expand is true, file content is also expanded. +func bootstrapEnv(sshConfigPath, sandboxName, repoDir string, h *harness.Harness) error { + remoteEnvFile := sandbox.SandboxWorkspace + "/.env" + outputDir := sandbox.SandboxWorkspace + "/output" + + var lines []string + + // Infrastructure vars. + lines = append(lines, fmt.Sprintf("export PATH=%s/bin:$PATH", sandbox.SandboxWorkspace)) + lines = append(lines, fmt.Sprintf("export CLAUDE_CONFIG_DIR=%s", sandbox.SandboxClaudeConfig)) + lines = append(lines, fmt.Sprintf("export FULLSEND_OUTPUT_DIR=%s", outputDir)) + lines = append(lines, fmt.Sprintf("export FULLSEND_TARGET_REPO_DIR=%s", repoDir)) + + // Source all env files from .env.d/ (populated by host_files with expand: true). + lines = append(lines, fmt.Sprintf("for f in %s/.env.d/*.env; do [ -f \"$f\" ] && . \"$f\"; done", sandbox.SandboxWorkspace)) + + content := strings.Join(lines, "\n") + "\n" + + tmpFile, err := os.CreateTemp("", "fullsend-env-*.sh") + if err != nil { + return fmt.Errorf("creating temp env file: %w", err) + } + defer os.Remove(tmpFile.Name()) + + if _, err := tmpFile.WriteString(content); err != nil { + tmpFile.Close() + return fmt.Errorf("writing temp env file: %w", err) + } + tmpFile.Close() + + if err := sandbox.SCP(sshConfigPath, sandboxName, tmpFile.Name(), remoteEnvFile); err != nil { + return fmt.Errorf("copying .env file to sandbox: %w", err) + } + + // Copy host files into the sandbox. + for _, hf := range h.HostFiles { + hostPath := os.ExpandEnv(hf.Src) + if hostPath == "" { + return fmt.Errorf("host_files: src %q expanded to empty string", hf.Src) + } + + if hf.Expand { + // Read file, expand ${VAR} in content, write expanded version. + raw, err := os.ReadFile(hostPath) + if err != nil { + return fmt.Errorf("reading host file %s for expansion: %w", hf.Src, err) + } + expanded := os.ExpandEnv(string(raw)) + + tmp, err := os.CreateTemp("", "fullsend-expand-*") + if err != nil { + return fmt.Errorf("creating temp file for expanded %s: %w", hf.Src, err) + } + if _, err := tmp.WriteString(expanded); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return fmt.Errorf("writing expanded %s: %w", hf.Src, err) + } + tmp.Close() + + if err := sandbox.SCP(sshConfigPath, sandboxName, tmp.Name(), hf.Dest); err != nil { + os.Remove(tmp.Name()) + return fmt.Errorf("copying expanded file %s to %s: %w", hf.Src, hf.Dest, err) + } + os.Remove(tmp.Name()) + } else { + if err := sandbox.SCP(sshConfigPath, sandboxName, hostPath, hf.Dest); err != nil { + return fmt.Errorf("copying host file %s to %s: %w", hf.Src, hf.Dest, err) + } + } + } + + return nil +} + +// envToList converts a map of env vars to a sorted list of KEY=VALUE strings. +func envToList(env map[string]string) []string { + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + list := make([]string, 0, len(env)) + for _, k := range keys { + list = append(list, fmt.Sprintf("%s=%s", k, env[k])) + } + return list +} + +func buildClaudeCommand(agentName, model, repoDir string) string { + envFile := sandbox.SandboxWorkspace + "/.env" + + // Defense-in-depth: escape single quotes even though Validate() rejects them. + safe := strings.ReplaceAll(agentName, "'", "'\\''") + + modelFlag := "" + if model != "" { + modelFlag = fmt.Sprintf("--model '%s' ", strings.ReplaceAll(model, "'", "'\\''")) + } + + return fmt.Sprintf( + "cd %s && source %s && claude --print %s--agent '%s' --dangerously-skip-permissions 'Run the agent task'", + repoDir, envFile, modelFlag, safe, + ) +} diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go new file mode 100644 index 0000000000..6eb1c91fde --- /dev/null +++ b/internal/cli/run_test.go @@ -0,0 +1,87 @@ +package cli + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunCommand_RequiresAgentName(t *testing.T) { + cmd := newRunCmd() + cmd.SetArgs([]string{}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "accepts 1 arg(s)") +} + +func TestRunCommand_HasFullsendDirFlag(t *testing.T) { + cmd := newRunCmd() + flag := cmd.Flags().Lookup("fullsend-dir") + require.NotNil(t, flag) + assert.Equal(t, "", flag.DefValue) + + annotations := flag.Annotations + require.Contains(t, annotations, "cobra_annotation_bash_completion_one_required_flag") +} + +func TestRunCommand_RegisteredOnRoot(t *testing.T) { + root := newRootCmd() + found := false + for _, sub := range root.Commands() { + if sub.Name() == "run" { + found = true + break + } + } + assert.True(t, found, "run command should be registered on root") +} + +func TestRunCommand_HasOutputDirFlag(t *testing.T) { + cmd := newRunCmd() + flag := cmd.Flags().Lookup("output-dir") + require.NotNil(t, flag) + assert.Equal(t, "", flag.DefValue) +} + +func TestRunCommand_HasTargetRepoFlag(t *testing.T) { + cmd := newRunCmd() + flag := cmd.Flags().Lookup("target-repo") + require.NotNil(t, flag) + assert.Equal(t, "", flag.DefValue) + + annotations := flag.Annotations + require.Contains(t, annotations, "cobra_annotation_bash_completion_one_required_flag") +} + +func TestBuildClaudeCommand_Basic(t *testing.T) { + cmd := buildClaudeCommand("hello-world", "", "/tmp/workspace/repo") + assert.Contains(t, cmd, "cd /tmp/workspace/repo") + assert.Contains(t, cmd, "--agent 'hello-world'") + assert.NotContains(t, cmd, "--model") +} + +func TestBuildClaudeCommand_WithModel(t *testing.T) { + cmd := buildClaudeCommand("hello-world", "sonnet", "/tmp/workspace/repo") + assert.Contains(t, cmd, "--model 'sonnet'") + assert.Contains(t, cmd, "--agent 'hello-world'") +} + +func TestBuildClaudeCommand_EscapesQuotes(t *testing.T) { + cmd := buildClaudeCommand("test'name", "", "/tmp/workspace/repo") + assert.NotContains(t, cmd, "'test'name'") + assert.Contains(t, cmd, "'test'\\''name'") +} + +func TestEnvToList_Sorted(t *testing.T) { + env := map[string]string{ + "Z_VAR": "z", + "A_VAR": "a", + "M_VAR": "m", + } + list := envToList(env) + require.Len(t, list, 3) + assert.Equal(t, "A_VAR=a", list[0]) + assert.Equal(t, "M_VAR=m", list[1]) + assert.Equal(t, "Z_VAR=z", list[2]) +} diff --git a/internal/harness/harness.go b/internal/harness/harness.go new file mode 100644 index 0000000000..38aa405087 --- /dev/null +++ b/internal/harness/harness.go @@ -0,0 +1,296 @@ +package harness + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +var ( + validAgentName = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + envVarRef = regexp.MustCompile(`\$\{([^}]+)\}`) +) + +// HostFile describes a file on the host that must be copied into the sandbox +// during bootstrap. Src may contain ${VAR} references that are expanded from +// the host environment at bootstrap time. Use this for any file that must +// exist inside the sandbox (e.g. GCP service account JSON, CA certificates). +// +// When Expand is true, the file content is read and ${VAR} references in the +// content are expanded from the host environment before copying to the sandbox. +// Use this for env files that contain variable references which must be resolved +// on the host (because the sandbox does not have those variables set). +type HostFile struct { + Src string `yaml:"src"` // host path (may use ${VAR} expansion) + Dest string `yaml:"dest"` // destination path inside the sandbox + Expand bool `yaml:"expand,omitempty"` // expand ${VAR} in file content before copying +} + +// ProviderDef is a declarative definition of an OpenShell provider. Files in +// the experiment's providers/ directory are loaded as ProviderDefs and +// reconciled against the gateway before sandbox creation. +type ProviderDef struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Credentials map[string]string `yaml:"credentials"` // KEY: VALUE or KEY: ${HOST_VAR} + Config map[string]string `yaml:"config,omitempty"` // e.g. OPENAI_BASE_URL +} + +// LoadProviderDefs reads all YAML files from a providers/ directory and returns +// the parsed definitions. Returns nil (no error) if the directory does not exist. +func LoadProviderDefs(dir string) ([]ProviderDef, error) { + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading providers dir: %w", err) + } + + var defs []ProviderDef + for _, e := range entries { + if e.IsDir() || (!strings.HasSuffix(e.Name(), ".yaml") && !strings.HasSuffix(e.Name(), ".yml")) { + continue + } + data, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return nil, fmt.Errorf("reading provider file %s: %w", e.Name(), err) + } + var def ProviderDef + if err := yaml.Unmarshal(data, &def); err != nil { + return nil, fmt.Errorf("parsing provider file %s: %w", e.Name(), err) + } + if def.Name == "" { + return nil, fmt.Errorf("provider file %s: name is required", e.Name()) + } + if def.Type == "" { + return nil, fmt.Errorf("provider file %s: type is required", e.Name()) + } + defs = append(defs, def) + } + return defs, nil +} + +// APIServer describes a host-side REST proxy server. +type APIServer struct { + Name string `yaml:"name"` + Script string `yaml:"script"` + Port int `yaml:"port"` + Env map[string]string `yaml:"env,omitempty"` +} + +// ValidationLoop configures a deterministic validation step after the agent exits. +type ValidationLoop struct { + Script string `yaml:"script"` + MaxIterations int `yaml:"max_iterations"` + FeedbackMode string `yaml:"feedback_mode,omitempty"` +} + +// Harness is the per-agent configuration that the runner reads to provision +// a sandbox and launch one agent. It follows the ADR-0017 schema. +type Harness struct { + Agent string `yaml:"agent"` + Description string `yaml:"description,omitempty"` + Image string `yaml:"image,omitempty"` + Policy string `yaml:"policy,omitempty"` + Skills []string `yaml:"skills,omitempty"` + Providers []string `yaml:"providers,omitempty"` + HostFiles []HostFile `yaml:"host_files,omitempty"` + APIServers []APIServer `yaml:"api_servers,omitempty"` + Model string `yaml:"model,omitempty"` + PreScript string `yaml:"pre_script,omitempty"` + PostScript string `yaml:"post_script,omitempty"` + AgentInput string `yaml:"agent_input,omitempty"` + ValidationLoop *ValidationLoop `yaml:"validation_loop,omitempty"` + RunnerEnv map[string]string `yaml:"runner_env,omitempty"` + TimeoutMinutes int `yaml:"timeout_minutes,omitempty"` +} + +// Load reads a harness YAML file from path, unmarshals it, and validates it. +func Load(path string) (*Harness, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading harness file: %w", err) + } + + var h Harness + if err := yaml.Unmarshal(data, &h); err != nil { + return nil, fmt.Errorf("parsing harness YAML: %w", err) + } + + if err := h.Validate(); err != nil { + return nil, fmt.Errorf("invalid harness: %w", err) + } + + return &h, nil +} + +// Validate checks that required fields are present. +func (h *Harness) Validate() error { + if h.Agent == "" { + return fmt.Errorf("agent field is required") + } + // Agent name (filename without .md) must be safe for shell interpolation. + agentBase := strings.TrimSuffix(filepath.Base(h.Agent), ".md") + if !validAgentName.MatchString(agentBase) { + return fmt.Errorf("agent name %q contains invalid characters (allowed: a-z, A-Z, 0-9, _, -)", agentBase) + } + if h.Model != "" && !validAgentName.MatchString(h.Model) { + return fmt.Errorf("model %q contains invalid characters (allowed: a-z, A-Z, 0-9, _, -)", h.Model) + } + if h.TimeoutMinutes < 0 { + return fmt.Errorf("timeout_minutes must be non-negative, got %d", h.TimeoutMinutes) + } + for i, hf := range h.HostFiles { + if hf.Src == "" { + return fmt.Errorf("host_files[%d]: src is required", i) + } + if hf.Dest == "" { + return fmt.Errorf("host_files[%d]: dest is required", i) + } + } + if h.ValidationLoop != nil && h.ValidationLoop.Script == "" { + return fmt.Errorf("validation_loop.script is required when validation_loop is set") + } + return nil +} + +// ResolveRelativeTo resolves all relative paths in the harness against baseDir. +// Relative paths that resolve outside baseDir are rejected to prevent directory +// traversal (e.g. ../../etc/shadow). Absolute paths and ${VAR} paths are allowed. +func (h *Harness) ResolveRelativeTo(baseDir string) error { + cleanBase := filepath.Clean(baseDir) + string(filepath.Separator) + + resolve := func(field, p string) (string, error) { + if p == "" || filepath.IsAbs(p) { + return p, nil + } + resolved := filepath.Join(baseDir, p) + if !strings.HasPrefix(filepath.Clean(resolved), cleanBase) { + return "", fmt.Errorf("%s: path %q resolves outside fullsend directory", field, p) + } + return resolved, nil + } + + var err error + if h.Agent, err = resolve("agent", h.Agent); err != nil { + return err + } + if h.Policy, err = resolve("policy", h.Policy); err != nil { + return err + } + if h.PreScript, err = resolve("pre_script", h.PreScript); err != nil { + return err + } + if h.PostScript, err = resolve("post_script", h.PostScript); err != nil { + return err + } + if h.AgentInput, err = resolve("agent_input", h.AgentInput); err != nil { + return err + } + + for i := range h.Skills { + if h.Skills[i], err = resolve(fmt.Sprintf("skills[%d]", i), h.Skills[i]); err != nil { + return err + } + } + for i, hf := range h.HostFiles { + if !strings.Contains(hf.Src, "${") { + if h.HostFiles[i].Src, err = resolve(fmt.Sprintf("host_files[%d].src", i), hf.Src); err != nil { + return err + } + } + } + for i := range h.APIServers { + if h.APIServers[i].Script, err = resolve(fmt.Sprintf("api_servers[%d].script", i), h.APIServers[i].Script); err != nil { + return err + } + } + if h.ValidationLoop != nil { + if h.ValidationLoop.Script, err = resolve("validation_loop.script", h.ValidationLoop.Script); err != nil { + return err + } + } + return nil +} + +// ValidateRunnerEnv checks that all ${VAR} references in RunnerEnv and +// HostFiles.Src expand to non-empty values in the host environment. +func (h *Harness) ValidateRunnerEnv() error { + checkVarRefs := func(source, value string) error { + for _, match := range envVarRef.FindAllStringSubmatch(value, -1) { + varName := match[1] + if os.Getenv(varName) == "" { + return fmt.Errorf("%s: host variable %s is not set (referenced in %q)", source, varName, value) + } + } + return nil + } + + for k, v := range h.RunnerEnv { + if err := checkVarRefs(fmt.Sprintf("runner_env[%s]", k), v); err != nil { + return err + } + } + for i, hf := range h.HostFiles { + if err := checkVarRefs(fmt.Sprintf("host_files[%d].src", i), hf.Src); err != nil { + return err + } + } + return nil +} + +// ValidateFilesExist checks that all file paths referenced by the harness +// exist on disk. Call after ResolveRelativeTo so paths are absolute. +// Pre/post scripts run on the host and must be file paths (no inline args). +func (h *Harness) ValidateFilesExist() error { + check := func(label, path string) error { + if path == "" { + return nil + } + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("%s: %w", label, err) + } + return nil + } + + if err := check("agent", h.Agent); err != nil { + return err + } + if err := check("policy", h.Policy); err != nil { + return err + } + if err := check("pre_script", h.PreScript); err != nil { + return err + } + if err := check("post_script", h.PostScript); err != nil { + return err + } + if err := check("agent_input", h.AgentInput); err != nil { + return err + } + for i, s := range h.Skills { + if err := check(fmt.Sprintf("skills[%d]", i), s); err != nil { + return err + } + } + for i, hf := range h.HostFiles { + // Skip ${VAR} paths — they are expanded at bootstrap time. + if strings.Contains(hf.Src, "${") { + continue + } + if err := check(fmt.Sprintf("host_files[%d].src", i), hf.Src); err != nil { + return err + } + } + if h.ValidationLoop != nil { + if err := check("validation_loop.script", h.ValidationLoop.Script); err != nil { + return err + } + } + return nil +} diff --git a/internal/harness/harness_test.go b/internal/harness/harness_test.go new file mode 100644 index 0000000000..77d59591c6 --- /dev/null +++ b/internal/harness/harness_test.go @@ -0,0 +1,343 @@ +package harness + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoad_ValidHarness(t *testing.T) { + content := ` +agent: agents/hello-world.md +image: registry.example.com/sandbox:v1 +skills: + - skills/hello-world-summary +validation_loop: + script: scripts/validate-output.sh + max_iterations: 1 +runner_env: + REPO_NAME: "${REPO_NAME}" +timeout_minutes: 5 +` + dir := t.TempDir() + path := filepath.Join(dir, "hello-world.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + h, err := Load(path) + require.NoError(t, err) + + assert.Equal(t, "agents/hello-world.md", h.Agent) + assert.Equal(t, "registry.example.com/sandbox:v1", h.Image) + assert.Equal(t, []string{"skills/hello-world-summary"}, h.Skills) + require.NotNil(t, h.ValidationLoop) + assert.Equal(t, "scripts/validate-output.sh", h.ValidationLoop.Script) + assert.Equal(t, 1, h.ValidationLoop.MaxIterations) + assert.Equal(t, `${REPO_NAME}`, h.RunnerEnv["REPO_NAME"]) + assert.Equal(t, 5, h.TimeoutMinutes) +} + +func TestResolveRelativeTo_ImageUnchanged(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + Image: "registry.example.com/sandbox:v1", + } + + require.NoError(t, h.ResolveRelativeTo("/base/dir")) + + // Image is a registry reference, not a filesystem path — must not be resolved. + assert.Equal(t, "registry.example.com/sandbox:v1", h.Image) +} + +func TestLoad_MissingAgent(t *testing.T) { + content := ` +skills: + - skills/hello-world-summary +` + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + _, err := Load(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "agent field is required") +} + +func TestLoad_ValidationLoopMissingScript(t *testing.T) { + content := ` +agent: agents/test.md +validation_loop: + max_iterations: 3 +` + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + _, err := Load(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "validation_loop.script is required") +} + +func TestLoad_HostFiles(t *testing.T) { + content := ` +agent: agents/test.md +host_files: + - src: ${GOOGLE_APPLICATION_CREDENTIALS} + dest: /tmp/workspace/.gcp-credentials.json + - src: /etc/ssl/certs/ca-certificates.crt + dest: /etc/ssl/certs/ca-certificates.crt + - src: env/gcp-vertex.env + dest: /tmp/workspace/.env.d/gcp-vertex.env + expand: true +` + dir := t.TempDir() + path := filepath.Join(dir, "test.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + h, err := Load(path) + require.NoError(t, err) + + require.Len(t, h.HostFiles, 3) + assert.Equal(t, "${GOOGLE_APPLICATION_CREDENTIALS}", h.HostFiles[0].Src) + assert.Equal(t, "/tmp/workspace/.gcp-credentials.json", h.HostFiles[0].Dest) + assert.False(t, h.HostFiles[0].Expand) + assert.Equal(t, "/etc/ssl/certs/ca-certificates.crt", h.HostFiles[1].Src) + assert.Equal(t, "/etc/ssl/certs/ca-certificates.crt", h.HostFiles[1].Dest) + assert.False(t, h.HostFiles[1].Expand) + assert.Equal(t, "env/gcp-vertex.env", h.HostFiles[2].Src) + assert.Equal(t, "/tmp/workspace/.env.d/gcp-vertex.env", h.HostFiles[2].Dest) + assert.True(t, h.HostFiles[2].Expand) +} + +func TestValidate_HostFileMissingSrc(t *testing.T) { + content := ` +agent: agents/test.md +host_files: + - dest: /tmp/workspace/.gcp-credentials.json +` + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + _, err := Load(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "host_files[0]: src is required") +} + +func TestValidate_HostFileMissingDest(t *testing.T) { + content := ` +agent: agents/test.md +host_files: + - src: ${GOOGLE_APPLICATION_CREDENTIALS} +` + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + _, err := Load(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "host_files[0]: dest is required") +} + +func TestResolveRelativeTo(t *testing.T) { + h := &Harness{ + Agent: "agents/hello-world.md", + Policy: "policies/readonly.yaml", + Skills: []string{"skills/hello-world-summary"}, + PreScript: "scripts/pre.sh", + PostScript: "scripts/post.sh", + AgentInput: "agent-input", + ValidationLoop: &ValidationLoop{ + Script: "scripts/validate.sh", + }, + } + + require.NoError(t, h.ResolveRelativeTo("/base/dir")) + + assert.Equal(t, "/base/dir/agents/hello-world.md", h.Agent) + assert.Equal(t, "/base/dir/policies/readonly.yaml", h.Policy) + assert.Equal(t, []string{"/base/dir/skills/hello-world-summary"}, h.Skills) + assert.Equal(t, "/base/dir/scripts/pre.sh", h.PreScript) + assert.Equal(t, "/base/dir/scripts/post.sh", h.PostScript) + assert.Equal(t, "/base/dir/agent-input", h.AgentInput) + assert.Equal(t, "/base/dir/scripts/validate.sh", h.ValidationLoop.Script) +} + +func TestResolveRelativeTo_HostFiles(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + HostFiles: []HostFile{ + {Src: "env/gcp-vertex.env", Dest: "/tmp/workspace/.env.d/gcp-vertex.env", Expand: true}, + {Src: "${GOOGLE_APPLICATION_CREDENTIALS}", Dest: "/tmp/workspace/.gcp-credentials.json"}, + {Src: "/absolute/path/file.txt", Dest: "/tmp/workspace/file.txt"}, + }, + } + + require.NoError(t, h.ResolveRelativeTo("/base/dir")) + + // Relative path without ${VAR} gets resolved. + assert.Equal(t, "/base/dir/env/gcp-vertex.env", h.HostFiles[0].Src) + // ${VAR} path is NOT resolved (expanded at bootstrap time). + assert.Equal(t, "${GOOGLE_APPLICATION_CREDENTIALS}", h.HostFiles[1].Src) + // Absolute path is unchanged. + assert.Equal(t, "/absolute/path/file.txt", h.HostFiles[2].Src) +} + +func TestResolveRelativeTo_AbsolutePathsUnchanged(t *testing.T) { + h := &Harness{ + Agent: "/absolute/path/agent.md", + } + + require.NoError(t, h.ResolveRelativeTo("/base/dir")) + + assert.Equal(t, "/absolute/path/agent.md", h.Agent) +} + +func TestResolveRelativeTo_TraversalRejected(t *testing.T) { + h := &Harness{Agent: "../../etc/shadow.md"} + err := h.ResolveRelativeTo("/base/dir") + require.Error(t, err) + assert.Contains(t, err.Error(), "resolves outside fullsend directory") +} + +func TestResolveRelativeTo_HostFileTraversalRejected(t *testing.T) { + h := &Harness{ + Agent: "agents/test.md", + HostFiles: []HostFile{ + {Src: "../../../etc/shadow", Dest: "/tmp/workspace/shadow"}, + }, + } + err := h.ResolveRelativeTo("/base/dir") + require.Error(t, err) + assert.Contains(t, err.Error(), "resolves outside fullsend directory") +} + +func TestLoad_FileNotFound(t *testing.T) { + _, err := Load("/nonexistent/path.yaml") + require.Error(t, err) + assert.Contains(t, err.Error(), "reading harness file") +} + +func TestValidateRunnerEnv_UnsetVar(t *testing.T) { + h := &Harness{ + Agent: "test.md", + RunnerEnv: map[string]string{"KEY": "${DEFINITELY_NOT_SET_VAR_XYZ}"}, + } + err := h.ValidateRunnerEnv() + require.Error(t, err) + assert.Contains(t, err.Error(), "DEFINITELY_NOT_SET_VAR_XYZ") +} + +func TestValidateRunnerEnv_LiteralValue(t *testing.T) { + h := &Harness{ + Agent: "test.md", + RunnerEnv: map[string]string{"KEY": "literal_value"}, + } + require.NoError(t, h.ValidateRunnerEnv()) +} + +func TestValidateRunnerEnv_HostFileSrcUnset(t *testing.T) { + h := &Harness{ + Agent: "test.md", + HostFiles: []HostFile{ + {Src: "${DEFINITELY_NOT_SET_VAR_XYZ}", Dest: "/tmp/dest"}, + }, + } + err := h.ValidateRunnerEnv() + require.Error(t, err) + assert.Contains(t, err.Error(), "DEFINITELY_NOT_SET_VAR_XYZ") +} + +func TestValidateRunnerEnv_PartialExpansion(t *testing.T) { + h := &Harness{ + Agent: "test.md", + RunnerEnv: map[string]string{"ENDPOINT": "https://${DEFINITELY_NOT_SET_VAR_XYZ}/api"}, + } + err := h.ValidateRunnerEnv() + require.Error(t, err) + assert.Contains(t, err.Error(), "DEFINITELY_NOT_SET_VAR_XYZ") +} + +func TestValidate_AgentNameInvalid(t *testing.T) { + h := &Harness{Agent: "agents/test';echo hack;echo '.md"} + err := h.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid characters") +} + +func TestValidate_AgentNameValid(t *testing.T) { + h := &Harness{Agent: "agents/hello-world_v2.md"} + require.NoError(t, h.Validate()) +} + +func TestValidate_ModelInvalid(t *testing.T) { + h := &Harness{Agent: "agents/test.md", Model: "sonnet'; echo hack"} + err := h.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "model") + assert.Contains(t, err.Error(), "invalid characters") +} + +func TestValidate_ModelValid(t *testing.T) { + h := &Harness{Agent: "agents/test.md", Model: "claude-sonnet-4-6"} + require.NoError(t, h.Validate()) +} + +func TestValidate_NegativeTimeout(t *testing.T) { + h := &Harness{Agent: "agents/test.md", TimeoutMinutes: -1} + err := h.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "timeout_minutes must be non-negative") +} + +func TestLoad_ModelField(t *testing.T) { + content := ` +agent: agents/test.md +model: sonnet +` + dir := t.TempDir() + path := filepath.Join(dir, "test.yaml") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + h, err := Load(path) + require.NoError(t, err) + assert.Equal(t, "sonnet", h.Model) +} + +func TestValidateFilesExist_MissingAgent(t *testing.T) { + h := &Harness{Agent: "/nonexistent/agent.md"} + err := h.ValidateFilesExist() + require.Error(t, err) + assert.Contains(t, err.Error(), "agent") +} + +func TestValidateFilesExist_MissingSkill(t *testing.T) { + dir := t.TempDir() + agentFile := filepath.Join(dir, "agent.md") + require.NoError(t, os.WriteFile(agentFile, []byte("agent"), 0o644)) + + h := &Harness{ + Agent: agentFile, + Skills: []string{"/nonexistent/skill"}, + } + err := h.ValidateFilesExist() + require.Error(t, err) + assert.Contains(t, err.Error(), "skills[0]") +} + +func TestValidateFilesExist_SkipsVarPaths(t *testing.T) { + dir := t.TempDir() + agentFile := filepath.Join(dir, "agent.md") + require.NoError(t, os.WriteFile(agentFile, []byte("agent"), 0o644)) + + h := &Harness{ + Agent: agentFile, + HostFiles: []HostFile{ + {Src: "${SOME_VAR}", Dest: "/tmp/dest"}, + }, + } + // Should not error — ${VAR} paths are expanded at bootstrap time. + require.NoError(t, h.ValidateFilesExist()) +} diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go new file mode 100644 index 0000000000..365156d79d --- /dev/null +++ b/internal/sandbox/sandbox.go @@ -0,0 +1,415 @@ +package sandbox + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +const ( + // SandboxWorkspace is the workspace directory inside the sandbox. + SandboxWorkspace = "/tmp/workspace" //nolint:gosec // not a credential + // SandboxClaudeConfig is the Claude config directory inside the sandbox. + SandboxClaudeConfig = "/tmp/claude-config" //nolint:gosec // not a credential + + createTimeout = 65 * time.Second + readyTimeout = 60 * time.Second + readyPoll = 2 * time.Second + transferTimeout = 5 * time.Minute +) + +// EnsureProvider creates or updates a provider on the gateway. Credential +// values may contain ${VAR} references which are expanded from the host +// environment before being passed to openshell. +// +// Credentials use the bare-key form (--credential KEY) so that secret values +// never appear on the process command line. The expanded values are injected +// into the child process environment, where openshell reads them directly. +// See https://docs.nvidia.com/openshell/latest/sandboxes/manage-providers#bare-key-form +func EnsureProvider(name, providerType string, credentials, config map[string]string) error { + args, extraEnv, secrets := buildProviderArgs(name, providerType, credentials, config) + + cmd := exec.Command("openshell", args...) + cmd.Env = append(os.Environ(), extraEnv...) + out, err := cmd.CombinedOutput() + if err != nil { + // Redact known credential values from error output. + outStr := string(out) + for _, s := range secrets { + outStr = strings.ReplaceAll(outStr, s, "***") + } + return fmt.Errorf("provider create %q failed: %s", name, outStr) + } + return nil +} + +// buildProviderArgs constructs the CLI args and child environment entries for +// openshell provider create. Credentials use the bare-key form (--credential KEY) +// so secret values never appear on the process command line. The expanded values +// are returned as extra env vars to be set on the child process. +// See https://docs.nvidia.com/openshell/latest/sandboxes/manage-providers#bare-key-form +func buildProviderArgs(name, providerType string, credentials, config map[string]string) (args, extraEnv, secrets []string) { + args = []string{"provider", "create", + "--name", name, + "--type", providerType, + } + + for k, v := range credentials { + expanded := os.ExpandEnv(v) + if expanded != "" { + secrets = append(secrets, expanded) + } + extraEnv = append(extraEnv, fmt.Sprintf("%s=%s", k, expanded)) + args = append(args, "--credential", k) + } + for k, v := range config { + expanded := os.ExpandEnv(v) + args = append(args, "--config", k+"="+expanded) + } + + return args, extraEnv, secrets +} + +// EnsureAvailable checks that the openshell binary is in PATH. +func EnsureAvailable() error { + _, err := exec.LookPath("openshell") + if err != nil { + return fmt.Errorf("openshell not found in PATH: %w", err) + } + return nil +} + +// EnsureGateway starts a local gateway if none is active. It is idempotent — +// if a gateway is already running the command is a no-op. +func EnsureGateway() error { + // Check if a gateway is already active. + check := exec.Command("openshell", "gateway", "info") + if err := check.Run(); err == nil { + return nil + } + + cmd := exec.Command("openshell", "gateway", "start") + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("gateway start failed: %s", string(out)) + } + return nil +} + +// Create creates a persistent OpenShell sandbox and waits for it to be ready. +// If providers are given, they are passed as --provider flags. If image is +// non-empty, it is passed as --from to start the sandbox from a container image. +// If policy is non-empty, it is applied at creation time via --policy. +func Create(name string, providers []string, image, policy string) error { + ctx, cancel := context.WithTimeout(context.Background(), createTimeout) + defer cancel() + + args := []string{"60", + "openshell", "sandbox", "create", + "--name", name, + "--keep", + "--no-auto-providers", + "--no-tty", + } + if image != "" { + args = append(args, "--from", image) + } + if policy != "" { + args = append(args, "--policy", policy) + } + for _, p := range providers { + args = append(args, "--provider", p) + } + cmd := exec.CommandContext(ctx, "timeout", args...) + cmd.Stdin = nil + out, err := cmd.CombinedOutput() + + // timeout exits 124 — sandbox create may exit non-zero after the + // interactive shell is killed. Check if the sandbox actually exists. + if err != nil && (cmd.ProcessState == nil || cmd.ProcessState.ExitCode() != 124) { + check := exec.Command("openshell", "sandbox", "get", name) + if checkErr := check.Run(); checkErr != nil { + return fmt.Errorf("sandbox create failed: %s", string(out)) + } + } + + // Wait for sandbox to be fully ready (image pull can take a while). + deadline := time.Now().Add(readyTimeout) + for time.Now().Before(deadline) { + check := exec.Command("openshell", "sandbox", "get", name) + output, checkErr := check.Output() + if checkErr == nil && strings.Contains(string(output), "Ready") { + return nil + } + time.Sleep(readyPoll) + } + + return fmt.Errorf("sandbox %q not ready after %s", name, readyTimeout) +} + +// Delete deletes a sandbox, returning any error for the caller to log. +func Delete(name string) error { + out, err := exec.Command("openshell", "sandbox", "delete", name).CombinedOutput() + if err != nil { + return fmt.Errorf("sandbox delete %q failed: %s", name, string(out)) + } + return nil +} + +// GetSSHConfig retrieves the SSH config for a sandbox. +func GetSSHConfig(name string) (string, error) { + out, err := exec.Command("openshell", "sandbox", "ssh-config", name).Output() + if err != nil { + return "", fmt.Errorf("getting SSH config for sandbox %q: %w", name, err) + } + return string(out), nil +} + +// SCP copies a local file or directory into a sandbox. +func SCP(sshConfigPath, sandboxName, localPath, remotePath string) error { + ctx, cancel := context.WithTimeout(context.Background(), transferTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "scp", + "-F", sshConfigPath, + "-r", + localPath, + fmt.Sprintf("openshell-%s:%s", sandboxName, remotePath), + ) + out, err := cmd.CombinedOutput() + if err != nil { + if ctx.Err() != nil { + return fmt.Errorf("scp to sandbox %q timed out after %s", sandboxName, transferTimeout) + } + return fmt.Errorf("scp to sandbox %q failed: %s: %w", sandboxName, string(out), err) + } + return nil +} + +// SSH runs a command inside a sandbox and returns stdout, stderr, and exit code. +func SSH(sshConfigPath, sandboxName, command string, timeout time.Duration) (stdout, stderr string, exitCode int, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "ssh", + "-F", sshConfigPath, + fmt.Sprintf("openshell-%s", sandboxName), + command, + ) + + var stdoutBuf, stderrBuf strings.Builder + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + + runErr := cmd.Run() + exitCode = -1 + if cmd.ProcessState != nil { + exitCode = cmd.ProcessState.ExitCode() + } + + if runErr != nil && ctx.Err() != nil { + return stdoutBuf.String(), stderrBuf.String(), exitCode, + fmt.Errorf("ssh command timed out after %s", timeout) + } + + if runErr != nil && cmd.ProcessState == nil { + return "", "", exitCode, fmt.Errorf("ssh failed to start: %w", runErr) + } + + return stdoutBuf.String(), stderrBuf.String(), exitCode, nil +} + +// SSHStream runs a command inside a sandbox, streaming output to the given writers. +func SSHStream(sshConfigPath, sandboxName, command string, timeout time.Duration, stdoutW, stderrW *os.File) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "ssh", + "-F", sshConfigPath, + fmt.Sprintf("openshell-%s", sandboxName), + command, + ) + cmd.Stdout = stdoutW + cmd.Stderr = stderrW + + err := cmd.Run() + exitCode := -1 + if cmd.ProcessState != nil { + exitCode = cmd.ProcessState.ExitCode() + } + + if err != nil && ctx.Err() != nil { + return exitCode, fmt.Errorf("ssh command timed out after %s", timeout) + } + + if err != nil && cmd.ProcessState == nil { + return exitCode, fmt.Errorf("ssh failed to start: %w", err) + } + + return exitCode, nil +} + +// RsyncFrom copies a directory from a sandbox to the local machine using rsync +// with safety flags: symlinks are skipped (--no-links) and .git/hooks/ is +// excluded to prevent a compromised sandbox from injecting executable content +// into the host repo. Requires rsync on both host and sandbox. +func RsyncFrom(sshConfigPath, sandboxName, remoteDir, localDir string) error { + // Trailing slashes ensure rsync copies contents, not the directory itself. + if !strings.HasSuffix(remoteDir, "/") { + remoteDir += "/" + } + if !strings.HasSuffix(localDir, "/") { + localDir += "/" + } + + ctx, cancel := context.WithTimeout(context.Background(), transferTimeout) + defer cancel() + + remote := fmt.Sprintf("openshell-%s:%s", sandboxName, remoteDir) + cmd := exec.CommandContext(ctx, "rsync", + "-a", + "--no-links", + "--exclude", ".git/hooks/", + "-e", fmt.Sprintf("ssh -F %s", sshConfigPath), + remote, + localDir, + ) + out, err := cmd.CombinedOutput() + if err != nil { + if ctx.Err() != nil { + return fmt.Errorf("rsync from sandbox %q timed out after %s", sandboxName, transferTimeout) + } + return fmt.Errorf("rsync from sandbox %q failed: %s: %w", sandboxName, string(out), err) + } + return nil +} + +// SCPFrom copies a file or directory from a sandbox to the local machine. +func SCPFrom(sshConfigPath, sandboxName, remotePath, localPath string) error { + ctx, cancel := context.WithTimeout(context.Background(), transferTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, "scp", + "-F", sshConfigPath, + "-r", + fmt.Sprintf("openshell-%s:%s", sandboxName, remotePath), + localPath, + ) + out, err := cmd.CombinedOutput() + if err != nil { + if ctx.Err() != nil { + return fmt.Errorf("scp from sandbox %q timed out after %s", sandboxName, transferTimeout) + } + return fmt.Errorf("scp from sandbox %q failed: %s: %w", sandboxName, string(out), err) + } + return nil +} + +// ExtractTranscripts copies Claude transcript files (.jsonl) from the sandbox +// to a local output directory. +func ExtractTranscripts(sshConfigPath, sandboxName, agentName, outputDir string) error { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return fmt.Errorf("creating output dir: %w", err) + } + + // Find transcript files in the sandbox. + stdout, _, _, err := SSH(sshConfigPath, sandboxName, + fmt.Sprintf("find %s -name '*.jsonl' 2>/dev/null || true", SandboxClaudeConfig), + 10*time.Second, + ) + if err != nil { + return fmt.Errorf("finding transcripts: %w", err) + } + + trimmed := strings.TrimSpace(stdout) + if trimmed == "" { + fmt.Fprintf(os.Stderr, " [%s] No transcripts found\n", agentName) + return nil + } + files := strings.Split(trimmed, "\n") + + cleanBase := filepath.Clean(outputDir) + string(filepath.Separator) + + for _, remotePath := range files { + remotePath = strings.TrimSpace(remotePath) + if remotePath == "" { + continue + } + localName := fmt.Sprintf("%s-%s", agentName, filepath.Base(remotePath)) + localPath := filepath.Join(outputDir, localName) + + // Prevent path traversal from sandbox-controlled filenames. + if !strings.HasPrefix(filepath.Clean(localPath), cleanBase) { + fmt.Fprintf(os.Stderr, " [%s] Skipping path traversal attempt: %s\n", agentName, localName) + continue + } + + if scpErr := SCPFrom(sshConfigPath, sandboxName, remotePath, localPath); scpErr != nil { + fmt.Fprintf(os.Stderr, " [%s] Failed to copy transcript: %v\n", agentName, scpErr) + continue + } + fmt.Fprintf(os.Stderr, " [%s] Saved transcript: %s\n", agentName, localName) + } + + return nil +} + +// ExtractOutputFiles copies all files under a remote directory in the sandbox +// to a local output directory, preserving relative paths. +func ExtractOutputFiles(sshConfigPath, sandboxName, remoteDir, localDir string) ([]string, error) { + if err := os.MkdirAll(localDir, 0o755); err != nil { + return nil, fmt.Errorf("creating local output dir: %w", err) + } + + // List files in the sandbox output directory. + stdout, _, _, err := SSH(sshConfigPath, sandboxName, + fmt.Sprintf("find %s -type f 2>/dev/null || true", remoteDir), + 10*time.Second, + ) + if err != nil { + return nil, fmt.Errorf("listing output files: %w", err) + } + + trimmed := strings.TrimSpace(stdout) + if trimmed == "" { + return nil, nil + } + lines := strings.Split(trimmed, "\n") + + cleanBase := filepath.Clean(localDir) + string(filepath.Separator) + + var extracted []string + for _, remotePath := range lines { + remotePath = strings.TrimSpace(remotePath) + if remotePath == "" { + continue + } + // Preserve the relative path under remoteDir. + relPath := strings.TrimPrefix(remotePath, remoteDir) + relPath = strings.TrimPrefix(relPath, "/") + localPath := filepath.Join(localDir, relPath) + + // Prevent path traversal from sandbox-controlled filenames. + if !strings.HasPrefix(filepath.Clean(localPath), cleanBase) { + fmt.Fprintf(os.Stderr, " Skipping path traversal attempt: %s\n", relPath) + continue + } + + if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil { + fmt.Fprintf(os.Stderr, " Failed to create dir for %s: %v\n", relPath, err) + continue + } + + if scpErr := SCPFrom(sshConfigPath, sandboxName, remotePath, localPath); scpErr != nil { + fmt.Fprintf(os.Stderr, " Failed to copy %s: %v\n", relPath, scpErr) + continue + } + extracted = append(extracted, localPath) + } + + return extracted, nil +} diff --git a/internal/sandbox/sandbox_test.go b/internal/sandbox/sandbox_test.go new file mode 100644 index 0000000000..f69e2b3f85 --- /dev/null +++ b/internal/sandbox/sandbox_test.go @@ -0,0 +1,131 @@ +package sandbox + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEnsureAvailable_OpenshellNotInPath(t *testing.T) { + // Save and clear PATH to ensure openshell is not found. + t.Setenv("PATH", "") + + err := EnsureAvailable() + assert.Error(t, err) + assert.Contains(t, err.Error(), "openshell not found in PATH") +} + +func TestConstants(t *testing.T) { + assert.Equal(t, "/tmp/workspace", SandboxWorkspace) + assert.Equal(t, "/tmp/claude-config", SandboxClaudeConfig) +} + +func TestBuildProviderArgs_BareKeyCredentials(t *testing.T) { + t.Setenv("MY_SECRET", "super-secret-value") + + credentials := map[string]string{ + "API_KEY": "${MY_SECRET}", + } + config := map[string]string{ + "BASE_URL": "https://api.example.com", + } + + args, extraEnv, secrets := buildProviderArgs("test-provider", "anthropic", credentials, config) + + // Args must use bare-key form: --credential API_KEY (no =value). + assert.Contains(t, args, "--credential") + for _, arg := range args { + if strings.HasPrefix(arg, "API_KEY") { + assert.Equal(t, "API_KEY", arg, "credential arg must be bare key, not KEY=VALUE") + } + } + + // Secret value must NOT appear anywhere in args. + for _, arg := range args { + assert.NotContains(t, arg, "super-secret-value", + "secret value must not appear in CLI args") + } + + // Secret value must be in extraEnv for the child process. + require.Len(t, extraEnv, 1) + assert.Equal(t, "API_KEY=super-secret-value", extraEnv[0]) + + // Secrets list captures expanded values for redaction. + require.Len(t, secrets, 1) + assert.Equal(t, "super-secret-value", secrets[0]) + + // Config values are not secrets — they appear as KEY=VALUE in args. + found := false + for _, arg := range args { + if arg == "BASE_URL=https://api.example.com" { + found = true + } + } + assert.True(t, found, "config should appear as KEY=VALUE in args") +} + +func TestBuildProviderArgs_KeyRemapping(t *testing.T) { + // Credential key name differs from the host env var name. + t.Setenv("HOST_VAR_NAME", "the-secret") + + credentials := map[string]string{ + "PROVIDER_KEY": "${HOST_VAR_NAME}", + } + + args, extraEnv, _ := buildProviderArgs("p", "custom", credentials, nil) + + // Bare key uses the credential key name, not the host var name. + for _, arg := range args { + assert.NotContains(t, arg, "the-secret") + } + + // The child env maps the credential key to the expanded value. + require.Len(t, extraEnv, 1) + assert.Equal(t, "PROVIDER_KEY=the-secret", extraEnv[0]) +} + +func TestBuildProviderArgs_EmptyCredential(t *testing.T) { + t.Setenv("EMPTY_VAR", "") + + credentials := map[string]string{ + "KEY": "${EMPTY_VAR}", + } + + _, extraEnv, secrets := buildProviderArgs("p", "custom", credentials, nil) + + // Empty values should still be set in env (openshell may accept empty). + require.Len(t, extraEnv, 1) + assert.Equal(t, "KEY=", extraEnv[0]) + + // Empty string is not added to secrets (nothing to redact). + assert.Empty(t, secrets) +} + +func TestPathTraversalContainment(t *testing.T) { + // Simulate the containment check used in ExtractOutputFiles. + localDir := "/tmp/output" + cleanBase := filepath.Clean(localDir) + string(filepath.Separator) + + tests := []struct { + name string + relPath string + safe bool + }{ + {"normal file", "report.md", true}, + {"nested file", "subdir/report.md", true}, + {"traversal", "../../../etc/passwd", false}, + {"traversal with prefix", "../../home/runner/.bashrc", false}, + {"dot segments in middle", "subdir/../../etc/shadow", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + localPath := filepath.Join(localDir, tt.relPath) + contained := strings.HasPrefix(filepath.Clean(localPath), cleanBase) + assert.Equal(t, tt.safe, contained, "relPath=%q localPath=%q", tt.relPath, localPath) + }) + } +}