From 13e3634d9a0df12fcad991246fc9e6a1a4bb01ab Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 11:47:13 +0300 Subject: [PATCH 01/22] ci(e2e): add Gherkin behaviour tests with dummy runtime Introduce godog behaviour tests with pluggable SCM, CI, and install drivers; dummy runtime for deterministic agent execution; runtime selection via org and per-repo config plus CLI flags; forge issue-label and artifact APIs; and ADR 0063. Signed-off-by: Barak Korren Co-authored-by: Cursor --- .codecov.yml | 1 + Makefile | 2 +- docs/ADRs/0003-org-config-repo-convention.md | 6 +- ...ehaviour-tests-with-gherkin-and-drivers.md | 40 +++ docs/architecture.md | 6 +- docs/cli/github.md | 1 + docs/guides/README.md | 2 + docs/guides/dev/behaviour-drivers.md | 68 ++++ docs/guides/dev/behaviour-testing.md | 85 +++++ docs/guides/dev/e2e-testing.md | 2 + docs/problems/testing-agents.md | 2 + docs/runtimes.md | 11 +- e2e/admin/admin_test.go | 2 +- e2e/admin/auth.go | 2 +- e2e/admin/cleanup.go | 90 +++-- e2e/admin/lock.go | 10 +- e2e/admin/testutil.go | 82 ++++- e2e/behaviour/artifacts/find.go | 46 +++ e2e/behaviour/drivers/ci/driver.go | 18 + .../drivers/ci/githubactions/githubactions.go | 334 ++++++++++++++++++ .../ci/githubactions/githubactions_test.go | 40 +++ e2e/behaviour/drivers/env/env.go | 42 +++ e2e/behaviour/drivers/install/driver.go | 28 ++ e2e/behaviour/drivers/install/factory.go | 27 ++ .../drivers/install/perrepo_github.go | 128 +++++++ .../drivers/install/perrepo_github_test.go | 59 ++++ e2e/behaviour/drivers/scm/driver.go | 17 + e2e/behaviour/drivers/scm/github/github.go | 57 +++ e2e/behaviour/features/triage/triage.feature | 25 ++ e2e/behaviour/fixtures/triage/sufficient.json | 24 ++ e2e/behaviour/steps/artifacts.go | 84 +++++ e2e/behaviour/steps/cleanup.go | 38 ++ e2e/behaviour/steps/dummy_agent.go | 165 +++++++++ e2e/behaviour/steps/registry.go | 12 + e2e/behaviour/steps/triage.go | 115 ++++++ e2e/behaviour/suite_test.go | 125 ++++++- e2e/behaviour/world/world.go | 52 +++ go.mod | 10 + go.sum | 35 ++ internal/cli/admin.go | 39 +- internal/cli/admin_test.go | 37 +- internal/cli/github.go | 5 + internal/cli/github_test.go | 37 ++ internal/cli/run.go | 79 ++++- internal/cli/run_test.go | 121 +++++++ internal/config/config.go | 20 ++ internal/config/config_test.go | 35 ++ internal/forge/fake.go | 115 +++++- internal/forge/fake_test.go | 104 ++++++ internal/forge/forge.go | 35 ++ internal/forge/github/github.go | 200 ++++++++++- internal/forge/github/github_test.go | 221 ++++++++++++ internal/runtime/dummy.go | 270 ++++++++++++++ internal/runtime/dummy_test.go | 332 +++++++++++++++++ internal/runtime/registry.go | 41 +++ internal/runtime/registry_test.go | 50 +++ internal/runtime/runtime.go | 3 +- 57 files changed, 3577 insertions(+), 60 deletions(-) create mode 100644 docs/ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md create mode 100644 docs/guides/dev/behaviour-drivers.md create mode 100644 docs/guides/dev/behaviour-testing.md create mode 100644 e2e/behaviour/artifacts/find.go create mode 100644 e2e/behaviour/drivers/ci/driver.go create mode 100644 e2e/behaviour/drivers/ci/githubactions/githubactions.go create mode 100644 e2e/behaviour/drivers/ci/githubactions/githubactions_test.go create mode 100644 e2e/behaviour/drivers/env/env.go create mode 100644 e2e/behaviour/drivers/install/driver.go create mode 100644 e2e/behaviour/drivers/install/factory.go create mode 100644 e2e/behaviour/drivers/install/perrepo_github.go create mode 100644 e2e/behaviour/drivers/install/perrepo_github_test.go create mode 100644 e2e/behaviour/drivers/scm/driver.go create mode 100644 e2e/behaviour/drivers/scm/github/github.go create mode 100644 e2e/behaviour/features/triage/triage.feature create mode 100644 e2e/behaviour/fixtures/triage/sufficient.json create mode 100644 e2e/behaviour/steps/artifacts.go create mode 100644 e2e/behaviour/steps/cleanup.go create mode 100644 e2e/behaviour/steps/dummy_agent.go create mode 100644 e2e/behaviour/steps/registry.go create mode 100644 e2e/behaviour/steps/triage.go create mode 100644 e2e/behaviour/world/world.go create mode 100644 internal/runtime/dummy.go create mode 100644 internal/runtime/dummy_test.go create mode 100644 internal/runtime/registry.go create mode 100644 internal/runtime/registry_test.go diff --git a/.codecov.yml b/.codecov.yml index f0a5adef8f..5267a2d56c 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -18,6 +18,7 @@ coverage: ignore: - "**/*_test.go" - "**/testdata/**" + - "e2e/behaviour/**" - "docs/**" - "hack/**" - "experiments/**" diff --git a/Makefile b/Makefile index 3d6dd395a0..5ab49133c5 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ help: @echo " script-test - Run shell script tests (post-triage, post-code, post-review, pre-fetch-prior-review, reconcile-repos, validate-output-schema)" @echo " test - Run all checks: lint-all, go-test, script-test, lint-eval-cases" @echo " e2e-test - Run admin e2e tests (CI: OIDC mint; local: gh auth login or GH_TOKEN)" - @echo " behaviour-test - Run Gherkin behaviour tests (CI: OIDC mint; local: gh auth login or GH_TOKEN)" + @echo " behaviour-test - Run Gherkin behaviour tests (installs fullsend per-repo; CI: OIDC mint)" @echo " lint-eval-cases - Lint eval case definitions (annotations.yaml completeness)" @echo " functional-tests - Run functional agent tests (requires EVAL_ORG, FULLSEND_DIR, GH_TOKEN, GCP creds)" diff --git a/docs/ADRs/0003-org-config-repo-convention.md b/docs/ADRs/0003-org-config-repo-convention.md index 8a677964ac..526fd27b7e 100644 --- a/docs/ADRs/0003-org-config-repo-convention.md +++ b/docs/ADRs/0003-org-config-repo-convention.md @@ -212,7 +212,11 @@ version: 1 # Schema version for future evolution intent_repo: /features # or /intent architecture_repo: /architecture -# Agent runtime defaults +# Agent runtime defaults (see ADR 0063 for behaviour-test dummy runtime) +defaults: + runtime: claude # claude or dummy (dummy: behaviour test orgs only) + +# Legacy harness/model settings (separate from defaults.runtime backend selection) runtime: harness: claude-code # or opencode model: claude-sonnet-4-6 diff --git a/docs/ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md b/docs/ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md new file mode 100644 index 0000000000..991fc97a91 --- /dev/null +++ b/docs/ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md @@ -0,0 +1,40 @@ +--- +title: "63. Behaviour tests with Gherkin and pluggable drivers" +status: Accepted +relates_to: + - agent-infrastructure + - agent-architecture +topics: + - e2e + - testing + - behaviour-tests +--- + +# 63. Behaviour tests with Gherkin and pluggable drivers + +Date: 2026-06-07 + +## Status + +Accepted + +## Context + +Fullsend needs end-to-end tests that validate **deterministic platform behaviour** — dispatch routing, harness loading, schema validation, post-scripts, token scoping, sandbox policy, and SCM mutations — without depending on LLM output. This is distinct from admin install e2e ([ADR 0040](0040-org-pool-for-parallel-e2e-tests.md)) and from LLM/instruction testing ([testing-agents.md](../problems/testing-agents.md)). + +Runtime selection is shared with production via `defaults.runtime` in org `config.yaml` ([runtimes.md](../runtimes.md)). Harness definitions remain as in [ADR 0024](0024-harness-definitions.md). Per-repo install mode ([ADR 0033](0033-per-repo-installation-mode.md)) is the behaviour v1 default; per-org install is deferred. + +## Decision + +- Add **behaviour tests** under `e2e/behaviour/` using **godog** and portable Gherkin feature files. +- Exercise **real SCM + real CI** through **driver interfaces** (`scm.Driver`, `ci.Driver`, `install.Driver`); v1 implementations target GitHub and GitHub Actions. +- Substitute inference with a **dummy runtime** (`runtime: dummy` in per-repo config, or `defaults.runtime: dummy` for per-org) that executes scripted operations in the real OpenShell sandbox and emits `behaviour-results.json`. +- Select backends via **runner env** (`BEHAVIOUR_SCM`, `BEHAVIOUR_CI`, `BEHAVIOUR_INSTALL_MODE`); feature files stay install-mode agnostic. v1 runs **per-repo** against the halfsend org pool; the suite provisions fullsend via `fullsend github setup` rather than requiring pre-installed orgs. +- Use **compatibility tags** (`@skip:*`, `@requires:*`) to filter scenarios for future backends; tags do not select configuration. + +## Consequences + +- Behaviour tests can pass while prompt quality regresses; LLM evals remain necessary for instruction coverage. +- Behaviour orgs are provisioned at suite start with `--runtime dummy`; production orgs must not use dummy unintentionally. +- Adding GitLab or Tekton requires new drivers and runner env values, not feature file rewrites. +- Dummy runtime op vocabulary stays minimal; new ops require runtime + docs updates when scenarios need them. diff --git a/docs/architecture.md b/docs/architecture.md index 1ebc53de87..4bb29e7915 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -127,7 +127,11 @@ This is the thing that actually reasons and acts. Everything else in this docume **Decided (implementation):** -- The `fullsend run` runner delegates in-sandbox agent execution to a `runtime.Runtime` interface; the MVP registers Claude Code only. Bootstrap uses a portable `BootstrapInput` interface with optional extensions such as `ClaudeHooksBootstrap` for sandbox tool hooks. Transcript and debug artifact handling use a separate `TranscriptHandler` interface. See [runtimes.md](runtimes.md) for the per-runtime security feature matrix required when adding a new backend. +- The `fullsend run` runner delegates in-sandbox agent execution to a `runtime.Runtime` interface; production orgs default to Claude Code. Runtime selection is configured in `defaults.runtime` on the org `config.yaml` and resolved via `runtime.ResolveFromConfig()`. A **dummy** runtime executes scripted operations in the real OpenShell sandbox for behaviour tests (inference removed). Bootstrap uses a portable `BootstrapInput` interface with optional extensions such as `ClaudeHooksBootstrap` for sandbox tool hooks. Transcript and debug artifact handling use a separate `TranscriptHandler` interface. See [runtimes.md](runtimes.md) for the per-runtime security feature matrix required when adding a new backend. + +### Behaviour testing + +End-to-end **behaviour tests** under `e2e/behaviour/` validate deterministic platform code — dispatch routing, harness loading, sandbox policy, SCM mutations — with the LLM layer removed via the dummy runtime. Tests exercise real GitHub and GitHub Actions through pluggable SCM and CI drivers; Gherkin scenarios stay install-mode agnostic while runner env vars select backends. This coverage is **orthogonal** to LLM and instruction testing in [testing-agents.md](problems/testing-agents.md). See [ADR 0063](ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md). **Open questions:** diff --git a/docs/cli/github.md b/docs/cli/github.md index ad1be57196..a7d2daccd2 100644 --- a/docs/cli/github.md +++ b/docs/cli/github.md @@ -52,6 +52,7 @@ fullsend github setup \ | `--app-set` | `fullsend-ai` | App set name prefix for GitHub Apps | | `--agents` | `fullsend,triage,coder,review,retro,prioritize` | Agent roles to provision | | `--direct` | `false` | Push scaffold directly instead of creating a PR | +| `--runtime` | `claude` | Agent runtime backend (`claude` or `dummy`; `dummy` is for behaviour test orgs only) | ### Required OAuth scopes diff --git a/docs/guides/README.md b/docs/guides/README.md index 75dc93433f..1757f9046b 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -45,4 +45,6 @@ Guides for contributors developing and testing fullsend itself. - [E2E testing](dev/e2e-testing.md) — Local and CI e2e runs, including PR authorization and `ok-to-test` - [CLI internals](dev/cli-internals.md) — Command structure, installation pipeline, and sandbox runtime +- [Behaviour testing](dev/behaviour-testing.md) — Write Gherkin scenarios for end-to-end agent behaviour +- [Behaviour test drivers](dev/behaviour-drivers.md) — Implement SCM and CI drivers for behaviour tests - [Testing workflow changes](dev/testing-workflows.md) — Point a live GitHub org at a branch to test workflow, action, and agent changes before release diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md new file mode 100644 index 0000000000..8d7821aefc --- /dev/null +++ b/docs/guides/dev/behaviour-drivers.md @@ -0,0 +1,68 @@ +# Behaviour test drivers + +Behaviour tests isolate forge-specific code behind drivers so Gherkin scenarios stay portable. + +## Interfaces + +| Interface | Package | Responsibility | +|-----------|---------|----------------| +| `scm.Driver` | `e2e/behaviour/drivers/scm` | Issues, comments, labels (via GetIssue), file commits | +| `ci.Driver` | `e2e/behaviour/drivers/ci` | Workflow polling, logs, artifact download | +| `install.Driver` | `e2e/behaviour/drivers/install` | Provision and tear down fullsend in the acquired pool org | +| `install.State` | `e2e/behaviour/drivers/install` | Post-install config paths (script commits, workflow polling) | + +v1 reference implementations: + +- `e2e/behaviour/drivers/scm/github/` +- `e2e/behaviour/drivers/ci/githubactions/` +- `e2e/behaviour/drivers/install/github/perrepo/` (`BEHAVIOUR_INSTALL_MODE=per-repo`) + +## Runner configuration + +Set when starting the suite (not in feature files): + +``` +BEHAVIOUR_SCM=github # future: gitlab, forgejo +BEHAVIOUR_CI=githubactions # future: tekton, gitlabci +BEHAVIOUR_INSTALL_MODE=per-repo # v1 default and only supported value +``` + +The suite in `e2e/behaviour/suite_test.go` acquires a pool org, runs pre-install cleanup, calls `install.Driver.Install`, then constructs SCM and CI drivers. Unsupported `BEHAVIOUR_INSTALL_MODE` values fail at suite startup. + +### Install driver (v1 per-repo) + +Uses `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with inference flags from `E2E_GCP_PROJECT_ID` and `E2E_GCP_WIF_PROVIDER`. Pool orgs must already have shared GitHub Apps and mint enrollment; the driver does not run `fullsend admin install`. + +Teardown removes shim workflows, stale branches, and open fullsend PRs on `test-repo` via shared helpers in `e2e/admin/cleanup.go`. + +## Adding an SCM driver + +1. Implement `scm.Driver` in `e2e/behaviour/drivers/scm//`. +2. Register the driver in `suite_test.go` when `BEHAVIOUR_SCM=`. +3. Document the env var value here. +4. Add `@skip:` tags on scenarios that cannot run until the driver is complete. + +Use `forge.Client` for operations it already exposes; add REST helpers inside the driver package only when necessary (e.g. `GetIssue` with labels). + +## Adding a CI driver + +1. Implement `ci.Driver` — `WaitForWorkflow`, `AssertNoWorkflow`, `GetRunLogs`, `DownloadArtifacts`. +2. Map forge `WorkflowRun` types to portable polling logic; reuse patterns from `e2e/admin/admin_test.go`. +3. Register in suite init for the matching `BEHAVIOUR_CI` value. + +## Step definitions + +Steps must **not** import `internal/forge/github` directly — only drivers. This keeps scenarios vendor-agnostic. + +Steps use `world.Install` for config repo paths (`ConfigOwner`, `ConfigRepo`, `ConfigPathPrefix`) instead of hardcoding the per-org `.fullsend` config repo. + +## Testing drivers + +Prefer unit tests with `httptest` for REST helpers. Optional smoke scenarios against live backends mirror admin e2e credentials (`GITHUB_TOKEN`, halfsend org pool). + +## Future backends checklist + +- [ ] GitLab SCM driver + `@skip:gitlab` tag removal +- [ ] Tekton or GitLab CI driver +- [ ] Per-org install driver (`BEHAVIOUR_INSTALL_MODE=per-org`) +- [ ] Non-GitHub install backends diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md new file mode 100644 index 0000000000..941ddc5823 --- /dev/null +++ b/docs/guides/dev/behaviour-testing.md @@ -0,0 +1,85 @@ +# Behaviour testing + +End-to-end Gherkin tests under `e2e/behaviour/` validate **deterministic platform code** with inference removed. They are **orthogonal** to LLM and instruction testing in [testing-agents.md](../../problems/testing-agents.md) and to admin install e2e in `e2e/admin/`. + +| | Behaviour tests | LLM evals | Admin e2e | Unit tests | +|---|-----------------|-----------|-----------|------------| +| **Target** | Platform workflows, sandbox, SCM | Prompts, models | Install/uninstall | Go functions | +| **Inference** | Dummy runtime | Real LLM | Real LLM | N/A | +| **Infrastructure** | Live GitHub + GHA | Varies | Live GitHub + GHA | None | + +## When to add a behaviour test + +Add one when a **user-visible workflow** must be verified end-to-end (dispatch → workflow → post-script → SCM state) and the assertion is **binary**. Prefer unit tests for pure Go logic and admin e2e for install provisioning. + +## Layout + +``` +e2e/behaviour/ + features/ # Portable Gherkin scenarios + fixtures/ # Static content for write_fixture ops + steps/ # Step definitions + world/ # Scenario state + drivers/ # SCM, CI, env interfaces + v1 impls + suite_test.go # godog entry (build tag: behaviour) +``` + +## Writing scenarios + +Describe **user-visible behaviour** only. Do not encode SCM vendor, CI platform, or install mode in feature files. + +### Dummy agent tables + +```gherkin +Given a dummy agent that would: + | description | op | args | + | Emit triage JSON | write_fixture | output/agent-result.json, fixtures/triage/sufficient.json | +``` + +| Column | Meaning | +|--------|---------| +| `description` | Human label matched by assertion steps | +| `op` | `read_file`, `url_get`, `write_fixture` | +| `args` | Op-specific; see below | + +**`write_fixture`:** `dest_path, fixtures/...` — content lives in `e2e/behaviour/fixtures/`, embedded in the committed scenario script at `.fullsend/behaviour/current-scenario.yaml`. + +### Assertion steps + +Each assertion verifies immediately against workflow artifacts. If the triage workflow has not been waited on yet, the step waits for completion and downloads artifacts first (same as `Then the triage workflow completes successfully`). + +```gherkin +Then the agent will succeed to Emit triage JSON +And the agent will fail to Search for foo +And the agent will output issues.out with: + """ + expected content + """ +``` + +### Compatibility tags + +Use tags only for **exceptions** when a backend cannot run a scenario yet: `@skip:gitlab`, `@skip:per-org`, `@requires:per-repo`. Untagged scenarios run everywhere applicable. + +## Running locally + +```bash +# Local: gh auth login or export GH_TOKEN/GITHUB_TOKEN with access to halfsend org pool +make behaviour-test +``` + +In CI, behaviour tests mint cross-org `e2e` installation tokens via OIDC (same as admin e2e). Pool orgs need only exist with mint-enrolled apps and a `test-repo`; the suite installs fullsend per-repo via `fullsend github setup` before scenarios run. + +Runner env (defaults shown): + +``` +BEHAVIOUR_SCM=github +BEHAVIOUR_CI=githubactions +BEHAVIOUR_INSTALL_MODE=per-repo +E2E_GCP_PROJECT_ID=... # required in CI for per-repo inference secrets +E2E_GCP_WIF_PROVIDER=... # required in CI for per-repo WIF secret +``` + +Triage scenarios apply the `ready-for-triage` label (not `/fs-triage` comments) because the per-repo shim ignores `issue_comment` events from bot users and CI uses minted e2e installation tokens. + +See [behaviour-drivers.md](behaviour-drivers.md) for driver configuration and [ADR 0063](../../ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md) for the decision record. diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md index 51773ea661..d9f2d970bb 100644 --- a/docs/guides/dev/e2e-testing.md +++ b/docs/guides/dev/e2e-testing.md @@ -38,6 +38,8 @@ Optional environment variables: | `E2E_LOCK_TIMEOUT` | Max wait for a free pool org (default 10m) | | `E2E_GCP_PROJECT_ID` | GCP project for inference-related setup (if needed) | +Behaviour tests use the same pool orgs but install via `fullsend github setup` (per-repo) instead of `fullsend admin install`. See [behaviour-testing.md](behaviour-testing.md) and [behaviour-drivers.md](behaviour-drivers.md). + Tests acquire an exclusive lock on one org from the pool (`halfsend-01` … `halfsend-06`) — see [ADR 0040](../../ADRs/0040-org-pool-for-parallel-e2e-tests.md). diff --git a/docs/problems/testing-agents.md b/docs/problems/testing-agents.md index 9a055be8c8..40120bfa36 100644 --- a/docs/problems/testing-agents.md +++ b/docs/problems/testing-agents.md @@ -8,6 +8,8 @@ Testing application code is a solved problem with mature tooling: unit tests, in Today, if someone modifies a review agent's instructions, the only verification is human review of the prose change. There is no automated way to confirm the agent still behaves correctly after the modification. This is the equivalent of shipping code changes with no test suite — something we would never accept for application code. +**Behaviour tests are orthogonal.** [ADR 0063](../ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md) describes Gherkin end-to-end tests under `e2e/behaviour/` that validate deterministic platform code (workflows, harness, sandbox policy, post-scripts) with inference explicitly removed via the dummy runtime. They do not evaluate instructions, prompts, or models, and they do not replace golden-set or statistical LLM evals described below. + ## What makes agent testing hard ### Non-determinism diff --git a/docs/runtimes.md b/docs/runtimes.md index fb7f09f038..642911a8ab 100644 --- a/docs/runtimes.md +++ b/docs/runtimes.md @@ -1,8 +1,15 @@ # Agent runtimes -Fullsend's `fullsend run` command delegates in-sandbox agent execution to a pluggable **runtime**. Today only **Claude Code** is registered; the `internal/runtime` package defines the contracts new runtimes must implement. +Fullsend's `fullsend run` command delegates in-sandbox agent execution to a pluggable **runtime**. Recognized values in org `config.yaml` `defaults.runtime` are **`claude`** (production default) and **`dummy`** (behaviour tests only). Install with `fullsend admin install --runtime dummy` on dedicated test orgs. The runner resolves the backend via `runtime.ResolveFromConfig()` after loading the org config. -When adding a runtime, fill in the security matrix below and wire the implementation through `runtime.Default()`. +When adding a runtime, fill in the security matrix below and register it in `runtime.Resolve()`. + +## Registered runtimes + +| Runtime | Purpose | Inference | +|---------|---------|-----------| +| `claude` | Production agent runs via Claude Code | Required | +| `dummy` | Behaviour tests — scripted ops in real sandbox | None | ## Security feature matrix diff --git a/e2e/admin/admin_test.go b/e2e/admin/admin_test.go index b271c08e3e..4f04c207c7 100644 --- a/e2e/admin/admin_test.go +++ b/e2e/admin/admin_test.go @@ -66,7 +66,7 @@ func setupE2ETest(t *testing.T) *e2eEnv { releaseLock(context.Background(), client, org, runID, t) }) - cleanupStaleResources(context.Background(), client, token, org, t) + CleanupStaleResources(context.Background(), client, token, org, t) return &e2eEnv{ cfg: cfg, diff --git a/e2e/admin/auth.go b/e2e/admin/auth.go index 6182849457..b288115a48 100644 --- a/e2e/admin/auth.go +++ b/e2e/admin/auth.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e || behaviour package admin diff --git a/e2e/admin/cleanup.go b/e2e/admin/cleanup.go index 315002b942..5819e1ece0 100644 --- a/e2e/admin/cleanup.go +++ b/e2e/admin/cleanup.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e || behaviour package admin @@ -13,9 +13,14 @@ import ( "github.com/fullsend-ai/fullsend/internal/forge" ) -// cleanupStaleResources removes leftover resources from previous test runs. +type cleanupLogger interface { + Helper() + Logf(format string, args ...any) +} + +// CleanupStaleResources removes leftover resources from previous test runs. // This is the "teardown-first" part of the dual cleanup strategy. -func cleanupStaleResources(ctx context.Context, client forge.Client, token, org string, t *testing.T) { +func CleanupStaleResources(ctx context.Context, client forge.Client, token, org string, t *testing.T) { t.Helper() t.Log("[cleanup] Scanning for stale resources from previous runs...") @@ -95,6 +100,39 @@ func cleanupStaleResources(ctx context.Context, client forge.Client, token, org t.Log("[cleanup] Stale resource scan complete") } +// TeardownPerRepoInstall removes per-repo fullsend artifacts from a test repository. +func TeardownPerRepoInstall(ctx context.Context, client forge.Client, token, org, repo string, logf func(string, ...any)) { + if logf == nil { + logf = func(string, ...any) {} + } + log := &funcLogger{logf: logf} + log.Logf("[cleanup] Tearing down per-repo install on %s/%s", org, repo) + deleteBranch(ctx, token, org, repo, "fullsend/onboard", log) + deleteBranch(ctx, token, org, repo, "fullsend/offboard", log) + deleteShimWorkflow(ctx, token, org, repo, log) + prs, err := client.ListRepoPullRequests(ctx, org, repo) + if err != nil { + log.Logf("[cleanup] Warning: could not list PRs: %v", err) + return + } + for _, pr := range prs { + if strings.Contains(pr.Title, "fullsend") { + log.Logf("[cleanup] Closing stale PR #%d: %s", pr.Number, pr.Title) + closePR(ctx, token, org, repo, pr.Number, log) + } + } +} + +type funcLogger struct { + logf func(string, ...any) +} + +func (f *funcLogger) Helper() {} + +func (f *funcLogger) Logf(format string, args ...any) { + f.logf(format, args...) +} + // listOrgRepoNames returns repository names in an org via the GitHub REST API. func listOrgRepoNames(ctx context.Context, token, org string, t *testing.T) []string { t.Helper() @@ -145,13 +183,13 @@ func listOrgRepoNames(ctx context.Context, token, org string, t *testing.T) []st // deleteBranch deletes a branch from a repo using the GitHub API directly // (forge.Client doesn't have DeleteBranch). -func deleteBranch(ctx context.Context, token, org, repo, branch string, t *testing.T) { - t.Helper() +func deleteBranch(ctx context.Context, token, org, repo, branch string, log cleanupLogger) { + log.Helper() branchRef := "heads/" + branch url := fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/%s", org, repo, branchRef) req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) if err != nil { - t.Logf("[cleanup] Warning: could not create branch delete request: %v", err) + log.Logf("[cleanup] Warning: could not create branch delete request: %v", err) return } req.Header.Set("Authorization", "Bearer "+token) @@ -159,31 +197,31 @@ func deleteBranch(ctx context.Context, token, org, repo, branch string, t *testi resp, err := http.DefaultClient.Do(req) if err != nil { - t.Logf("[cleanup] Warning: could not delete branch %s: %v", branch, err) + log.Logf("[cleanup] Warning: could not delete branch %s: %v", branch, err) return } defer resp.Body.Close() if resp.StatusCode == http.StatusNoContent { - t.Logf("[cleanup] Deleted stale branch %s", branch) + log.Logf("[cleanup] Deleted stale branch %s", branch) } else if resp.StatusCode == http.StatusNotFound { // Branch doesn't exist, nothing to do. } else { - t.Logf("[cleanup] Warning: unexpected status deleting branch %s: %d", branch, resp.StatusCode) + log.Logf("[cleanup] Warning: unexpected status deleting branch %s: %d", branch, resp.StatusCode) } } // deleteShimWorkflow removes the fullsend shim workflow from a repo's default // branch. This cleans up after Phase 2.5 which merges the enrollment PR. -func deleteShimWorkflow(ctx context.Context, token, org, repo string, t *testing.T) { - t.Helper() +func deleteShimWorkflow(ctx context.Context, token, org, repo string, log cleanupLogger) { + log.Helper() shimPath := ".github/workflows/fullsend.yaml" // Get the file's SHA (required for deletion). getURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s", org, repo, shimPath) req, err := http.NewRequestWithContext(ctx, http.MethodGet, getURL, nil) if err != nil { - t.Logf("[cleanup] Warning: could not create request to check shim file: %v", err) + log.Logf("[cleanup] Warning: could not create request to check shim file: %v", err) return } req.Header.Set("Authorization", "Bearer "+token) @@ -191,7 +229,7 @@ func deleteShimWorkflow(ctx context.Context, token, org, repo string, t *testing resp, err := http.DefaultClient.Do(req) if err != nil { - t.Logf("[cleanup] Warning: could not check shim file: %v", err) + log.Logf("[cleanup] Warning: could not check shim file: %v", err) return } defer resp.Body.Close() @@ -200,7 +238,7 @@ func deleteShimWorkflow(ctx context.Context, token, org, repo string, t *testing return // File doesn't exist, nothing to do. } if resp.StatusCode != http.StatusOK { - t.Logf("[cleanup] Warning: unexpected status checking shim file: %d", resp.StatusCode) + log.Logf("[cleanup] Warning: unexpected status checking shim file: %d", resp.StatusCode) return } @@ -208,7 +246,7 @@ func deleteShimWorkflow(ctx context.Context, token, org, repo string, t *testing SHA string `json:"sha"` } if err := json.NewDecoder(resp.Body).Decode(&fileInfo); err != nil { - t.Logf("[cleanup] Warning: could not decode shim file info: %v", err) + log.Logf("[cleanup] Warning: could not decode shim file info: %v", err) return } @@ -222,12 +260,12 @@ func deleteShimWorkflow(ctx context.Context, token, org, repo string, t *testing } deletePayload, err := json.Marshal(deleteBody) if err != nil { - t.Logf("[cleanup] Warning: could not marshal delete payload: %v", err) + log.Logf("[cleanup] Warning: could not marshal delete payload: %v", err) return } delReq, err := http.NewRequestWithContext(ctx, http.MethodDelete, getURL, strings.NewReader(string(deletePayload))) if err != nil { - t.Logf("[cleanup] Warning: could not create delete request for shim: %v", err) + log.Logf("[cleanup] Warning: could not create delete request for shim: %v", err) return } delReq.Header.Set("Authorization", "Bearer "+token) @@ -236,26 +274,26 @@ func deleteShimWorkflow(ctx context.Context, token, org, repo string, t *testing delResp, err := http.DefaultClient.Do(delReq) if err != nil { - t.Logf("[cleanup] Warning: could not delete shim file: %v", err) + log.Logf("[cleanup] Warning: could not delete shim file: %v", err) return } defer delResp.Body.Close() if delResp.StatusCode == http.StatusOK || delResp.StatusCode == http.StatusNoContent { - t.Logf("[cleanup] Deleted stale shim workflow from %s", repo) + log.Logf("[cleanup] Deleted stale shim workflow from %s", repo) } else { - t.Logf("[cleanup] Warning: unexpected status deleting shim file: %d", delResp.StatusCode) + log.Logf("[cleanup] Warning: unexpected status deleting shim file: %d", delResp.StatusCode) } } // closePR closes a pull request using the GitHub API directly. -func closePR(ctx context.Context, token, org, repo string, number int, t *testing.T) { - t.Helper() +func closePR(ctx context.Context, token, org, repo string, number int, log cleanupLogger) { + log.Helper() url := fmt.Sprintf("https://api.github.com/repos/%s/%s/pulls/%d", org, repo, number) body := strings.NewReader(`{"state":"closed"}`) req, err := http.NewRequestWithContext(ctx, http.MethodPatch, url, body) if err != nil { - t.Logf("[cleanup] Warning: could not create PR close request: %v", err) + log.Logf("[cleanup] Warning: could not create PR close request: %v", err) return } req.Header.Set("Authorization", "Bearer "+token) @@ -264,15 +302,15 @@ func closePR(ctx context.Context, token, org, repo string, number int, t *testin resp, err := http.DefaultClient.Do(req) if err != nil { - t.Logf("[cleanup] Warning: could not close PR #%d: %v", number, err) + log.Logf("[cleanup] Warning: could not close PR #%d: %v", number, err) return } defer resp.Body.Close() if resp.StatusCode == http.StatusOK { - t.Logf("[cleanup] Closed stale PR #%d", number) + log.Logf("[cleanup] Closed stale PR #%d", number) } else { - t.Logf("[cleanup] Warning: unexpected status closing PR #%d: %d", number, resp.StatusCode) + log.Logf("[cleanup] Warning: unexpected status closing PR #%d: %d", number, resp.StatusCode) } } diff --git a/e2e/admin/lock.go b/e2e/admin/lock.go index b203537729..a7f2f00ce6 100644 --- a/e2e/admin/lock.go +++ b/e2e/admin/lock.go @@ -1,4 +1,7 @@ -//go:build e2e +//go:build e2e || behaviour + +// +// Shared org-pool helpers for admin e2e and behaviour tests (both use AcquireOrg). package admin @@ -196,6 +199,11 @@ func releaseLock(ctx context.Context, client forge.Client, org, runID string, t t.Logf("[e2e-lock] Lock released (run: %s)", truncateUUID(runID)) } +// ReleaseLock deletes the org lock repo when the run still holds it. +func ReleaseLock(ctx context.Context, client forge.Client, org, runID string, t *testing.T) { + releaseLock(ctx, client, org, runID, t) +} + // tryReclaimStaleLock checks whether the lock on org is stale (older than // staleLockTimeout) and force-acquires it if so. Returns true if the lock // was reclaimed. This runs during the first pass so stale locks from diff --git a/e2e/admin/testutil.go b/e2e/admin/testutil.go index 72b47c45a2..ef1689f0c7 100644 --- a/e2e/admin/testutil.go +++ b/e2e/admin/testutil.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e || behaviour package admin @@ -264,9 +264,44 @@ type envConfig struct { mintURL string useMint bool gcpProjectID string + wifProvider string lockTimeout time.Duration } +// EnvConfig is the exported view of envConfig for behaviour tests. +type EnvConfig struct { + MintURL string + UseMint bool + GCPProjectID string + WIFProvider string + LockTimeout time.Duration +} + +func (c envConfig) exported() EnvConfig { + return EnvConfig{ + MintURL: c.mintURL, + UseMint: c.useMint, + GCPProjectID: c.gcpProjectID, + WIFProvider: c.wifProvider, + LockTimeout: c.lockTimeout, + } +} + +func (c EnvConfig) internal() envConfig { + return envConfig{ + mintURL: c.MintURL, + useMint: c.UseMint, + gcpProjectID: c.GCPProjectID, + wifProvider: c.WIFProvider, + lockTimeout: c.LockTimeout, + } +} + +// LoadEnvConfig reads and validates required env vars for e2e and behaviour tests. +func LoadEnvConfig(t *testing.T) EnvConfig { + return loadEnvConfig(t).exported() +} + // loadEnvConfig reads and validates required env vars. Calls t.Skip if // credentials are not set (allows running `go test -tags e2e` without // credentials to check compilation). @@ -282,6 +317,7 @@ func loadEnvConfig(t *testing.T) envConfig { } gcpProjectID := os.Getenv("E2E_GCP_PROJECT_ID") + wifProvider := os.Getenv("E2E_GCP_WIF_PROVIDER") lockTimeout := defaultLockTimeout if v := os.Getenv("E2E_LOCK_TIMEOUT"); v != "" { @@ -296,6 +332,7 @@ func loadEnvConfig(t *testing.T) envConfig { mintURL: mintURL, useMint: useMint, gcpProjectID: gcpProjectID, + wifProvider: wifProvider, lockTimeout: lockTimeout, } } @@ -393,6 +430,34 @@ func addIssueLabel(ctx context.Context, token, owner, repo string, issueNum int, return fmt.Errorf("unexpected status %d adding label %q: %s", resp.StatusCode, label, body) } +// BuildCLIBinary compiles the fullsend CLI binary once per test run. +func BuildCLIBinary(t *testing.T) string { + return buildCLIBinary(t) +} + +// RunCLI executes the fullsend CLI with the given args, passing GITHUB_TOKEN. +func RunCLI(t *testing.T, binary, token string, args ...string) string { + return runCLI(t, binary, token, args...) +} + +// TryRunCLI is like RunCLI but returns an error instead of calling t.Fatalf. +func TryRunCLI(binary, token string, args ...string) (string, error) { + modRoot, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}").Output() + if err != nil { + return "", fmt.Errorf("finding module root: %w", err) + } + dir := strings.TrimSpace(string(modRoot)) + cmd := exec.Command(binary, args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GITHUB_TOKEN="+token, "CI=true") + out, runErr := cmd.CombinedOutput() + output := string(out) + if runErr != nil { + return output, fmt.Errorf("[cli] fullsend %s failed: %w\n%s", strings.Join(args, " "), runErr, output) + } + return output, nil +} + // buildCLIBinary compiles the fullsend CLI binary once per test run. func buildCLIBinary(t *testing.T) string { t.Helper() @@ -482,3 +547,18 @@ func retryOnNotFound(ctx context.Context, maxAttempts int, fn func() error) erro } return err } + +// AcquireOrg exports org pool acquisition for behaviour tests. +func AcquireOrg(ctx context.Context, cfg EnvConfig, runID string, pool []string, timeout time.Duration, logf func(string, ...any)) (string, string, error) { + return acquireOrg(ctx, cfg.internal(), runID, pool, timeout, logf) +} + +// OrgPool returns the halfsend org names used for parallel e2e runs. +func OrgPool() []string { + return orgPool +} + +// NewLiveClient creates a GitHub API client from a token. +func NewLiveClient(token string) *gh.LiveClient { + return newLiveClient(token) +} diff --git a/e2e/behaviour/artifacts/find.go b/e2e/behaviour/artifacts/find.go new file mode 100644 index 0000000000..a508b2e1d6 --- /dev/null +++ b/e2e/behaviour/artifacts/find.go @@ -0,0 +1,46 @@ +package artifacts + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// FindBehaviourResults locates behaviour-results.json in downloaded artifacts. +func FindBehaviourResults(artifactRoot string) ([]byte, error) { + return findFileByName(artifactRoot, "behaviour-results.json") +} + +// FindOutputFile searches artifact downloads for a sandbox output file by name. +func FindOutputFile(artifactRoot, fileName string) ([]byte, error) { + return findFileByName(artifactRoot, fileName) +} + +func findFileByName(artifactRoot, fileName string) ([]byte, error) { + var found []byte + err := filepath.WalkDir(artifactRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if filepath.Base(path) != fileName { + return nil + } + data, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + found = data + return filepath.SkipAll + }) + if err != nil { + return nil, err + } + if found == nil { + return nil, fmt.Errorf("%s not found under %s", fileName, artifactRoot) + } + return found, nil +} diff --git a/e2e/behaviour/drivers/ci/driver.go b/e2e/behaviour/drivers/ci/driver.go new file mode 100644 index 0000000000..7ca154fbd4 --- /dev/null +++ b/e2e/behaviour/drivers/ci/driver.go @@ -0,0 +1,18 @@ +package ci + +import ( + "context" + "time" + + "github.com/fullsend-ai/fullsend/internal/forge" +) + +// Driver abstracts CI workflow operations for behaviour tests. +type Driver interface { + WaitForWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time, event string) (*forge.WorkflowRun, error) + FindCompletedWorkflowRun(ctx context.Context, owner, repo, workflowFile string, after time.Time) (*forge.WorkflowRun, error) + AssertNoWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) error + GetRunLogs(ctx context.Context, owner, repo string, runID int) (string, error) + DownloadArtifacts(ctx context.Context, owner, repo string, runID int, destDir string) error + DownloadNamedArtifactAfter(ctx context.Context, owner, repo, artifactName string, after time.Time, destDir string) error +} diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions.go b/e2e/behaviour/drivers/ci/githubactions/githubactions.go new file mode 100644 index 0000000000..dee5ac6321 --- /dev/null +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions.go @@ -0,0 +1,334 @@ +package githubactions + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/ci" + "github.com/fullsend-ai/fullsend/internal/forge" +) + +const ( + pollInterval = 15 * time.Second + dispatchWait = 12 * time.Minute + dispatchPoll = 5 * time.Second + dispatchMaxTry = 24 + + artifactRunPoll = 5 * time.Second + artifactRunWait = 5 * time.Minute + + agentWorkflowName = "Triage Agent" + + assertNoWorkflowChecks = 3 + assertNoWorkflowDelay = 10 * time.Second +) + +// Driver implements ci.Driver against GitHub Actions. +type Driver struct { + Client forge.Client + Token string +} + +func New(client forge.Client, token string) ci.Driver { + return &Driver{Client: client, Token: token} +} + +func (d *Driver) WaitForWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time, event string) (*forge.WorkflowRun, error) { + workflowFile = filepath.Base(workflowFile) + var triageRun *forge.WorkflowRun + for attempt := 0; attempt < dispatchMaxTry; attempt++ { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(dispatchPoll): + } + runs, err := d.Client.ListWorkflowRuns(ctx, owner, repo, workflowFile) + if err != nil { + continue + } + if candidate := selectWorkflowRun(runs, after, event); candidate != nil { + triageRun = candidate + break + } + } + if triageRun == nil { + if event != "" { + return nil, fmt.Errorf("workflow %s (%s) was not dispatched", workflowFile, event) + } + return nil, fmt.Errorf("workflow %s was not dispatched", workflowFile) + } + + deadline := time.Now().Add(dispatchWait) + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(pollInterval): + } + run, err := d.Client.GetWorkflowRun(ctx, owner, repo, triageRun.ID) + if err != nil { + continue + } + if run.Status == "completed" { + if run.Conclusion == "success" { + return run, nil + } + if replacement := selectWorkflowRun(latestRuns(ctx, d, owner, repo, workflowFile), after, event); replacement != nil && replacement.ID > triageRun.ID { + triageRun = replacement + continue + } + return run, fmt.Errorf("workflow %s run %d concluded with %q", workflowFile, run.ID, run.Conclusion) + } + } + return nil, fmt.Errorf("workflow %s run %d did not complete within deadline", workflowFile, triageRun.ID) +} + +func latestRuns(ctx context.Context, d *Driver, owner, repo, workflowFile string) []forge.WorkflowRun { + runs, err := d.Client.ListWorkflowRuns(ctx, owner, repo, workflowFile) + if err != nil { + return nil + } + return runs +} + +// selectWorkflowRun returns the newest workflow run after triggerTime that is +// still active or completed successfully. Skipped/cancelled/failed runs are +// ignored so a concurrent stale dispatch does not mask the real triage run. +// When event is non-empty, only runs with a matching GitHub event are considered. +func selectWorkflowRun(runs []forge.WorkflowRun, triggerTime time.Time, event string) *forge.WorkflowRun { + var best *forge.WorkflowRun + for _, run := range runs { + if !workflowRunMatches(run, triggerTime, event) { + continue + } + if run.Status == "completed" && run.Conclusion != "success" { + continue + } + if best == nil || run.ID > best.ID { + r := run + best = &r + } + } + return best +} + +func workflowRunMatches(run forge.WorkflowRun, triggerTime time.Time, event string) bool { + runTime, parseErr := time.Parse(time.RFC3339, run.CreatedAt) + if parseErr != nil || runTime.Before(triggerTime) { + return false + } + if event != "" && run.Event != event { + return false + } + return true +} + +func (d *Driver) FindCompletedWorkflowRun(ctx context.Context, owner, repo, workflowFile string, after time.Time) (*forge.WorkflowRun, error) { + workflowFile = filepath.Base(workflowFile) + deadline := time.Now().Add(artifactRunWait) + for time.Now().Before(deadline) { + run, err := d.findCompletedWorkflowRunOnce(ctx, owner, repo, workflowFile, after) + if err != nil { + return nil, err + } + if run != nil { + return run, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(artifactRunPoll): + } + } + return nil, fmt.Errorf("no completed workflow run for %s after %s", workflowFile, after.Format(time.RFC3339)) +} + +func (d *Driver) findCompletedWorkflowRunOnce(ctx context.Context, owner, repo, workflowFile string, after time.Time) (*forge.WorkflowRun, error) { + workflowFile = filepath.Base(workflowFile) + for _, wf := range []string{workflowFile, ".github/workflows/" + workflowFile} { + runs, err := d.Client.ListWorkflowRuns(ctx, owner, repo, wf) + if err != nil { + continue + } + if run := selectCompletedSuccessRun(runs, after); run != nil { + return run, nil + } + } + runs, err := d.Client.ListRecentWorkflowRuns(ctx, owner, repo, 30) + if err != nil { + return nil, err + } + return selectCompletedSuccessRunByName(runs, after, agentWorkflowName), nil +} + +func selectCompletedSuccessRunByName(runs []forge.WorkflowRun, after time.Time, name string) *forge.WorkflowRun { + var best *forge.WorkflowRun + for _, run := range runs { + if run.Name != name { + continue + } + runTime, parseErr := time.Parse(time.RFC3339, run.CreatedAt) + if parseErr != nil || runTime.Before(after) { + continue + } + if run.Status != "completed" || run.Conclusion != "success" { + continue + } + if best == nil || run.ID > best.ID { + r := run + best = &r + } + } + return best +} + +func selectCompletedSuccessRun(runs []forge.WorkflowRun, after time.Time) *forge.WorkflowRun { + var best *forge.WorkflowRun + for _, run := range runs { + runTime, parseErr := time.Parse(time.RFC3339, run.CreatedAt) + if parseErr != nil || runTime.Before(after) { + continue + } + if run.Status != "completed" || run.Conclusion != "success" { + continue + } + if best == nil || run.ID > best.ID { + r := run + best = &r + } + } + return best +} + +func (d *Driver) AssertNoWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) error { + for attempt := 0; attempt < assertNoWorkflowChecks; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(assertNoWorkflowDelay): + } + } + runs, err := d.Client.ListWorkflowRuns(ctx, owner, repo, workflowFile) + if err != nil { + return err + } + for _, run := range runs { + runTime, parseErr := time.Parse(time.RFC3339, run.CreatedAt) + if parseErr != nil { + continue + } + if !runTime.Before(after) { + return fmt.Errorf("unexpected workflow run %d for %s", run.ID, workflowFile) + } + } + } + return nil +} + +func (d *Driver) GetRunLogs(ctx context.Context, owner, repo string, runID int) (string, error) { + return d.Client.GetWorkflowRunLogs(ctx, owner, repo, runID) +} + +func (d *Driver) DownloadArtifacts(ctx context.Context, owner, repo string, runID int, destDir string) error { + artifacts, err := d.Client.ListWorkflowRunArtifacts(ctx, owner, repo, runID) + if err != nil { + return err + } + for _, art := range artifacts { + zipData, err := d.Client.DownloadWorkflowRunArtifact(ctx, owner, repo, art.ID) + if err != nil { + return err + } + if err := extractArtifactZip(art.Name, zipData, destDir); err != nil { + return err + } + } + return nil +} + +func (d *Driver) DownloadNamedArtifactAfter(ctx context.Context, owner, repo, artifactName string, after time.Time, destDir string) error { + deadline := time.Now().Add(artifactRunWait) + for time.Now().Before(deadline) { + arts, err := d.Client.ListRepositoryArtifacts(ctx, owner, repo, 100) + if err != nil { + return err + } + if art := selectRepositoryArtifactAfter(arts, artifactName, after); art != nil { + zipData, err := d.Client.DownloadWorkflowRunArtifact(ctx, owner, repo, art.ID) + if err != nil { + return err + } + return extractArtifactZip(art.Name, zipData, destDir) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(artifactRunPoll): + } + } + return fmt.Errorf("artifact %q not found after %s", artifactName, after.Format(time.RFC3339)) +} + +func selectRepositoryArtifactAfter(arts []forge.RepositoryArtifact, name string, after time.Time) *forge.RepositoryArtifact { + var best *forge.RepositoryArtifact + for _, art := range arts { + if art.Name != name { + continue + } + artTime, parseErr := time.Parse(time.RFC3339, art.CreatedAt) + if parseErr != nil || artTime.Before(after) { + continue + } + if best == nil || art.ID > best.ID { + a := art + best = &a + } + } + return best +} + +func extractArtifactZip(name string, zipData []byte, destDir string) error { + artDir := filepath.Join(destDir, name) + if err := os.MkdirAll(artDir, 0o755); err != nil { + return err + } + + zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) + if err != nil { + rawPath := filepath.Join(destDir, name+".bin") + return os.WriteFile(rawPath, zipData, 0o644) + } + + for _, f := range zr.File { + outPath := filepath.Join(artDir, f.Name) + if !strings.HasPrefix(filepath.Clean(outPath), filepath.Clean(artDir)+string(os.PathSeparator)) { + continue + } + if f.FileInfo().IsDir() { + _ = os.MkdirAll(outPath, 0o755) + continue + } + _ = os.MkdirAll(filepath.Dir(outPath), 0o755) + rc, err := f.Open() + if err != nil { + return err + } + data, err := io.ReadAll(io.LimitReader(rc, 10<<20)) + rc.Close() + if err != nil { + return err + } + if err := os.WriteFile(outPath, data, 0o644); err != nil { + return err + } + } + return nil +} diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go new file mode 100644 index 0000000000..c98a8e8d0d --- /dev/null +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go @@ -0,0 +1,40 @@ +package githubactions + +import ( + "testing" + "time" + + "github.com/fullsend-ai/fullsend/internal/forge" +) + +func TestSelectWorkflowRun_FiltersByEvent(t *testing.T) { + t.Parallel() + after := time.Date(2026, 7, 1, 11, 33, 0, 0, time.UTC) + runs := []forge.WorkflowRun{ + {ID: 1, Event: "issues", Status: "completed", Conclusion: "success", CreatedAt: "2026-07-01T11:33:09Z"}, + {ID: 2, Event: "issue_comment", Status: "completed", Conclusion: "skipped", CreatedAt: "2026-07-01T11:33:09Z"}, + {ID: 3, Event: "issue_comment", Status: "in_progress", Conclusion: "", CreatedAt: "2026-07-01T11:33:10Z"}, + } + + got := selectWorkflowRun(runs, after, "issue_comment") + if got == nil || got.ID != 3 { + t.Fatalf("selectWorkflowRun() = %#v, want issue_comment run 3", got) + } + + got = selectWorkflowRun(runs, after, "issues") + if got == nil || got.ID != 1 { + t.Fatalf("selectWorkflowRun(issues) = %#v, want run 1", got) + } +} + +func TestSelectWorkflowRun_IgnoresFailedBeforeSuccess(t *testing.T) { + t.Parallel() + after := time.Date(2026, 7, 1, 11, 33, 0, 0, time.UTC) + runs := []forge.WorkflowRun{ + {ID: 10, Event: "issue_comment", Status: "completed", Conclusion: "skipped", CreatedAt: "2026-07-01T11:33:09Z"}, + } + + if got := selectWorkflowRun(runs, after, "issue_comment"); got != nil { + t.Fatalf("selectWorkflowRun() = %#v, want nil for skipped-only runs", got) + } +} diff --git a/e2e/behaviour/drivers/env/env.go b/e2e/behaviour/drivers/env/env.go new file mode 100644 index 0000000000..567a4a7e02 --- /dev/null +++ b/e2e/behaviour/drivers/env/env.go @@ -0,0 +1,42 @@ +package env + +import ( + "fmt" + "os" + "strings" +) + +// RunnerConfig holds behaviour test runner configuration from environment. +type RunnerConfig struct { + SCM string + CI string + InstallMode string +} + +func LoadRunnerConfig() RunnerConfig { + return RunnerConfig{ + SCM: stringsTrimOrDefault(os.Getenv("BEHAVIOUR_SCM"), "github"), + CI: stringsTrimOrDefault(os.Getenv("BEHAVIOUR_CI"), "githubactions"), + InstallMode: stringsTrimOrDefault(os.Getenv("BEHAVIOUR_INSTALL_MODE"), "per-repo"), + } +} + +func (c RunnerConfig) Validate() error { + if c.InstallMode != "per-repo" { + return fmt.Errorf("behaviour tests v1 only support BEHAVIOUR_INSTALL_MODE=per-repo, got %q", c.InstallMode) + } + if c.SCM != "github" { + return fmt.Errorf("unsupported BEHAVIOUR_SCM %q", c.SCM) + } + if c.CI != "githubactions" { + return fmt.Errorf("unsupported BEHAVIOUR_CI %q", c.CI) + } + return nil +} + +func stringsTrimOrDefault(value, fallback string) string { + if v := strings.TrimSpace(value); v != "" { + return v + } + return fallback +} diff --git a/e2e/behaviour/drivers/install/driver.go b/e2e/behaviour/drivers/install/driver.go new file mode 100644 index 0000000000..9d0ef673fa --- /dev/null +++ b/e2e/behaviour/drivers/install/driver.go @@ -0,0 +1,28 @@ +package install + +import "context" + +// Driver provisions and tears down fullsend in an acquired pool org. +type Driver interface { + Install(ctx context.Context, org string) (State, error) + Teardown(ctx context.Context, org string, state State) error +} + +// State describes where behaviour tests find fullsend configuration after install. +type State interface { + Mode() string + TestRepo() string + // ConfigOwner and ConfigRepo locate commits for behaviour scripts and config reads. + ConfigOwner() string + ConfigRepo() string + // ConfigPathPrefix is "" for per-org (.fullsend repo root) or ".fullsend" for per-repo. + ConfigPathPrefix() string + // TriageWorkflowRepo is the repository polled for triage workflow runs. + TriageWorkflowRepo() string + // TriageWorkflowFile is the workflow path passed to ListWorkflowRuns. + TriageWorkflowFile() string + // AgentWorkflowFile is the reusable workflow that runs the agent and uploads artifacts. + AgentWorkflowFile() string + // AgentArtifactName is the upload-artifact name for triage agent output. + AgentArtifactName() string +} diff --git a/e2e/behaviour/drivers/install/factory.go b/e2e/behaviour/drivers/install/factory.go new file mode 100644 index 0000000000..c56af48647 --- /dev/null +++ b/e2e/behaviour/drivers/install/factory.go @@ -0,0 +1,27 @@ +//go:build behaviour + +package install + +import ( + "fmt" + + "github.com/fullsend-ai/fullsend/e2e/admin" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/env" + "github.com/fullsend-ai/fullsend/internal/forge" +) + +// NewDriver returns the install driver for the configured BEHAVIOUR_INSTALL_MODE. +func NewDriver( + cfg env.RunnerConfig, + e2eCfg admin.EnvConfig, + client forge.Client, + token, binary string, + logf func(string, ...any), +) (Driver, error) { + switch cfg.InstallMode { + case "per-repo": + return newPerRepoDriver(e2eCfg, client, token, binary, logf), nil + default: + return nil, fmt.Errorf("unsupported BEHAVIOUR_INSTALL_MODE %q", cfg.InstallMode) + } +} diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go new file mode 100644 index 0000000000..302209a391 --- /dev/null +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -0,0 +1,128 @@ +//go:build behaviour + +package install + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/fullsend-ai/fullsend/e2e/admin" + "github.com/fullsend-ai/fullsend/internal/config" + "github.com/fullsend-ai/fullsend/internal/forge" + "github.com/fullsend-ai/fullsend/internal/layers" + "github.com/fullsend-ai/fullsend/internal/scaffold" +) + +const ( + perRepoTestRepo = "test-repo" + // Vendored per-repo: issue_comment triggers the fullsend.yaml shim, which + // workflow_calls reusable-dispatch → reusable-triage synchronously. + perRepoTriageWorkflow = "fullsend.yaml" + perRepoAgentWorkflow = "reusable-triage.yml" + perRepoAgentArtifact = "fullsend-triage" +) + +// perRepoDriver installs fullsend in per-repo mode via fullsend github setup. +type perRepoDriver struct { + e2eCfg admin.EnvConfig + client forge.Client + token string + binary string + logf func(string, ...any) +} + +type perRepoState struct { + org string + repo string +} + +func (s *perRepoState) Mode() string { return "per-repo" } +func (s *perRepoState) TestRepo() string { return s.repo } +func (s *perRepoState) ConfigOwner() string { return s.org } +func (s *perRepoState) ConfigRepo() string { return s.repo } +func (s *perRepoState) ConfigPathPrefix() string { return ".fullsend" } +func (s *perRepoState) TriageWorkflowRepo() string { return s.repo } +func (s *perRepoState) TriageWorkflowFile() string { return perRepoTriageWorkflow } +func (s *perRepoState) AgentWorkflowFile() string { return perRepoAgentWorkflow } +func (s *perRepoState) AgentArtifactName() string { return perRepoAgentArtifact } + +func newPerRepoDriver(e2eCfg admin.EnvConfig, client forge.Client, token, binary string, logf func(string, ...any)) Driver { + return &perRepoDriver{ + e2eCfg: e2eCfg, + client: client, + token: token, + binary: binary, + logf: logf, + } +} + +func (d *perRepoDriver) Install(ctx context.Context, org string) (State, error) { + repo := perRepoTestRepo + target := org + "/" + repo + + args := []string{ + "github", "setup", target, + "--vendor", "--direct", + "--skip-app-setup", + "--mint-url", d.e2eCfg.MintURL, + "--runtime", "dummy", + } + if project := strings.TrimSpace(d.e2eCfg.GCPProjectID); project != "" { + args = append(args, "--inference-project", project) + } + if wif := strings.TrimSpace(d.e2eCfg.WIFProvider); wif != "" { + args = append(args, "--inference-wif-provider", wif) + } + + d.logf("[install] running fullsend %s", strings.Join(args, " ")) + if _, err := admin.TryRunCLI(d.binary, d.token, args...); err != nil { + return nil, fmt.Errorf("github setup %s: %w", target, err) + } + + st := &perRepoState{org: org, repo: repo} + if err := validatePerRepoPostInstall(ctx, d.client, org, repo); err != nil { + return nil, err + } + return st, nil +} + +func (d *perRepoDriver) Teardown(ctx context.Context, org string, state State) error { + repo := state.TestRepo() + d.logf("[install] tearing down per-repo install on %s/%s", org, repo) + admin.TeardownPerRepoInstall(ctx, d.client, d.token, org, repo, d.logf) + return nil +} + +func validatePerRepoPostInstall(ctx context.Context, client forge.Client, org, repo string) error { + shimPath := ".github/workflows/fullsend.yaml" + if _, err := client.GetFileContent(ctx, org, repo, shimPath); err != nil { + return fmt.Errorf("post-install: missing %s on %s/%s: %w", shimPath, org, repo, err) + } + + cfgPath := filepath.Join(".fullsend", "config.yaml") + cfgData, err := client.GetFileContent(ctx, org, repo, cfgPath) + if err != nil { + return fmt.Errorf("post-install: reading %s: %w", cfgPath, err) + } + cfg, err := config.ParsePerRepoConfig(cfgData) + if err != nil { + return fmt.Errorf("post-install: parsing %s: %w", cfgPath, err) + } + if err := cfg.Validate(); err != nil { + return fmt.Errorf("post-install: invalid %s: %w", cfgPath, err) + } + if cfg.Runtime != "dummy" { + return fmt.Errorf("post-install: %s runtime is %q, want dummy", cfgPath, cfg.Runtime) + } + + markerPath := scaffold.VendoredMarkerPath() + if _, err := client.GetFileContent(ctx, org, repo, markerPath); err != nil { + return fmt.Errorf("post-install: missing vendored marker %s: %w", markerPath, err) + } + if _, err := client.GetFileContent(ctx, org, repo, layers.VendoredBinaryPathPerRepo); err != nil { + return fmt.Errorf("post-install: missing vendored binary at %s: %w", layers.VendoredBinaryPathPerRepo, err) + } + return nil +} diff --git a/e2e/behaviour/drivers/install/perrepo_github_test.go b/e2e/behaviour/drivers/install/perrepo_github_test.go new file mode 100644 index 0000000000..e50b97e15d --- /dev/null +++ b/e2e/behaviour/drivers/install/perrepo_github_test.go @@ -0,0 +1,59 @@ +//go:build behaviour + +package install + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/fullsend-ai/fullsend/internal/config" + "github.com/fullsend-ai/fullsend/internal/forge" + "github.com/fullsend-ai/fullsend/internal/scaffold" +) + +func TestValidatePerRepoPostInstall_OK(t *testing.T) { + client := forge.NewFakeClient() + org, repo := "acme", "test-repo" + perRepoCfg := config.NewPerRepoConfig(config.PerRepoDefaultRoles(), org+"/"+repo) + perRepoCfg.Runtime = "dummy" + cfg, err := perRepoCfg.Marshal() + require.NoError(t, err) + + client.FileContents = map[string][]byte{ + org + "/" + repo + "/.github/workflows/fullsend.yaml": []byte("name: fullsend"), + org + "/" + repo + "/.fullsend/config.yaml": cfg, + org + "/" + repo + "/" + scaffold.VendoredMarkerPath(): []byte("marker"), + org + "/" + repo + "/.fullsend/bin/fullsend": []byte("binary"), + } + + err = validatePerRepoPostInstall(context.Background(), client, org, repo) + require.NoError(t, err) +} + +func TestValidatePerRepoPostInstall_MissingShim(t *testing.T) { + client := forge.NewFakeClient() + err := validatePerRepoPostInstall(context.Background(), client, "acme", "test-repo") + require.Error(t, err) + assert.Contains(t, err.Error(), "fullsend.yaml") +} + +func TestValidatePerRepoPostInstall_WrongRuntime(t *testing.T) { + client := forge.NewFakeClient() + org, repo := "acme", "test-repo" + cfg, err := config.NewPerRepoConfig(nil, org+"/"+repo).Marshal() + require.NoError(t, err) + + client.FileContents = map[string][]byte{ + org + "/" + repo + "/.github/workflows/fullsend.yaml": []byte("name: fullsend"), + org + "/" + repo + "/.fullsend/config.yaml": cfg, + org + "/" + repo + "/" + scaffold.VendoredMarkerPath(): []byte("marker"), + org + "/" + repo + "/.fullsend/bin/fullsend": []byte("binary"), + } + + err = validatePerRepoPostInstall(context.Background(), client, org, repo) + require.Error(t, err) + assert.Contains(t, err.Error(), "want dummy") +} diff --git a/e2e/behaviour/drivers/scm/driver.go b/e2e/behaviour/drivers/scm/driver.go new file mode 100644 index 0000000000..cec1cd67f4 --- /dev/null +++ b/e2e/behaviour/drivers/scm/driver.go @@ -0,0 +1,17 @@ +package scm + +import ( + "context" + + "github.com/fullsend-ai/fullsend/internal/forge" +) + +// Driver abstracts SCM operations for behaviour tests. +type Driver interface { + CreateIssue(ctx context.Context, owner, repo, title, body string, labels ...string) (*forge.Issue, error) + AddIssueLabels(ctx context.Context, owner, repo string, number int, labels ...string) error + AddComment(ctx context.Context, owner, repo string, number int, body string) (*forge.IssueComment, error) + GetIssue(ctx context.Context, owner, repo string, number int) (*forge.Issue, error) + CommitFile(ctx context.Context, owner, repo, path, message string, content []byte) error + CloseIssue(ctx context.Context, owner, repo string, number int) error +} diff --git a/e2e/behaviour/drivers/scm/github/github.go b/e2e/behaviour/drivers/scm/github/github.go new file mode 100644 index 0000000000..1bd80efaca --- /dev/null +++ b/e2e/behaviour/drivers/scm/github/github.go @@ -0,0 +1,57 @@ +package github + +import ( + "context" + "fmt" + "strings" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm" + "github.com/fullsend-ai/fullsend/internal/forge" +) + +// Driver implements scm.Driver using forge.Client. +type Driver struct { + Client forge.Client +} + +func New(client forge.Client) scm.Driver { + return &Driver{Client: client} +} + +func (d *Driver) CreateIssue(ctx context.Context, owner, repo, title, body string, labels ...string) (*forge.Issue, error) { + return d.Client.CreateIssue(ctx, owner, repo, title, body, labels...) +} + +func (d *Driver) AddIssueLabels(ctx context.Context, owner, repo string, number int, labels ...string) error { + return d.Client.AddIssueLabels(ctx, owner, repo, number, labels...) +} + +func (d *Driver) AddComment(ctx context.Context, owner, repo string, number int, body string) (*forge.IssueComment, error) { + return d.Client.CreateIssueComment(ctx, owner, repo, number, body) +} + +func (d *Driver) CloseIssue(ctx context.Context, owner, repo string, number int) error { + return d.Client.CloseIssue(ctx, owner, repo, number) +} + +func (d *Driver) CommitFile(ctx context.Context, owner, repo, path, message string, content []byte) error { + _, err := d.Client.CommitFiles(ctx, owner, repo, message, []forge.TreeFile{{ + Path: path, + Content: content, + Mode: "100644", + }}) + return err +} + +func (d *Driver) GetIssue(ctx context.Context, owner, repo string, number int) (*forge.Issue, error) { + return d.Client.GetIssue(ctx, owner, repo, number) +} + +// ParseRepo splits "owner/repo" into owner and repo name. +func ParseRepo(fullName string) (owner, repo string, err error) { + parts := strings.Split(fullName, "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid repository %q: expected owner/repo", fullName) + } + return parts[0], parts[1], nil +} diff --git a/e2e/behaviour/features/triage/triage.feature b/e2e/behaviour/features/triage/triage.feature new file mode 100644 index 0000000000..c0c31463ec --- /dev/null +++ b/e2e/behaviour/features/triage/triage.feature @@ -0,0 +1,25 @@ +# Behaviour tests request triage via the ready-for-triage label (issues event). +# The per-repo shim ignores /fs-triage issue_comment events from bot users; CI +# uses minted e2e installation tokens, so label dispatch is the supported path. +Feature: Manual triage via ready-for-triage label + + Scenario: Triage applies ready-to-code on sufficient issue + Given the enrolled test repository + And a dummy agent that would: + | description | op | args | + | Emit triage JSON | write_fixture | output/agent-result.json, fixtures/triage/sufficient.json | + And an issue with title "Login fails" and body containing "steps to reproduce" + When the issue is labeled "ready-for-triage" + Then the triage workflow completes successfully + And the agent will succeed to Emit triage JSON + And the issue has label "ready-to-code" + + Scenario: Sandbox blocks disallowed outbound URL + Given the enrolled test repository + And a dummy agent that would: + | description | op | args | + | Search for foo | url_get | https://www.google.com/search?q=foo | + And an issue + When the issue is labeled "ready-for-triage" + Then the triage workflow completes successfully + And the agent will fail to Search for foo diff --git a/e2e/behaviour/fixtures/triage/sufficient.json b/e2e/behaviour/fixtures/triage/sufficient.json new file mode 100644 index 0000000000..0a6c692c60 --- /dev/null +++ b/e2e/behaviour/fixtures/triage/sufficient.json @@ -0,0 +1,24 @@ +{ + "action": "sufficient", + "reasoning": "Issue includes clear reproduction steps and environment details.", + "clarity_scores": { + "symptom": 0.9, + "cause": 0.85, + "reproduction": 0.9, + "impact": 0.8, + "overall": 0.87 + }, + "triage_summary": { + "title": "Login fails", + "severity": "high", + "category": "bug", + "problem": "Application crashes on login", + "root_cause_hypothesis": "Session handling regression", + "reproduction_steps": ["Open app", "Enter credentials", "Click login"], + "environment": "Linux", + "impact": "Users cannot authenticate", + "recommended_fix": "Fix session token validation", + "proposed_test_case": "test_login_success" + }, + "comment": "## Triage Summary\n\nThis issue is ready for implementation." +} diff --git a/e2e/behaviour/steps/artifacts.go b/e2e/behaviour/steps/artifacts.go new file mode 100644 index 0000000000..d7980fb4e2 --- /dev/null +++ b/e2e/behaviour/steps/artifacts.go @@ -0,0 +1,84 @@ +package steps + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/artifacts" + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" +) + +const issueOpenEvent = "issues" + +func triageWorkflowEvent(w *world.World) string { + if w.TriageTriggerEvent != "" { + return w.TriageTriggerEvent + } + return issueOpenEvent +} + +func ensureTriageWorkflowComplete(w *world.World) error { + if w.WorkflowRun != nil { + return nil + } + if w.ScenarioStart.IsZero() { + return fmt.Errorf("no workflow trigger time: create an issue and label it first") + } + ctx := context.Background() + run, err := w.CI.WaitForWorkflow(ctx, w.Org, w.Install.TriageWorkflowRepo(), w.Install.TriageWorkflowFile(), w.ScenarioStart, triageWorkflowEvent(w)) + if err != nil { + return err + } + w.WorkflowRun = run + return nil +} + +func ensureArtifacts(w *world.World) error { + if err := ensureTriageWorkflowComplete(w); err != nil { + return err + } + if w.ArtifactDir != "" { + return nil + } + ctx := context.Background() + dest, err := prepareArtifactDir() + if err != nil { + return err + } + if w.WorkflowRun != nil { + if err := w.CI.DownloadArtifacts(ctx, w.Org, w.Install.TriageWorkflowRepo(), w.WorkflowRun.ID, dest); err == nil { + if _, findErr := artifacts.FindBehaviourResults(dest); findErr == nil { + w.ArtifactDir = dest + return nil + } + } + _ = os.RemoveAll(dest) + dest, err = prepareArtifactDir() + if err != nil { + return err + } + } + if err := w.CI.DownloadNamedArtifactAfter(ctx, w.Org, w.Install.TriageWorkflowRepo(), w.Install.AgentArtifactName(), w.ScenarioStart, dest); err != nil { + _ = os.RemoveAll(dest) + return err + } + if _, err := artifacts.FindBehaviourResults(dest); err != nil { + _ = os.RemoveAll(dest) + return err + } + w.ArtifactDir = dest + return nil +} + +func prepareArtifactDir() (string, error) { + ciArtifactDir := strings.TrimSpace(os.Getenv("BEHAVIOUR_ARTIFACT_DIR")) + if ciArtifactDir != "" { + if err := os.MkdirAll(ciArtifactDir, 0o755); err != nil { + return "", fmt.Errorf("creating behaviour artifact dir: %w", err) + } + return ciArtifactDir, nil + } + return os.MkdirTemp("", "behaviour-artifacts-*") +} diff --git a/e2e/behaviour/steps/cleanup.go b/e2e/behaviour/steps/cleanup.go new file mode 100644 index 0000000000..a6dab36a45 --- /dev/null +++ b/e2e/behaviour/steps/cleanup.go @@ -0,0 +1,38 @@ +package steps + +import ( + "context" + "os" + "strings" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" +) + +func CleanupScenario(w *world.World) { + ctx := context.Background() + if w.IssueNumber > 0 { + if err := w.SCM.CloseIssue(ctx, w.RepoOwner, w.RepoName, w.IssueNumber); err != nil { + worldLogf(w, "behaviour cleanup: close issue #%d: %v", w.IssueNumber, err) + } + } + if w.ArtifactDir != "" { + ciArtifactDir := strings.TrimSpace(os.Getenv("BEHAVIOUR_ARTIFACT_DIR")) + if ciArtifactDir == "" || w.ArtifactDir != ciArtifactDir { + if err := os.RemoveAll(w.ArtifactDir); err != nil { + worldLogf(w, "behaviour cleanup: remove artifact dir: %v", err) + } + } + } + if len(w.DummyOps) > 0 { + empty := []byte("ops: []\n") + if err := w.SCM.CommitFile(ctx, w.Install.ConfigOwner(), w.Install.ConfigRepo(), w.BehaviourScriptPath(), "behaviour: clear dummy agent script", empty); err != nil { + worldLogf(w, "behaviour cleanup: clear dummy script: %v", err) + } + } +} + +func worldLogf(w *world.World, format string, args ...any) { + if w.Logf != nil { + w.Logf(format, args...) + } +} diff --git a/e2e/behaviour/steps/dummy_agent.go b/e2e/behaviour/steps/dummy_agent.go new file mode 100644 index 0000000000..ef71ded4b5 --- /dev/null +++ b/e2e/behaviour/steps/dummy_agent.go @@ -0,0 +1,165 @@ +package steps + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/cucumber/godog" + "gopkg.in/yaml.v3" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/artifacts" + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" + "github.com/fullsend-ai/fullsend/internal/runtime" +) + +const behaviourModuleRoot = "e2e/behaviour" + +func registerDummyAgentSteps(ctx *godog.ScenarioContext, w *world.World) { + ctx.Step(`^a dummy agent that would:$`, func(table *godog.Table) error { + return parseDummyAgentTable(w, table) + }) + ctx.Step(`^the agent will succeed to (.+)$`, func(description string) error { + return assertAgentSucceeds(w, description) + }) + ctx.Step(`^the agent will fail to (.+)$`, func(description string) error { + return assertAgentFails(w, description) + }) + ctx.Step(`^the agent will output ([^\s]+) with:$`, func(fileName, doc string) error { + return assertAgentOutput(w, fileName, doc) + }) +} + +func parseDummyAgentTable(w *world.World, table *godog.Table) error { + if len(table.Rows) < 2 { + return fmt.Errorf("dummy agent table requires a header and at least one row") + } + header := table.Rows[0] + col := map[string]int{} + for i, cell := range header.Cells { + col[strings.TrimSpace(cell.Value)] = i + } + for _, required := range []string{"description", "op", "args"} { + if _, ok := col[required]; !ok { + return fmt.Errorf("dummy agent table missing %q column", required) + } + } + + moduleRoot, err := findModuleSubdir(behaviourModuleRoot) + if err != nil { + return err + } + + var ops []runtime.BehaviourOperation + for _, row := range table.Rows[1:] { + op := runtime.BehaviourOperation{ + Description: strings.TrimSpace(row.Cells[col["description"]].Value), + Op: strings.TrimSpace(row.Cells[col["op"]].Value), + Args: strings.TrimSpace(row.Cells[col["args"]].Value), + } + if op.Op == "write_fixture" { + parts := strings.SplitN(op.Args, ",", 2) + if len(parts) != 2 { + return fmt.Errorf("write_fixture args must be dest_path, fixture_path") + } + fixtureRel := strings.TrimSpace(parts[1]) + fixturePath := filepath.Join(moduleRoot, fixtureRel) + content, err := os.ReadFile(fixturePath) + if err != nil { + return fmt.Errorf("reading fixture %s: %w", fixturePath, err) + } + op.Content = string(content) + } + ops = append(ops, op) + } + + script := runtime.BehaviourScript{Ops: ops} + data, err := yaml.Marshal(script) + if err != nil { + return fmt.Errorf("marshaling behaviour script: %w", err) + } + + message := fmt.Sprintf("behaviour: set dummy agent script (%s)", time.Now().UTC().Format(time.RFC3339)) + if err := w.SCM.CommitFile(context.Background(), w.Install.ConfigOwner(), w.Install.ConfigRepo(), w.BehaviourScriptPath(), message, data); err != nil { + return fmt.Errorf("committing behaviour script: %w", err) + } + + w.DummyOps = ops + return nil +} + +func assertAgentSucceeds(w *world.World, description string) error { + return assertAgentOutcome(w, description, true) +} + +func assertAgentFails(w *world.World, description string) error { + return assertAgentOutcome(w, description, false) +} + +func assertAgentOutcome(w *world.World, description string, expectSuccess bool) error { + if err := ensureArtifacts(w); err != nil { + return err + } + data, err := artifacts.FindBehaviourResults(w.ArtifactDir) + if err != nil { + return err + } + var results runtime.BehaviourResults + if err := json.Unmarshal(data, &results); err != nil { + return fmt.Errorf("parsing behaviour-results.json: %w", err) + } + description = strings.TrimSpace(description) + for _, res := range results.Operations { + if res.Description != description { + continue + } + if res.Success != expectSuccess { + return fmt.Errorf("operation %q: expected success=%v, got success=%v (error: %s)", description, expectSuccess, res.Success, res.Error) + } + return nil + } + return fmt.Errorf("operation %q not found in behaviour-results.json", description) +} + +func assertAgentOutput(w *world.World, fileName, doc string) error { + if err := ensureArtifacts(w); err != nil { + return err + } + fileName = strings.TrimSpace(fileName) + data, err := artifacts.FindOutputFile(w.ArtifactDir, fileName) + if err != nil { + return err + } + actual := strings.TrimSpace(string(data)) + expected := strings.TrimSpace(doc) + if actual != expected { + return fmt.Errorf("output file %q: expected %q, got %q", fileName, expected, actual) + } + return nil +} + +func findModuleSubdir(rel string) (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + candidate := filepath.Join(dir, rel) + if st, err := os.Stat(candidate); err == nil && st.IsDir() { + return candidate, nil + } + return "", fmt.Errorf("could not find %s under module root %s", rel, dir) + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return "", fmt.Errorf("could not find go.mod while searching for %s", rel) +} diff --git a/e2e/behaviour/steps/registry.go b/e2e/behaviour/steps/registry.go new file mode 100644 index 0000000000..1f2e73ffda --- /dev/null +++ b/e2e/behaviour/steps/registry.go @@ -0,0 +1,12 @@ +package steps + +import ( + "github.com/cucumber/godog" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" +) + +func Register(ctx *godog.ScenarioContext, w *world.World) { + registerDummyAgentSteps(ctx, w) + registerTriageSteps(ctx, w) +} diff --git a/e2e/behaviour/steps/triage.go b/e2e/behaviour/steps/triage.go new file mode 100644 index 0000000000..841e246d62 --- /dev/null +++ b/e2e/behaviour/steps/triage.go @@ -0,0 +1,115 @@ +package steps + +import ( + "context" + "fmt" + "time" + + "github.com/cucumber/godog" + + scmgh "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm/github" + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" +) + +func registerTriageSteps(ctx *godog.ScenarioContext, w *world.World) { + ctx.Step(`^the enrolled test repository$`, func() error { return givenEnrolledTestRepository(w) }) + ctx.Step(`^an enrolled repository "([^"]+)"$`, func(fullName string) error { + return givenEnrolledRepository(w, fullName) + }) + ctx.Step(`^an issue with title "([^"]+)" and body containing "([^"]+)"$`, func(title, bodyContains string) error { + return givenIssueWithTitleAndBody(w, title, bodyContains) + }) + ctx.Step(`^an issue$`, func() error { return givenIssue(w) }) + ctx.Step(`^the issue is labeled "([^"]+)"$`, func(label string) error { + return whenIssueLabeled(w, label) + }) + ctx.Step(`^the triage workflow completes successfully$`, func() error { + return thenTriageWorkflowCompletes(w) + }) + ctx.Step(`^the issue has label "([^"]+)"$`, func(label string) error { + return thenIssueHasLabel(w, label) + }) +} + +func givenEnrolledTestRepository(w *world.World) error { + w.RepoOwner = w.Org + w.RepoName = w.Install.TestRepo() + w.RepoFull = w.Org + "/" + w.RepoName + return nil +} + +func givenEnrolledRepository(w *world.World, fullName string) error { + owner, repo, err := scmgh.ParseRepo(fullName) + if err != nil { + return err + } + if owner != w.Org { + return fmt.Errorf("repository owner %q does not match test org %q", owner, w.Org) + } + if repo != w.Install.TestRepo() { + return fmt.Errorf("repository %q is not the enrolled test repo %q", repo, w.Install.TestRepo()) + } + w.RepoFull = fullName + w.RepoOwner = owner + w.RepoName = repo + return nil +} + +func givenIssueWithTitleAndBody(w *world.World, title, bodyContains string) error { + body := fmt.Sprintf("Behaviour test issue\n\n%s\n", bodyContains) + return createIssue(w, title, body) +} + +func givenIssue(w *world.World) error { + title := fmt.Sprintf("behaviour-issue-%d", time.Now().UnixNano()) + body := "Behaviour test issue body with steps to reproduce for triage." + return createIssue(w, title, body) +} + +func createIssue(w *world.World, title, body string) error { + if w.RepoOwner == "" || w.RepoName == "" { + w.RepoOwner = w.Org + w.RepoName = w.Install.TestRepo() + w.RepoFull = w.Org + "/" + w.RepoName + } + trigger := time.Now() + issue, err := w.SCM.CreateIssue(context.Background(), w.RepoOwner, w.RepoName, title, body) + if err != nil { + return err + } + w.IssueNumber = issue.Number + w.IssueTitle = title + // fullsend.yaml triggers on issues opened and labeled. Drain the issue-open + // run before applying ready-for-triage so the labeled dispatch is not skipped. + ctx := context.Background() + if _, err := w.CI.WaitForWorkflow(ctx, w.Org, w.Install.TriageWorkflowRepo(), w.Install.TriageWorkflowFile(), trigger, issueOpenEvent); err != nil { + return fmt.Errorf("waiting for issue-open workflow: %w", err) + } + return nil +} + +func whenIssueLabeled(w *world.World, label string) error { + if w.IssueNumber == 0 { + return fmt.Errorf("no issue created") + } + w.ScenarioStart = time.Now() + w.TriageTriggerEvent = issueOpenEvent + return w.SCM.AddIssueLabels(context.Background(), w.RepoOwner, w.RepoName, w.IssueNumber, label) +} + +func thenTriageWorkflowCompletes(w *world.World) error { + return ensureTriageWorkflowComplete(w) +} + +func thenIssueHasLabel(w *world.World, label string) error { + issue, err := w.SCM.GetIssue(context.Background(), w.RepoOwner, w.RepoName, w.IssueNumber) + if err != nil { + return err + } + for _, name := range issue.Labels { + if name == label { + return nil + } + } + return fmt.Errorf("issue #%d labels %v do not include %q", w.IssueNumber, issue.Labels, label) +} diff --git a/e2e/behaviour/suite_test.go b/e2e/behaviour/suite_test.go index e69df5c756..0a4c0e8db3 100644 --- a/e2e/behaviour/suite_test.go +++ b/e2e/behaviour/suite_test.go @@ -2,8 +2,125 @@ package behaviour_test -import "testing" +import ( + "context" + "os" + "strings" + "testing" + "time" -// TestBehaviourCIWiring is a placeholder until the full Gherkin suite lands. -// It verifies the behaviour build tag, Makefile target, and e2e workflow job compile and run. -func TestBehaviourCIWiring(t *testing.T) {} + "github.com/cucumber/godog" + "github.com/google/uuid" + + "github.com/fullsend-ai/fullsend/e2e/admin" + gaci "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/ci/githubactions" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/env" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/install" + scmgh "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm/github" + "github.com/fullsend-ai/fullsend/e2e/behaviour/steps" + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" +) + +func TestBehaviourSuite(t *testing.T) { + if testing.Short() { + t.Skip("skipping behaviour tests in short mode") + } + + cfg := env.LoadRunnerConfig() + if err := cfg.Validate(); err != nil { + t.Fatalf("invalid behaviour runner config: %v", err) + } + + e2eCfg := admin.LoadEnvConfig(t) + ctx := context.Background() + + runID := uuid.New().String() + org, token, err := admin.AcquireOrg(ctx, e2eCfg, runID, admin.OrgPool(), e2eCfg.LockTimeout, t.Logf) + if err != nil { + t.Fatalf("acquiring org: %v", err) + } + client := admin.NewLiveClient(token) + + binary := admin.BuildCLIBinary(t) + installDriver, err := install.NewDriver(cfg, e2eCfg, client, token, binary, t.Logf) + if err != nil { + t.Fatalf("creating install driver: %v", err) + } + + admin.CleanupStaleResources(ctx, client, token, org, t) + + installState, err := installDriver.Install(ctx, org) + if err != nil { + t.Fatalf("installing fullsend on %s: %v", org, err) + } + + t.Cleanup(func() { + teardownCtx := context.Background() + if teardownErr := installDriver.Teardown(teardownCtx, org, installState); teardownErr != nil { + t.Logf("install teardown: %v", teardownErr) + } + admin.ReleaseLock(teardownCtx, client, org, runID, t) + }) + + testRepo := installState.TestRepo() + w := &world.World{ + Config: cfg, + SCM: scmgh.New(client), + CI: gaci.New(client, token), + Install: installState, + Org: org, + Token: token, + Logf: t.Logf, + RepoOwner: org, + RepoName: testRepo, + RepoFull: org + "/" + testRepo, + } + + // World is shared across scenarios; godog runs serially by default. Do not + // pass --concurrency without giving each scenario its own World in Before. + suite := godog.TestSuite{ + Name: "behaviour", + ScenarioInitializer: func(sc *godog.ScenarioContext) { initializeScenario(sc, w) }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"features"}, + TestingT: t, + Tags: os.Getenv("GODOG_TAGS"), + }, + } + if st := suite.Run(); st != 0 { + t.Fatalf("behaviour suite failed with status %d", st) + } +} + +func initializeScenario(sc *godog.ScenarioContext, w *world.World) { + sc.Before(func(ctx context.Context, sc *godog.Scenario) (context.Context, error) { + for _, tag := range sc.Tags { + name := strings.TrimPrefix(tag.Name, "@") + switch { + case name == "skip:per-org" && w.Config.InstallMode == "per-org": + return ctx, godog.ErrSkip + case name == "skip:per-repo" && w.Config.InstallMode == "per-repo": + return ctx, godog.ErrSkip + case name == "requires:per-repo" && w.Config.InstallMode != "per-repo": + return ctx, godog.ErrSkip + case name == "skip:gitlab" && w.Config.SCM == "gitlab": + return ctx, godog.ErrSkip + } + } + w.ScenarioStart = time.Now() + w.DummyOps = nil + w.IssueNumber = 0 + w.IssueTitle = "" + w.TriageWorkflow = "" + w.TriageTriggerEvent = "" + w.WorkflowRun = nil + w.ArtifactDir = "" + return ctx, nil + }) + sc.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { + steps.CleanupScenario(w) + return ctx, err + }) + steps.Register(sc, w) +} diff --git a/e2e/behaviour/world/world.go b/e2e/behaviour/world/world.go new file mode 100644 index 0000000000..477d428da8 --- /dev/null +++ b/e2e/behaviour/world/world.go @@ -0,0 +1,52 @@ +package world + +import ( + "path/filepath" + "time" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/ci" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/env" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/install" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm" + "github.com/fullsend-ai/fullsend/internal/forge" + "github.com/fullsend-ai/fullsend/internal/runtime" +) + +// World holds scenario state and injected drivers. +type World struct { + Config env.RunnerConfig + SCM scm.Driver + CI ci.Driver + Install install.State + + Org string + RepoFull string + RepoOwner string + RepoName string + Token string + Logf func(string, ...any) + + ScenarioStart time.Time + + DummyOps []runtime.BehaviourOperation + ArtifactDir string + TriageTriggerEvent string // GitHub event for triage dispatch (issues for label path) + + IssueNumber int + IssueTitle string + WorkflowRun *forge.WorkflowRun + TriageWorkflow string +} + +const BehaviourScriptRepoPath = "behaviour/current-scenario.yaml" + +// BehaviourScriptPath returns the repo-relative path for the dummy agent script. +func (w *World) BehaviourScriptPath() string { + if w.Install == nil { + return BehaviourScriptRepoPath + } + if prefix := w.Install.ConfigPathPrefix(); prefix != "" { + return filepath.Join(prefix, BehaviourScriptRepoPath) + } + return BehaviourScriptRepoPath +} diff --git a/go.mod b/go.mod index 85d0b8bec1..bdf33e7ed4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.0 require ( github.com/charmbracelet/lipgloss v1.1.0 + github.com/cucumber/godog v0.14.1 github.com/google/uuid v1.6.0 github.com/knights-analytics/hugot v0.7.1 github.com/playwright-community/playwright-go v0.5700.1 @@ -17,6 +18,15 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect + github.com/cucumber/messages/go/v21 v21.0.1 // indirect + github.com/gofrs/uuid v4.4.0+incompatible // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-memdb v1.3.4 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect +) + require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 6dcbb603c3..8a6a5fcd4b 100644 --- a/go.sum +++ b/go.sum @@ -30,10 +30,19 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= +github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= +github.com/cucumber/godog v0.14.1 h1:HGZhcOyyfaKclHjJ+r/q93iaTJZLKYW6Tv3HkmUE6+M= +github.com/cucumber/godog v0.14.1/go.mod h1:FX3rzIDybWABU4kuIXLZ/qtqEe1Ac5RdXmqvACJOces= +github.com/cucumber/messages/go/v21 v21.0.1 h1:wzA0LxwjlWQYZd32VTlAVDTkW6inOFmSM+RuOwHZiMI= +github.com/cucumber/messages/go/v21 v21.0.1/go.mod h1:zheH/2HS9JLVFukdrsPWoPdmUtmYQAQPLk7w5vWsk5s= +github.com/cucumber/messages/go/v22 v22.0.0/go.mod h1:aZipXTKc0JnjCsXrJnuZpWhtay93k7Rn3Dee7iyPJjs= github.com/daulet/tokenizers v1.27.0 h1:MmFYAEDFz69s/nNQfHg59DWqHz3v94m99kEZ/JbL+s4= github.com/daulet/tokenizers v1.27.0/go.mod h1:YjFY1o1HGMyWkQgbXJDghhvke/yFDp2vGdIO2hYs4MQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= @@ -50,6 +59,10 @@ github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/gomlx/exceptions v0.0.3 h1:HKnTgEjj4jlmhr8zVFkTP9qmV1ey7ypYYosQ8GzXWuM= @@ -67,6 +80,17 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= +github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/janpfeifer/go-benchmarks v0.1.1 h1:gLLy07/JrOKSnMWeUxSnjTdhkglgmrNR2IBDnR4kRqw= @@ -77,8 +101,11 @@ github.com/knights-analytics/hugot v0.7.1 h1:QljSB3o2qCg55zSidd9Aq6v5DFhDOFLHqhx github.com/knights-analytics/hugot v0.7.1/go.mod h1:D1VNAMjOPxBLnzXvwVKSnRxzAGEyD81uDzkfg8tat4k= github.com/knights-analytics/ortgenai v0.3.0 h1:ruHiWzxGnKVowlDQ4zKA3dafLLmFxus6R1vqA0rp6Ss= github.com/knights-analytics/ortgenai v0.3.0/go.mod h1:lSbQsRP5wY5NS+4W5CUGhdxjTzERQkR7WprAFxrBSt4= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= @@ -107,15 +134,23 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d h1:X4+kt6zM/OVO6gbJdAfJR60MGPsqCzbtXNnjoGqdfAs= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/viant/afs v1.30.0 h1:dbgVVSCPwGHUgpgkWJ5gdjKBqssT7OV7Z2M81CjwZEY= diff --git a/internal/cli/admin.go b/internal/cli/admin.go index 022a1fbb3a..0026bb0200 100644 --- a/internal/cli/admin.go +++ b/internal/cli/admin.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "regexp" + "slices" "sort" "strconv" "strings" @@ -247,6 +248,7 @@ func newInstallCmd() *cobra.Command { var skipMintCheck bool var publicApps bool var appSet string + var runtimeName string var direct bool // Per-repo flags. var mintURL string @@ -347,6 +349,17 @@ Inference authentication: return err } + selectedRuntime := runtimeName + if !cmd.Flags().Changed("runtime") { + selectedRuntime = loadExistingRuntime(ctx, client, org) + } + if selectedRuntime == "" { + selectedRuntime = "claude" + } + if !slices.Contains(config.ValidRuntimes(), selectedRuntime) { + return fmt.Errorf("invalid --runtime %q: must be one of %s", selectedRuntime, strings.Join(config.ValidRuntimes(), ", ")) + } + if skipMintCheck { if err := validateSkipMintCheck(mintURL); err != nil { return err @@ -506,7 +519,7 @@ Inference authentication: printer.Blank() if dryRun { - return runDryRun(ctx, client, printer, org, repos, roles, inferenceProvider, inferenceProviderName, skipMintCheck, mintURL, allRepos, vendor, fullsendBinary, fullsendSource) + return runDryRun(ctx, client, printer, org, repos, roles, selectedRuntime, inferenceProvider, inferenceProviderName, skipMintCheck, mintURL, allRepos, vendor, fullsendBinary, fullsendSource) } if err := checkInstallScopes(ctx, client, printer); err != nil { @@ -549,7 +562,7 @@ Inference authentication: agentCreds = creds } - return runInstall(ctx, client, printer, org, repos, roles, agentCreds, inferenceProvider, inferenceProviderName, vendor, fullsendBinary, fullsendSource, mintProvider, mintProject, mintRegion, mintSourceDir, mintSkipDeploy, mintURL, skipMintCheck, direct, allRepos) + return runInstall(ctx, client, printer, org, repos, roles, selectedRuntime, agentCreds, inferenceProvider, inferenceProviderName, vendor, fullsendBinary, fullsendSource, mintProvider, mintProject, mintRegion, mintSourceDir, mintSkipDeploy, mintURL, skipMintCheck, direct, allRepos) }, } @@ -570,6 +583,7 @@ Inference authentication: cmd.Flags().BoolVar(&skipMintCheck, "skip-mint-check", false, "skip mint validation, GCP provisioning, and app setup; requires --mint-url") cmd.Flags().BoolVar(&publicApps, "public", false, "create public (unlisted) GitHub Apps installable by other orgs") cmd.Flags().StringVar(&appSet, "app-set", appsetup.DefaultAppSet, "app set name prefix for GitHub Apps (e.g., myorg creates myorg-fullsend, myorg-coder)") + cmd.Flags().StringVar(&runtimeName, "runtime", "claude", "agent runtime for fullsend run (claude or dummy; dummy is for behaviour test orgs only)") // Shared flags. cmd.Flags().StringVar(&mintURL, "mint-url", DefaultMintURL, "token mint URL for OIDC token exchange (default: hosted public mint)") cmd.Flags().BoolVar(&direct, "direct", false, "push scaffold files directly to the default branch instead of creating a PR") @@ -1165,7 +1179,7 @@ func newAnalyzeCmd() *cobra.Command { // runDryRun builds a layer stack with empty credentials and analyzes. // If discoveredRepos is non-nil, it will be used instead of calling ListOrgRepos. -func runDryRun(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, inferenceProvider inference.Provider, inferenceProviderName string, skipMintCheck bool, mintURL string, discoveredRepos []forge.Repository, vendor bool, fullsendBinary, fullsendSource string) error { +func runDryRun(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, runtimeName string, inferenceProvider inference.Provider, inferenceProviderName string, skipMintCheck bool, mintURL string, discoveredRepos []forge.Repository, vendor bool, fullsendBinary, fullsendSource string) error { printer.Header("Dry run - analyzing what install would do") printer.Blank() @@ -1203,6 +1217,7 @@ func runDryRun(ctx context.Context, client forge.Client, printer *ui.Printer, or } cfg := config.NewOrgConfig(repoNames, enabledRepos, roles, inferenceProviderName, org) + cfg.Defaults.Runtime = runtimeName cfg.Dispatch.Mode = "oidc-mint" user, err := client.GetAuthenticatedUser(ctx) @@ -1466,7 +1481,7 @@ func validateEnabledRepos(enabledRepos, discoveredNames []string) error { // runInstall performs the full installation. // If discoveredRepos is non-nil, it will be used instead of calling ListOrgRepos. -func runInstall(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, agentCreds []layers.AgentCredentials, inferenceProvider inference.Provider, inferenceProviderName string, vendor bool, fullsendBinary, fullsendSource, mintProvider, mintProject, mintRegion, mintSourceDir string, mintSkipDeploy bool, mintURL string, skipMintCheck, direct bool, discoveredRepos []forge.Repository) error { +func runInstall(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, runtimeName string, agentCreds []layers.AgentCredentials, inferenceProvider inference.Provider, inferenceProviderName string, vendor bool, fullsendBinary, fullsendSource, mintProvider, mintProject, mintRegion, mintSourceDir string, mintSkipDeploy bool, mintURL string, skipMintCheck, direct bool, discoveredRepos []forge.Repository) error { var allRepos []forge.Repository var err error @@ -1505,6 +1520,7 @@ func runInstall(ctx context.Context, client forge.Client, printer *ui.Printer, o enrolledRepoIDs := collectEnrolledRepoIDs(allRepos, enabledRepos) cfg := config.NewOrgConfig(repoNames, enabledRepos, roles, inferenceProviderName, org) + cfg.Defaults.Runtime = runtimeName cfg.Dispatch.Mode = "oidc-mint" user, err := client.GetAuthenticatedUser(ctx) @@ -1989,6 +2005,21 @@ func printAnalysis(ctx context.Context, stack *layers.Stack, printer *ui.Printer return nil } +// loadExistingRuntime reads defaults.runtime from an existing config.yaml in +// .fullsend, if available. This prevents re-installs without --runtime from +// silently resetting the runtime selection. +func loadExistingRuntime(ctx context.Context, client forge.Client, org string) string { + data, err := client.GetFileContent(ctx, org, forge.ConfigRepoName, "config.yaml") + if err != nil { + return "" + } + cfg, err := config.ParseOrgConfig(data) + if err != nil { + return "" + } + return cfg.Defaults.Runtime +} + // loadExistingInferenceProvider reads the inference provider name from // an existing config.yaml in .fullsend, if available. This prevents // re-installs without --inference-project from silently erasing the inference section. diff --git a/internal/cli/admin_test.go b/internal/cli/admin_test.go index 5bf5f478ff..47e442306b 100644 --- a/internal/cli/admin_test.go +++ b/internal/cli/admin_test.go @@ -1387,6 +1387,15 @@ func TestInstallCmd_PerRepoRejectsInvalidRole(t *testing.T) { assert.Contains(t, err.Error(), "invalid role name") } +func TestInstallCmd_RejectsInvalidRuntime(t *testing.T) { + t.Setenv("GH_TOKEN", "test-token") + cmd := newRootCmd() + cmd.SetArgs([]string{"admin", "install", "acme", "--runtime", "bogus", "--dry-run"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid --runtime") +} + func TestInstallCmd_PerRepoRejectsOwnerWithDots(t *testing.T) { cmd := newRootCmd() cmd.SetArgs([]string{"admin", "install", "my.org/widget", @@ -1853,6 +1862,7 @@ func TestRunDryRun_WithDiscoveredRepos(t *testing.T) { context.Background(), client, printer, "testorg", []string{"myrepo"}, config.DefaultAgentRoles(), + "claude", nil, "", true, @@ -1866,6 +1876,23 @@ func TestRunDryRun_WithDiscoveredRepos(t *testing.T) { assert.Contains(t, buf.String(), "Layer: vendor") } +func TestLoadExistingRuntime(t *testing.T) { + t.Parallel() + + ctx := context.Background() + client := forge.NewFakeClient() + assert.Equal(t, "", loadExistingRuntime(ctx, client, "acme")) + + cfg := config.NewOrgConfig([]string{"widget"}, []string{"widget"}, config.DefaultAgentRoles(), "", "acme") + cfg.Defaults.Runtime = "dummy" + data, err := cfg.Marshal() + require.NoError(t, err) + client.FileContents = map[string][]byte{ + "acme/" + forge.ConfigRepoName + "/config.yaml": data, + } + assert.Equal(t, "dummy", loadExistingRuntime(ctx, client, "acme")) +} + func TestRunAnalyze_WithFakeClient(t *testing.T) { client := forge.NewFakeClient() client.AuthenticatedUser = "testuser" @@ -1889,7 +1916,7 @@ func TestRunInstall_RequiresAgentCredsWhenMintEnabled(t *testing.T) { err := runInstall( context.Background(), client, ui.New(&bytes.Buffer{}), "testorg", - []string{}, config.DefaultAgentRoles(), nil, + []string{}, config.DefaultAgentRoles(), "claude", nil, nil, "", false, "", "", "gcf", "test-project", "us-central1", "", true, @@ -1915,7 +1942,7 @@ func TestRunInstall_WithSkipMintCheck(t *testing.T) { err := runInstall( context.Background(), client, ui.New(&bytes.Buffer{}), "testorg", - nil, config.DefaultAgentRoles(), agentCreds, + nil, config.DefaultAgentRoles(), "claude", agentCreds, nil, "", false, "", "", "gcf", "test-project", "us-central1", "", true, @@ -1941,7 +1968,7 @@ func TestRunInstall_DiscoversRepos(t *testing.T) { var buf bytes.Buffer err := runInstall( context.Background(), client, ui.New(&buf), "testorg", - nil, config.DefaultAgentRoles(), agentCreds, + nil, config.DefaultAgentRoles(), "claude", agentCreds, nil, "", false, "", "", "gcf", "test-project", "us-central1", "", true, @@ -1962,7 +1989,7 @@ func TestRunInstall_InvalidEnabledRepo(t *testing.T) { err := runInstall( context.Background(), client, ui.New(&bytes.Buffer{}), "testorg", - []string{"missing-repo"}, config.DefaultAgentRoles(), nil, + []string{"missing-repo"}, config.DefaultAgentRoles(), "claude", nil, nil, "", false, "", "", "gcf", "test-project", "us-central1", "", true, @@ -1989,7 +2016,7 @@ func TestRunInstall_WithVendorAndSkipMint(t *testing.T) { var buf bytes.Buffer err := runInstall( context.Background(), client, ui.New(&buf), "testorg", - nil, config.DefaultAgentRoles(), agentCreds, + nil, config.DefaultAgentRoles(), "claude", agentCreds, nil, "", true, "", "", "gcf", "test-project", "us-central1", "", true, diff --git a/internal/cli/github.go b/internal/cli/github.go index 268fc52fd7..be93d38936 100644 --- a/internal/cli/github.go +++ b/internal/cli/github.go @@ -65,6 +65,7 @@ type githubSetupConfig struct { fullsendSource string dryRun bool direct bool + runtime string } func newGitHubSetupCmd() *cobra.Command { @@ -142,6 +143,7 @@ values (mint URL, WIF provider, project ID) are provided as flags.`, cmd.Flags().BoolVar(&cfg.enrollNone, "enroll-none", false, "skip repository enrollment without prompting") cmd.Flags().BoolVar(&cfg.dryRun, "dry-run", false, "print actions without making changes") cmd.Flags().BoolVar(&cfg.direct, "direct", false, "push scaffold files directly to the default branch instead of creating a PR") + cmd.Flags().StringVar(&cfg.runtime, "runtime", "", "agent runtime for per-repo config (e.g. claude, dummy)") addVendorFlags(cmd, &cfg.vendor, &cfg.fullsendBinary, &cfg.fullsendSource) return cmd @@ -212,6 +214,9 @@ func runGitHubSetupPerRepo(ctx context.Context, client forge.Client, printer *ui } perRepoCfg := config.NewPerRepoConfig(roles, cfg.target) + if cfg.runtime != "" { + perRepoCfg.Runtime = cfg.runtime + } if err := perRepoCfg.Validate(); err != nil { return fmt.Errorf("invalid config: %w", err) } diff --git a/internal/cli/github_test.go b/internal/cli/github_test.go index ecc181a893..299db82532 100644 --- a/internal/cli/github_test.go +++ b/internal/cli/github_test.go @@ -945,3 +945,40 @@ func TestRunGitHubSetupPerRepo_SecretCheckError(t *testing.T) { assert.Contains(t, err.Error(), "API rate limit exceeded") assert.Contains(t, err.Error(), "checking existing secret") } + +func TestRunGitHubSetupPerRepo_RuntimeInConfig(t *testing.T) { + t.Setenv("GH_TOKEN", "test-token") + client := forge.NewFakeClient() + client.AuthenticatedUser = "acme" + client.Repos = []forge.Repository{{FullName: "acme/widget", DefaultBranch: "main"}} + client.TokenScopes = []string{"repo", "workflow"} + client.Secrets = map[string]bool{ + "acme/widget/FULLSEND_GCP_PROJECT_ID": true, + "acme/widget/FULLSEND_GCP_WIF_PROVIDER": true, + } + printer := ui.New(&discardWriter{}) + + err := runGitHubSetupPerRepo(context.Background(), client, printer, githubSetupConfig{ + target: "acme/widget", + mintURL: "https://mint-test-abc123.run.app", + inferenceRegion: "global", + agents: strings.Join(config.PerRepoDefaultRoles(), ","), + runtime: "dummy", + }) + require.NoError(t, err) + require.NotEmpty(t, client.CommittedFilesToBranch) + + var cfgContent []byte + for _, batch := range client.CommittedFilesToBranch { + for _, f := range batch.Files { + if f.Path == ".fullsend/config.yaml" { + cfgContent = f.Content + break + } + } + } + require.NotEmpty(t, cfgContent, "expected .fullsend/config.yaml in committed files") + cfg, err := config.ParsePerRepoConfig(cfgContent) + require.NoError(t, err) + assert.Equal(t, "dummy", cfg.Runtime) +} diff --git a/internal/cli/run.go b/internal/cli/run.go index ca4f5a591c..f6dccb5fb7 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -17,6 +17,8 @@ import ( "sync" "time" + "gopkg.in/yaml.v3" + "github.com/spf13/cobra" "github.com/fullsend-ai/fullsend/internal/binary" @@ -104,6 +106,67 @@ func writeMetricsJSON(dir string, m aggregateMetrics) error { return os.WriteFile(filepath.Join(dir, metricsFile), append(data, '\n'), 0o644) } +func resolveBackendFromConfigData(orgConfigData []byte) (agentruntime.Backend, error) { + if isOrgConfigData(orgConfigData) { + orgCfg, orgErr := config.ParseOrgConfig(orgConfigData) + if orgErr != nil { + return agentruntime.Backend{}, fmt.Errorf("parsing config for runtime selection: %w", orgErr) + } + backend, resolveErr := agentruntime.ResolveFromConfig(orgCfg) + if resolveErr != nil { + return agentruntime.Backend{}, fmt.Errorf("resolving runtime: %w", resolveErr) + } + return backend, nil + } + perRepoCfg, perRepoErr := config.ParsePerRepoConfig(orgConfigData) + if perRepoErr != nil { + return agentruntime.Backend{}, fmt.Errorf("parsing config for runtime selection: %w", perRepoErr) + } + backend, resolveErr := agentruntime.ResolveFromPerRepoConfig(perRepoCfg) + if resolveErr != nil { + return agentruntime.Backend{}, fmt.Errorf("resolving runtime: %w", resolveErr) + } + return backend, nil +} + +func isOrgConfigData(data []byte) bool { + text := string(data) + if strings.Contains(text, "fullsend per-repo configuration") { + return false + } + if strings.Contains(text, "fullsend organization configuration") { + return true + } + var probe struct { + Dispatch *struct { + Platform string `yaml:"platform"` + } `yaml:"dispatch"` + Defaults *struct { + Roles []string `yaml:"roles"` + } `yaml:"defaults"` + Repos map[string]any `yaml:"repos"` + } + if err := yaml.Unmarshal(data, &probe); err != nil { + return false + } + return probe.Dispatch != nil || probe.Defaults != nil || len(probe.Repos) > 0 +} + +func backendFromConfigFile(path string) (agentruntime.Backend, error) { + data, readErr := os.ReadFile(path) + if readErr == nil { + backend, resolveErr := resolveBackendFromConfigData(data) + if resolveErr != nil { + return agentruntime.Backend{}, resolveErr + } + return backend, nil + } + if os.IsNotExist(readErr) { + return agentruntime.Default(), nil + } + return agentruntime.Backend{}, fmt.Errorf("reading org config for runtime selection: %w", readErr) +} + func newRunCmd() *cobra.Command { var fullsendDir string var outputBase string @@ -743,7 +806,20 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep } // 7. Bootstrap sandbox. - backend := agentruntime.Default() + var backend agentruntime.Backend + orgConfigPath = filepath.Join(absFullsendDir, "config.yaml") + backend, backendErr := backendFromConfigFile(orgConfigPath) + if backendErr != nil { + switch { + case strings.Contains(backendErr.Error(), "parsing config for runtime selection"): + printer.StepFail("Failed to parse org config") + case strings.Contains(backendErr.Error(), "resolving runtime"): + printer.StepFail("Failed to resolve runtime") + default: + printer.StepFail("Failed to load org config") + } + return backendErr + } rt := backend.Runtime tx := backend.Transcripts bootstrapStart := time.Now() @@ -992,6 +1068,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep AgentBaseName: agentBaseName, Model: h.Model, RepoDir: remoteRepositoryDir, + FullsendDir: absFullsendDir, PluginDirs: pluginDirs, Debug: debug, Timeout: timeout, diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index f3d9cf2292..e38fb5d3a3 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -2970,3 +2970,124 @@ func TestRunAgent_StatusNotifierSetup(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "openshell") } + +func TestResolveBackendFromConfigData_OrgConfig(t *testing.T) { + t.Parallel() + + cfg := config.NewOrgConfig([]string{"widget"}, []string{"widget"}, config.DefaultAgentRoles(), "", "acme") + cfg.Defaults.Runtime = "dummy" + data, err := cfg.Marshal() + require.NoError(t, err) + + backend, err := resolveBackendFromConfigData(data) + require.NoError(t, err) + assert.Equal(t, "dummy", backend.Runtime.Name()) +} + +func TestResolveBackendFromConfigData_PerRepoConfig(t *testing.T) { + t.Parallel() + + cfg := config.NewPerRepoConfig(config.PerRepoDefaultRoles(), "acme/test-repo") + cfg.Runtime = "dummy" + data, err := cfg.Marshal() + require.NoError(t, err) + + backend, err := resolveBackendFromConfigData(data) + require.NoError(t, err) + assert.Equal(t, "dummy", backend.Runtime.Name()) +} + +func TestResolveBackendFromConfigData_Invalid(t *testing.T) { + t.Parallel() + + _, err := resolveBackendFromConfigData([]byte("not: [valid: yaml")) + require.Error(t, err) + assert.Contains(t, err.Error(), "parsing config for runtime selection") +} + +func TestResolveBackendFromConfigData_UnknownRuntime(t *testing.T) { + t.Parallel() + + cfg := config.NewOrgConfig([]string{"widget"}, []string{"widget"}, config.DefaultAgentRoles(), "", "acme") + cfg.Defaults.Runtime = "nonexistent" + data, err := cfg.Marshal() + require.NoError(t, err) + + _, err = resolveBackendFromConfigData(data) + require.Error(t, err) + assert.Contains(t, err.Error(), "resolving runtime") +} + +func TestIsOrgConfigData(t *testing.T) { + t.Parallel() + + perRepo := config.NewPerRepoConfig(config.PerRepoDefaultRoles(), "acme/test-repo") + perRepoData, err := perRepo.Marshal() + require.NoError(t, err) + assert.False(t, isOrgConfigData(perRepoData)) + + org := config.NewOrgConfig([]string{"widget"}, []string{"widget"}, config.DefaultAgentRoles(), "", "acme") + orgData, err := org.Marshal() + require.NoError(t, err) + assert.True(t, isOrgConfigData(orgData)) +} + +func TestBackendFromConfigFile_MissingUsesDefault(t *testing.T) { + t.Parallel() + + backend, err := backendFromConfigFile(filepath.Join(t.TempDir(), "missing.yaml")) + require.NoError(t, err) + assert.Equal(t, "claude", backend.Runtime.Name()) +} + +func TestBackendFromConfigFile_PerRepoConfig(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + cfg := config.NewPerRepoConfig(config.PerRepoDefaultRoles(), "acme/test-repo") + cfg.Runtime = "dummy" + data, err := cfg.Marshal() + require.NoError(t, err) + path := filepath.Join(dir, "config.yaml") + require.NoError(t, os.WriteFile(path, data, 0o644)) + + backend, err := backendFromConfigFile(path) + require.NoError(t, err) + assert.Equal(t, "dummy", backend.Runtime.Name()) +} + +func TestBackendFromConfigFile_ReadError(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "windows" { + t.Skip("directory-as-file read error differs on Windows") + } + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.Mkdir(path, 0o755)) + + _, err := backendFromConfigFile(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "reading org config for runtime selection") +} + +func TestBackendFromConfigFile_ResolveError(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + cfg := config.NewOrgConfig([]string{"widget"}, []string{"widget"}, config.DefaultAgentRoles(), "", "acme") + cfg.Defaults.Runtime = "nonexistent" + data, err := cfg.Marshal() + require.NoError(t, err) + path := filepath.Join(dir, "config.yaml") + require.NoError(t, os.WriteFile(path, data, 0o644)) + + _, err = backendFromConfigFile(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "resolving runtime") +} + +func TestIsOrgConfigData_InvalidYAML(t *testing.T) { + t.Parallel() + + assert.False(t, isOrgConfigData([]byte("not: [valid"))) +} diff --git a/internal/config/config.go b/internal/config/config.go index 62723da4a7..5c4b42d1db 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -108,6 +108,7 @@ type CommentNotificationConfig struct { // RepoDefaults holds default settings applied to all repos. type RepoDefaults struct { Roles []string `yaml:"roles"` + Runtime string `yaml:"runtime,omitempty"` MaxImplementationRetries int `yaml:"max_implementation_retries"` AutoMerge bool `yaml:"auto_merge"` StatusNotifications *StatusNotificationConfig `yaml:"status_notifications,omitempty"` @@ -156,6 +157,11 @@ func ValidProviders() []string { return []string{"vertex"} } +// ValidRuntimes returns the set of recognized agent runtimes. +func ValidRuntimes() []string { + return []string{"claude", "dummy"} +} + // DefaultAgentRoles returns the standard set of agent roles installed // when no custom roles are specified. The fix stage reuses the coder // app (role: coder) so it does not need a separate app or PEM. @@ -220,6 +226,7 @@ func NewOrgConfig(allRepos, enabledRepos, roles []string, inferenceProvider, org }, Defaults: RepoDefaults{ Roles: roles, + Runtime: "claude", MaxImplementationRetries: 2, AutoMerge: false, }, @@ -295,6 +302,12 @@ func (c *OrgConfig) Validate() error { return fmt.Errorf("invalid inference provider %q: must be one of %s", c.Inference.Provider, strings.Join(validProviders, ", ")) } } + if rt := c.Defaults.Runtime; rt != "" { + validRuntimes := ValidRuntimes() + if !slices.Contains(validRuntimes, rt) { + return fmt.Errorf("invalid runtime %q: must be one of %s", rt, strings.Join(validRuntimes, ", ")) + } + } if err := validateStatusNotifications(c.Defaults.StatusNotifications); err != nil { return err } @@ -407,6 +420,7 @@ func (c *OrgConfig) DefaultRoles() []string { type PerRepoConfig struct { Version string `yaml:"version"` KillSwitch bool `yaml:"kill_switch,omitempty"` + Runtime string `yaml:"runtime,omitempty"` Roles []string `yaml:"roles,omitempty"` Agents []AgentEntry `yaml:"agents,omitempty"` AllowedRemoteResources []string `yaml:"allowed_remote_resources,omitempty"` @@ -480,6 +494,12 @@ func (c *PerRepoConfig) Validate() error { if err := validateCreateIssues(c.CreateIssues); err != nil { return err } + if rt := c.Runtime; rt != "" { + validRuntimes := ValidRuntimes() + if !slices.Contains(validRuntimes, rt) { + return fmt.Errorf("invalid runtime %q: must be one of %s", rt, strings.Join(validRuntimes, ", ")) + } + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 57800f9f9b..9f7182fd6d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -381,6 +381,27 @@ func TestValidProviders(t *testing.T) { assert.Equal(t, []string{"vertex"}, providers) } +func TestValidRuntimes(t *testing.T) { + runtimes := ValidRuntimes() + assert.Contains(t, runtimes, "claude") + assert.Contains(t, runtimes, "dummy") +} + +func TestOrgConfigValidateRuntime(t *testing.T) { + cfg := &OrgConfig{ + Version: "1", + Dispatch: DispatchConfig{Platform: "github-actions"}, + Defaults: RepoDefaults{ + Roles: []string{"triage"}, + Runtime: "dummy", + }, + } + require.NoError(t, cfg.Validate()) + + cfg.Defaults.Runtime = "invalid" + require.Error(t, cfg.Validate()) +} + func TestParseOrgConfig_KillSwitch(t *testing.T) { yamlData := ` version: "1" @@ -618,6 +639,20 @@ func TestPerRepoConfigValidate_EmptyRoles(t *testing.T) { assert.NoError(t, cfg.Validate()) } +func TestPerRepoConfigValidate_Runtime(t *testing.T) { + cfg := &PerRepoConfig{ + Version: "1", + Roles: []string{"triage"}, + Runtime: "dummy", + } + assert.NoError(t, cfg.Validate()) + + cfg.Runtime = "invalid" + err := cfg.Validate() + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid runtime") +} + func TestParsePerRepoConfig(t *testing.T) { yamlData := ` version: "1" diff --git a/internal/forge/fake.go b/internal/forge/fake.go index 77dc1ad6be..a0b171c867 100644 --- a/internal/forge/fake.go +++ b/internal/forge/fake.go @@ -111,12 +111,16 @@ type FakeClient struct { // Pre-populated data Repos []Repository - FileContents map[string][]byte // key: "owner/repo/path" - WorkflowRuns map[string]*WorkflowRun // key: "owner/repo/workflow" - Workflows map[string]*Workflow // key: "owner/repo/workflow" - AuthenticatedUser string // login returned by GetAuthenticatedUser - AuthenticatedUserIdentity *UserIdentity // identity returned by GetAuthenticatedUserIdentity - OrgPlan string // plan name returned by GetOrgPlan (default: "free") + FileContents map[string][]byte // key: "owner/repo/path" + WorkflowRuns map[string]*WorkflowRun // key: "owner/repo/workflow" + RecentWorkflowRuns map[string][]WorkflowRun // key: "owner/repo" + WorkflowRunArtifacts map[int][]WorkflowArtifact + WorkflowArtifactContents map[int][]byte + RepositoryArtifacts map[string][]RepositoryArtifact // key: owner/repo + Workflows map[string]*Workflow // key: "owner/repo/workflow" + AuthenticatedUser string // login returned by GetAuthenticatedUser + AuthenticatedUserIdentity *UserIdentity // identity returned by GetAuthenticatedUserIdentity + OrgPlan string // plan name returned by GetOrgPlan (default: "free") Installations []Installation Secrets map[string]bool // key: "owner/repo/name" PullRequests map[string][]ChangeProposal // key: "owner/repo" @@ -915,6 +919,49 @@ func (f *FakeClient) CreateIssue(_ context.Context, owner, repo, title, body str return &issue, nil } +func (f *FakeClient) AddIssueLabels(_ context.Context, owner, repo string, number int, labels ...string) error { + f.mu.Lock() + defer f.mu.Unlock() + if e := f.err("AddIssueLabels"); e != nil { + return e + } + key := owner + "/" + repo + issues := f.OpenIssues[key] + for i := range issues { + if issues[i].Number != number { + continue + } + seen := make(map[string]struct{}, len(issues[i].Labels)) + for _, l := range issues[i].Labels { + seen[l] = struct{}{} + } + for _, l := range labels { + if _, ok := seen[l]; !ok { + issues[i].Labels = append(issues[i].Labels, l) + seen[l] = struct{}{} + } + } + f.OpenIssues[key][i] = issues[i] + return nil + } + return fmt.Errorf("issue #%d not found", number) +} + +func (f *FakeClient) GetIssue(_ context.Context, owner, repo string, number int) (*Issue, error) { + f.mu.Lock() + defer f.mu.Unlock() + if e := f.err("GetIssue"); e != nil { + return nil, e + } + for _, issue := range f.OpenIssues[owner+"/"+repo] { + if issue.Number == number { + copy := issue + return ©, nil + } + } + return nil, ErrNotFound +} + func (f *FakeClient) CloseIssue(_ context.Context, _, _ string, _ int) error { f.mu.Lock() defer f.mu.Unlock() @@ -1183,6 +1230,62 @@ func (f *FakeClient) ListWorkflowRuns(_ context.Context, owner, repo, workflowFi return nil, nil } +func (f *FakeClient) ListWorkflowRunArtifacts(_ context.Context, _, _ string, runID int) ([]WorkflowArtifact, error) { + f.mu.Lock() + defer f.mu.Unlock() + if e := f.err("ListWorkflowRunArtifacts"); e != nil { + return nil, e + } + if f.WorkflowRunArtifacts == nil { + return nil, nil + } + return append([]WorkflowArtifact(nil), f.WorkflowRunArtifacts[runID]...), nil +} + +func (f *FakeClient) ListRecentWorkflowRuns(_ context.Context, owner, repo string, perPage int) ([]WorkflowRun, error) { + f.mu.Lock() + defer f.mu.Unlock() + if e := f.err("ListRecentWorkflowRuns"); e != nil { + return nil, e + } + key := owner + "/" + repo + runs := f.RecentWorkflowRuns[key] + if perPage > 0 && len(runs) > perPage { + runs = runs[:perPage] + } + return append([]WorkflowRun(nil), runs...), nil +} + +func (f *FakeClient) DownloadWorkflowRunArtifact(_ context.Context, _, _ string, artifactID int) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + if e := f.err("DownloadWorkflowRunArtifact"); e != nil { + return nil, e + } + if f.WorkflowArtifactContents == nil { + return nil, ErrNotFound + } + data, ok := f.WorkflowArtifactContents[artifactID] + if !ok { + return nil, ErrNotFound + } + return append([]byte(nil), data...), nil +} + +func (f *FakeClient) ListRepositoryArtifacts(_ context.Context, owner, repo string, perPage int) ([]RepositoryArtifact, error) { + f.mu.Lock() + defer f.mu.Unlock() + if e := f.err("ListRepositoryArtifacts"); e != nil { + return nil, e + } + key := owner + "/" + repo + arts := f.RepositoryArtifacts[key] + if perPage > 0 && len(arts) > perPage { + arts = arts[:perPage] + } + return append([]RepositoryArtifact(nil), arts...), nil +} + func (f *FakeClient) GetWorkflowRunLogs(_ context.Context, _, _ string, _ int) (string, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/internal/forge/fake_test.go b/internal/forge/fake_test.go index 5102b73e6c..47b59cd2cc 100644 --- a/internal/forge/fake_test.go +++ b/internal/forge/fake_test.go @@ -772,3 +772,107 @@ func TestIsForbidden(t *testing.T) { assert.False(t, IsForbidden(errors.New("some error"))) assert.False(t, IsForbidden(nil)) } + +func TestIsNonFastForward(t *testing.T) { + assert.True(t, IsNonFastForward(ErrNonFastForward)) + assert.True(t, IsNonFastForward(errors.Join(errors.New("wrapper"), ErrNonFastForward))) + assert.False(t, IsNonFastForward(errors.New("some error"))) +} + +func TestFakeClient_GetIssue(t *testing.T) { + fc := NewFakeClient() + issue, err := fc.CreateIssue(context.Background(), "org", "repo", "t", "b") + require.NoError(t, err) + + got, err := fc.GetIssue(context.Background(), "org", "repo", issue.Number) + require.NoError(t, err) + assert.Equal(t, issue.Number, got.Number) + + _, err = fc.GetIssue(context.Background(), "org", "repo", 999) + require.ErrorIs(t, err, ErrNotFound) +} + +func TestFakeClient_AddIssueLabels(t *testing.T) { + fc := NewFakeClient() + issue, err := fc.CreateIssue(context.Background(), "org", "repo", "t", "b") + require.NoError(t, err) + + require.NoError(t, fc.AddIssueLabels(context.Background(), "org", "repo", issue.Number, "ready-for-triage")) + got, err := fc.GetIssue(context.Background(), "org", "repo", issue.Number) + require.NoError(t, err) + assert.Contains(t, got.Labels, "ready-for-triage") + + require.Error(t, fc.AddIssueLabels(context.Background(), "org", "repo", 999, "x")) +} + +func TestFakeClient_ListRecentWorkflowRuns(t *testing.T) { + fc := NewFakeClient() + fc.RecentWorkflowRuns = map[string][]WorkflowRun{ + "org/repo": { + {ID: 1, Name: "Triage Agent", Status: "completed", Conclusion: "success", CreatedAt: "2024-01-01T00:00:00Z"}, + }, + } + runs, err := fc.ListRecentWorkflowRuns(context.Background(), "org", "repo", 10) + require.NoError(t, err) + require.Len(t, runs, 1) +} + +func TestFakeClient_ListWorkflowRunArtifacts(t *testing.T) { + fc := NewFakeClient() + fc.WorkflowRunArtifacts = map[int][]WorkflowArtifact{ + 42: {{ID: 5, Name: "fullsend-triage"}}, + } + + arts, err := fc.ListWorkflowRunArtifacts(context.Background(), "org", "repo", 42) + require.NoError(t, err) + require.Len(t, arts, 1) + assert.Equal(t, "fullsend-triage", arts[0].Name) + + arts, err = fc.ListWorkflowRunArtifacts(context.Background(), "org", "repo", 99) + require.NoError(t, err) + assert.Nil(t, arts) +} + +func TestFakeClient_DownloadWorkflowRunArtifact(t *testing.T) { + fc := NewFakeClient() + fc.WorkflowArtifactContents = map[int][]byte{ + 9: []byte("artifact-bytes"), + } + + data, err := fc.DownloadWorkflowRunArtifact(context.Background(), "org", "repo", 9) + require.NoError(t, err) + assert.Equal(t, []byte("artifact-bytes"), data) + + _, err = fc.DownloadWorkflowRunArtifact(context.Background(), "org", "repo", 99) + require.ErrorIs(t, err, ErrNotFound) +} + +func TestFakeClient_ListRepositoryArtifacts(t *testing.T) { + fc := NewFakeClient() + fc.RepositoryArtifacts = map[string][]RepositoryArtifact{ + "org/repo": { + {ID: 1, Name: "a"}, + {ID: 2, Name: "b"}, + }, + } + + arts, err := fc.ListRepositoryArtifacts(context.Background(), "org", "repo", 1) + require.NoError(t, err) + require.Len(t, arts, 1) + assert.Equal(t, 1, arts[0].ID) + + arts, err = fc.ListRepositoryArtifacts(context.Background(), "org", "repo", 0) + require.NoError(t, err) + assert.Len(t, arts, 2) +} + +func TestFakeClient_AddIssueLabels_Idempotent(t *testing.T) { + fc := NewFakeClient() + issue, err := fc.CreateIssue(context.Background(), "org", "repo", "t", "b") + require.NoError(t, err) + + require.NoError(t, fc.AddIssueLabels(context.Background(), "org", "repo", issue.Number, "ready-for-triage", "ready-for-triage")) + got, err := fc.GetIssue(context.Background(), "org", "repo", issue.Number) + require.NoError(t, err) + assert.Equal(t, []string{"ready-for-triage"}, got.Labels) +} diff --git a/internal/forge/forge.go b/internal/forge/forge.go index 2c7914368c..961a8084a6 100644 --- a/internal/forge/forge.go +++ b/internal/forge/forge.go @@ -41,6 +41,15 @@ func IsBranchProtected(err error) bool { return errors.Is(err, ErrBranchProtected) } +// ErrNonFastForward indicates that a ref update was rejected because the +// branch advanced concurrently (not a fast-forward). +var ErrNonFastForward = errors.New("non-fast-forward update") + +// IsNonFastForward reports whether err indicates a non-fast-forward rejection. +func IsNonFastForward(err error) bool { + return errors.Is(err, ErrNonFastForward) +} + // ErrForbidden indicates that the operation was denied due to // insufficient permissions (e.g., the user lacks push access). var ErrForbidden = errors.New("forbidden") @@ -83,12 +92,27 @@ type ChangeProposal struct { type WorkflowRun struct { ID int Name string + Event string // GitHub trigger event, e.g. "issues", "issue_comment" Status string // "queued", "in_progress", "completed" Conclusion string // "success", "failure", "cancelled", etc. HTMLURL string CreatedAt string } +// WorkflowArtifact is a file bundle uploaded by a workflow run. +type WorkflowArtifact struct { + ID int + Name string +} + +// RepositoryArtifact is an artifact stored for a repository, with metadata. +type RepositoryArtifact struct { + ID int + Name string + CreatedAt string + WorkflowRunID int +} + // Workflow represents a workflow definition registered with the forge. type Workflow struct { ID int @@ -368,6 +392,8 @@ type Client interface { // Issue operations CreateIssue(ctx context.Context, owner, repo, title, body string, labels ...string) (*Issue, error) + AddIssueLabels(ctx context.Context, owner, repo string, number int, labels ...string) error + GetIssue(ctx context.Context, owner, repo string, number int) (*Issue, error) CloseIssue(ctx context.Context, owner, repo string, number int) error ListOpenIssues(ctx context.Context, owner, repo string, labels ...string) ([]Issue, error) ListIssueComments(ctx context.Context, owner, repo string, number int) ([]IssueComment, error) @@ -405,6 +431,15 @@ type Client interface { // Workflow run listing ListWorkflowRuns(ctx context.Context, owner, repo, workflowFile string) ([]WorkflowRun, error) + // ListRecentWorkflowRuns returns recent workflow runs across all workflows. + ListRecentWorkflowRuns(ctx context.Context, owner, repo string, perPage int) ([]WorkflowRun, error) + + // ListWorkflowRunArtifacts returns artifacts uploaded by a workflow run. + ListWorkflowRunArtifacts(ctx context.Context, owner, repo string, runID int) ([]WorkflowArtifact, error) + // DownloadWorkflowRunArtifact returns the zip archive for a workflow artifact. + DownloadWorkflowRunArtifact(ctx context.Context, owner, repo string, artifactID int) ([]byte, error) + // ListRepositoryArtifacts returns recent artifacts stored for a repository. + ListRepositoryArtifacts(ctx context.Context, owner, repo string, perPage int) ([]RepositoryArtifact, error) // GetWorkflowRunLogs downloads the logs for a workflow run as plain text. // On GitHub, this fetches job logs for each job in the run. diff --git a/internal/forge/github/github.go b/internal/forge/github/github.go index 260e9e0d18..31d6d9cdf6 100644 --- a/internal/forge/github/github.go +++ b/internal/forge/github/github.go @@ -739,7 +739,15 @@ func (c *LiveClient) CommitFiles(ctx context.Context, owner, repo, message strin return false, fmt.Errorf("decode repo info: %w", err) } - return c.commitFilesTo(ctx, owner, repo, repoInfo.DefaultBranch, message, files) + return c.commitFilesWithRetry(ctx, owner, repo, repoInfo.DefaultBranch, message, files) +} + +func (c *LiveClient) commitFilesWithRetry(ctx context.Context, owner, repo, branch, message string, files []forge.TreeFile) (bool, error) { + changed, err := c.commitFilesTo(ctx, owner, repo, branch, message, files) + if err != nil && forge.IsNonFastForward(err) { + changed, err = c.commitFilesTo(ctx, owner, repo, branch, message, files) + } + return changed, err } // CommitFilesToBranch atomically commits multiple files to a specific @@ -748,7 +756,7 @@ func (c *LiveClient) CommitFilesToBranch(ctx context.Context, owner, repo, branc if len(files) == 0 { return false, nil } - return c.commitFilesTo(ctx, owner, repo, branch, message, files) + return c.commitFilesWithRetry(ctx, owner, repo, branch, message, files) } // commitFilesTo is the shared implementation for CommitFiles and @@ -899,8 +907,13 @@ func (c *LiveClient) commitFilesTo(ctx context.Context, owner, repo, branch, mes refUpdateResp, err := c.patch(ctx, fmt.Sprintf("/repos/%s/%s/git/refs/heads/%s", owner, repo, branch), refPayload) if err != nil { var apiErr *APIError - if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusUnprocessableEntity && isBranchProtectionError(apiErr) { - return false, fmt.Errorf("%w: %w", forge.ErrBranchProtected, err) + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusUnprocessableEntity { + if isBranchProtectionError(apiErr) { + return false, fmt.Errorf("%w: %w", forge.ErrBranchProtected, err) + } + if isNonFastForwardError(apiErr) { + return false, fmt.Errorf("%w: %w", forge.ErrNonFastForward, err) + } } return false, fmt.Errorf("update ref: %w", err) } @@ -1063,6 +1076,14 @@ func isBranchProtectionError(apiErr *APIError) bool { strings.Contains(msg, "rule violation") } +func isNonFastForwardError(apiErr *APIError) bool { + msg := strings.ToLower(apiErr.Message) + for _, d := range apiErr.Errors { + msg += " " + strings.ToLower(d.Message) + } + return strings.Contains(msg, "fast forward") +} + func isAlreadyExistsError(apiErr *APIError) bool { msg := strings.ToLower(apiErr.Message) for _, d := range apiErr.Errors { @@ -1841,6 +1862,7 @@ func (c *LiveClient) GetLatestWorkflowRun(ctx context.Context, owner, repo, work WorkflowRuns []struct { ID int `json:"id"` Name string `json:"name"` + Event string `json:"event"` Status string `json:"status"` Conclusion string `json:"conclusion"` HTMLURL string `json:"html_url"` @@ -1859,6 +1881,7 @@ func (c *LiveClient) GetLatestWorkflowRun(ctx context.Context, owner, repo, work return &forge.WorkflowRun{ ID: run.ID, Name: run.Name, + Event: run.Event, Status: run.Status, Conclusion: run.Conclusion, HTMLURL: run.HTMLURL, @@ -1876,6 +1899,7 @@ func (c *LiveClient) GetWorkflowRun(ctx context.Context, owner, repo string, run var run struct { ID int `json:"id"` Name string `json:"name"` + Event string `json:"event"` Status string `json:"status"` Conclusion string `json:"conclusion"` HTMLURL string `json:"html_url"` @@ -1888,6 +1912,7 @@ func (c *LiveClient) GetWorkflowRun(ctx context.Context, owner, repo string, run return &forge.WorkflowRun{ ID: run.ID, Name: run.Name, + Event: run.Event, Status: run.Status, Conclusion: run.Conclusion, HTMLURL: run.HTMLURL, @@ -1957,6 +1982,19 @@ func (c *LiveClient) CreateIssue(ctx context.Context, owner, repo, title, body s }, nil } +// AddIssueLabels adds labels to an existing issue. +func (c *LiveClient) AddIssueLabels(ctx context.Context, owner, repo string, number int, labels ...string) error { + if len(labels) == 0 { + return nil + } + resp, err := c.post(ctx, fmt.Sprintf("/repos/%s/%s/issues/%d/labels", owner, repo, number), labels) + if err != nil { + return fmt.Errorf("add issue labels: %w", err) + } + resp.Body.Close() + return nil +} + // CloseIssue closes an issue by number. func (c *LiveClient) CloseIssue(ctx context.Context, owner, repo string, number int) error { resp, err := c.patch(ctx, fmt.Sprintf("/repos/%s/%s/issues/%d", owner, repo, number), map[string]string{"state": "closed"}) @@ -1967,6 +2005,37 @@ func (c *LiveClient) CloseIssue(ctx context.Context, owner, repo string, number return nil } +// GetIssue returns an issue by number. +func (c *LiveClient) GetIssue(ctx context.Context, owner, repo string, number int) (*forge.Issue, error) { + resp, err := c.get(ctx, fmt.Sprintf("/repos/%s/%s/issues/%d", owner, repo, number)) + if err != nil { + var apiErr *APIError + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound { + return nil, forge.ErrNotFound + } + return nil, fmt.Errorf("get issue #%d: %w", number, err) + } + var result struct { + Number int `json:"number"` + Title string `json:"title"` + Body string `json:"body"` + HTMLURL string `json:"html_url"` + Labels []struct { + Name string `json:"name"` + } `json:"labels"` + } + if err := decodeJSON(resp, &result); err != nil { + return nil, fmt.Errorf("decode issue #%d: %w", number, err) + } + return &forge.Issue{ + Number: result.Number, + Title: result.Title, + Body: result.Body, + URL: result.HTMLURL, + Labels: labelNames(result.Labels), + }, nil +} + func labelNames(labels []struct { Name string `json:"name"` }) []string { @@ -2389,6 +2458,7 @@ func (c *LiveClient) ListWorkflowRuns(ctx context.Context, owner, repo, workflow WorkflowRuns []struct { ID int `json:"id"` Name string `json:"name"` + Event string `json:"event"` Status string `json:"status"` Conclusion string `json:"conclusion"` HTMLURL string `json:"html_url"` @@ -2403,6 +2473,48 @@ func (c *LiveClient) ListWorkflowRuns(ctx context.Context, owner, repo, workflow runs[i] = forge.WorkflowRun{ ID: r.ID, Name: r.Name, + Event: r.Event, + Status: r.Status, + Conclusion: r.Conclusion, + HTMLURL: r.HTMLURL, + CreatedAt: r.CreatedAt, + } + } + return runs, nil +} + +// ListRecentWorkflowRuns returns recent workflow runs across all workflows. +func (c *LiveClient) ListRecentWorkflowRuns(ctx context.Context, owner, repo string, perPage int) ([]forge.WorkflowRun, error) { + if perPage <= 0 { + perPage = 30 + } + if perPage > 100 { + perPage = 100 + } + resp, err := c.get(ctx, fmt.Sprintf("/repos/%s/%s/actions/runs?per_page=%d", owner, repo, perPage)) + if err != nil { + return nil, fmt.Errorf("list recent workflow runs: %w", err) + } + var result struct { + WorkflowRuns []struct { + ID int `json:"id"` + Name string `json:"name"` + Event string `json:"event"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + HTMLURL string `json:"html_url"` + CreatedAt string `json:"created_at"` + } `json:"workflow_runs"` + } + if err := decodeJSON(resp, &result); err != nil { + return nil, fmt.Errorf("decode recent workflow runs: %w", err) + } + runs := make([]forge.WorkflowRun, len(result.WorkflowRuns)) + for i, r := range result.WorkflowRuns { + runs[i] = forge.WorkflowRun{ + ID: r.ID, + Name: r.Name, + Event: r.Event, Status: r.Status, Conclusion: r.Conclusion, HTMLURL: r.HTMLURL, @@ -2412,6 +2524,86 @@ func (c *LiveClient) ListWorkflowRuns(ctx context.Context, owner, repo, workflow return runs, nil } +// ListWorkflowRunArtifacts returns artifacts uploaded by a workflow run. +func (c *LiveClient) ListWorkflowRunArtifacts(ctx context.Context, owner, repo string, runID int) ([]forge.WorkflowArtifact, error) { + resp, err := c.get(ctx, fmt.Sprintf("/repos/%s/%s/actions/runs/%d/artifacts", owner, repo, runID)) + if err != nil { + return nil, fmt.Errorf("list workflow run artifacts: %w", err) + } + var result struct { + Artifacts []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"artifacts"` + } + if err := decodeJSON(resp, &result); err != nil { + return nil, fmt.Errorf("decode workflow run artifacts: %w", err) + } + artifacts := make([]forge.WorkflowArtifact, len(result.Artifacts)) + for i, art := range result.Artifacts { + artifacts[i] = forge.WorkflowArtifact{ID: art.ID, Name: art.Name} + } + return artifacts, nil +} + +// DownloadWorkflowRunArtifact returns the zip archive for a workflow artifact. +func (c *LiveClient) DownloadWorkflowRunArtifact(ctx context.Context, owner, repo string, artifactID int) ([]byte, error) { + resp, err := c.do(ctx, http.MethodGet, fmt.Sprintf("/repos/%s/%s/actions/artifacts/%d/zip", owner, repo, artifactID), nil) + if err != nil { + return nil, fmt.Errorf("download workflow artifact %d: %w", artifactID, err) + } + defer resp.Body.Close() + if err := checkStatus(resp, http.StatusOK); err != nil { + return nil, fmt.Errorf("download workflow artifact %d: %w", artifactID, err) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, 50<<20)) + if err != nil { + return nil, fmt.Errorf("read workflow artifact %d: %w", artifactID, err) + } + return data, nil +} + +// ListRepositoryArtifacts returns recent artifacts stored for a repository. +func (c *LiveClient) ListRepositoryArtifacts(ctx context.Context, owner, repo string, perPage int) ([]forge.RepositoryArtifact, error) { + if perPage <= 0 { + perPage = 100 + } + if perPage > 100 { + perPage = 100 + } + resp, err := c.get(ctx, fmt.Sprintf("/repos/%s/%s/actions/artifacts?per_page=%d", owner, repo, perPage)) + if err != nil { + return nil, fmt.Errorf("list repository artifacts: %w", err) + } + var result struct { + Artifacts []struct { + ID int `json:"id"` + Name string `json:"name"` + CreatedAt string `json:"created_at"` + Expired bool `json:"expired"` + WorkflowRun struct { + ID int `json:"id"` + } `json:"workflow_run"` + } `json:"artifacts"` + } + if err := decodeJSON(resp, &result); err != nil { + return nil, fmt.Errorf("decode repository artifacts: %w", err) + } + artifacts := make([]forge.RepositoryArtifact, 0, len(result.Artifacts)) + for _, art := range result.Artifacts { + if art.Expired { + continue + } + artifacts = append(artifacts, forge.RepositoryArtifact{ + ID: art.ID, + Name: art.Name, + CreatedAt: art.CreatedAt, + WorkflowRunID: art.WorkflowRun.ID, + }) + } + return artifacts, nil +} + // GetWorkflowRunLogs downloads the logs for a workflow run. // It fetches the job list for the run and concatenates each job's log output. func (c *LiveClient) GetWorkflowRunLogs(ctx context.Context, owner, repo string, runID int) (string, error) { diff --git a/internal/forge/github/github_test.go b/internal/forge/github/github_test.go index 44de469b08..973baa8e64 100644 --- a/internal/forge/github/github_test.go +++ b/internal/forge/github/github_test.go @@ -883,6 +883,7 @@ func TestGetWorkflowRun(t *testing.T) { json.NewEncoder(w).Encode(map[string]any{ "id": 42, "name": "Deploy", + "event": "workflow_dispatch", "status": "in_progress", "conclusion": "", "html_url": "https://github.com/owner/repo/actions/runs/42", @@ -896,6 +897,7 @@ func TestGetWorkflowRun(t *testing.T) { require.NoError(t, err) assert.Equal(t, 42, run.ID) assert.Equal(t, "Deploy", run.Name) + assert.Equal(t, "workflow_dispatch", run.Event) assert.Equal(t, "in_progress", run.Status) } @@ -2389,3 +2391,222 @@ func TestListOrgVariables(t *testing.T) { require.NoError(t, err) require.Len(t, vars, 2) } + +func TestGetIssue(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/repos/org/repo/issues/7", r.URL.Path) + json.NewEncoder(w).Encode(map[string]any{ + "number": 7, + "title": "Bug", + "body": "details", + "html_url": "https://github.com/org/repo/issues/7", + "labels": []map[string]string{{"name": "ready-to-code"}}, + }) + })) + defer srv.Close() + + client := newTestClient(t, srv) + issue, err := client.GetIssue(context.Background(), "org", "repo", 7) + require.NoError(t, err) + assert.Equal(t, 7, issue.Number) + assert.Equal(t, []string{"ready-to-code"}, issue.Labels) +} + +func TestListRecentWorkflowRuns(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/repos/org/repo/actions/runs", r.URL.Path) + assert.Equal(t, "20", r.URL.Query().Get("per_page")) + json.NewEncoder(w).Encode(map[string]any{ + "workflow_runs": []map[string]any{ + { + "id": 99, "name": "Triage Agent", "event": "issue_comment", + "status": "completed", "conclusion": "success", "html_url": "https://example/run/99", + "created_at": "2024-01-01T00:00:00Z", + }, + }, + }) + })) + defer srv.Close() + + client := newTestClient(t, srv) + runs, err := client.ListRecentWorkflowRuns(context.Background(), "org", "repo", 20) + require.NoError(t, err) + require.Len(t, runs, 1) + assert.Equal(t, "Triage Agent", runs[0].Name) + assert.Equal(t, "issue_comment", runs[0].Event) +} + +func TestListWorkflowRuns_IncludesEvent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/repos/org/repo/actions/workflows/fullsend.yaml/runs", r.URL.Path) + json.NewEncoder(w).Encode(map[string]any{ + "workflow_runs": []map[string]any{ + { + "id": 7, "name": "fullsend", "event": "issues", + "status": "completed", "conclusion": "success", + "html_url": "https://example/run/7", "created_at": "2024-01-01T00:00:00Z", + }, + }, + }) + })) + defer srv.Close() + + client := newTestClient(t, srv) + runs, err := client.ListWorkflowRuns(context.Background(), "org", "repo", "fullsend.yaml") + require.NoError(t, err) + require.Len(t, runs, 1) + assert.Equal(t, "issues", runs[0].Event) +} + +func TestListWorkflowRunArtifacts(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/repos/org/repo/actions/runs/42/artifacts", r.URL.Path) + json.NewEncoder(w).Encode(map[string]any{ + "artifacts": []map[string]any{ + {"id": 5, "name": "fullsend-triage"}, + }, + }) + })) + defer srv.Close() + + client := newTestClient(t, srv) + arts, err := client.ListWorkflowRunArtifacts(context.Background(), "org", "repo", 42) + require.NoError(t, err) + require.Len(t, arts, 1) + assert.Equal(t, 5, arts[0].ID) +} + +func TestListRepositoryArtifacts(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/repos/org/repo/actions/artifacts", r.URL.Path) + json.NewEncoder(w).Encode(map[string]any{ + "artifacts": []map[string]any{ + { + "id": 9, "name": "fullsend-triage", "created_at": "2024-01-01T00:00:00Z", + "expired": false, "workflow_run": map[string]any{"id": 42}, + }, + }, + }) + })) + defer srv.Close() + + client := newTestClient(t, srv) + arts, err := client.ListRepositoryArtifacts(context.Background(), "org", "repo", 100) + require.NoError(t, err) + require.Len(t, arts, 1) + assert.Equal(t, 42, arts[0].WorkflowRunID) +} + +func TestAddIssueLabels(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "POST", r.Method) + assert.Equal(t, "/repos/org/repo/issues/7/labels", r.URL.Path) + var labels []string + require.NoError(t, json.NewDecoder(r.Body).Decode(&labels)) + assert.Equal(t, []string{"ready-for-triage"}, labels) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + client := newTestClient(t, srv) + require.NoError(t, client.AddIssueLabels(context.Background(), "org", "repo", 7, "ready-for-triage")) +} + +func TestAddIssueLabels_Empty(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("unexpected HTTP request for empty labels") + })) + defer srv.Close() + + client := newTestClient(t, srv) + require.NoError(t, client.AddIssueLabels(context.Background(), "org", "repo", 7)) +} + +func TestDownloadWorkflowRunArtifact(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/repos/org/repo/actions/artifacts/9/zip", r.URL.Path) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("PK\x03\x04fake-zip")) + })) + defer srv.Close() + + client := newTestClient(t, srv) + data, err := client.DownloadWorkflowRunArtifact(context.Background(), "org", "repo", 9) + require.NoError(t, err) + assert.Equal(t, []byte("PK\x03\x04fake-zip"), data) +} + +func TestListRepositoryArtifacts_SkipsExpired(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ + "artifacts": []map[string]any{ + {"id": 1, "name": "expired", "expired": true, "workflow_run": map[string]any{"id": 1}}, + {"id": 2, "name": "fresh", "expired": false, "created_at": "2024-01-01T00:00:00Z", "workflow_run": map[string]any{"id": 42}}, + }, + }) + })) + defer srv.Close() + + client := newTestClient(t, srv) + arts, err := client.ListRepositoryArtifacts(context.Background(), "org", "repo", 100) + require.NoError(t, err) + require.Len(t, arts, 1) + assert.Equal(t, "fresh", arts[0].Name) +} + +func TestDownloadWorkflowRunArtifact_NotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + client := newTestClient(t, srv) + _, err := client.DownloadWorkflowRunArtifact(context.Background(), "org", "repo", 9) + require.Error(t, err) + assert.Contains(t, err.Error(), "download workflow artifact") +} + +func TestCommitFiles_NonFastForwardRetry(t *testing.T) { + patchCount := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == "GET" && r.URL.Path == "/repos/org/repo": + json.NewEncoder(w).Encode(map[string]string{"default_branch": "main"}) + case r.Method == "GET" && r.URL.Path == "/repos/org/repo/git/ref/heads/main": + sha := "abc123" + if patchCount > 0 { + sha = "def456" + } + json.NewEncoder(w).Encode(map[string]any{"object": map[string]string{"sha": sha}}) + case r.Method == "GET" && strings.HasPrefix(r.URL.Path, "/repos/org/repo/git/commits/"): + json.NewEncoder(w).Encode(map[string]any{"tree": map[string]string{"sha": "tree000"}}) + case r.Method == "GET" && strings.HasPrefix(r.URL.Path, "/repos/org/repo/git/trees/"): + json.NewEncoder(w).Encode(map[string]any{"tree": []any{}, "truncated": false}) + case r.Method == "POST" && r.URL.Path == "/repos/org/repo/git/trees": + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(map[string]string{"sha": "newtree"}) + case r.Method == "POST" && r.URL.Path == "/repos/org/repo/git/commits": + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(map[string]string{"sha": "newcommit"}) + case r.Method == "PATCH" && r.URL.Path == "/repos/org/repo/git/refs/heads/main": + patchCount++ + if patchCount == 1 { + w.WriteHeader(http.StatusUnprocessableEntity) + json.NewEncoder(w).Encode(map[string]string{"message": "Update is not a fast forward"}) + return + } + json.NewEncoder(w).Encode(map[string]any{}) + default: + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + })) + defer srv.Close() + + client := newTestClient(t, srv) + committed, err := client.CommitFiles(context.Background(), "org", "repo", "msg", []forge.TreeFile{ + {Path: "f.txt", Content: []byte("x"), Mode: "100644"}, + }) + require.NoError(t, err) + assert.True(t, committed) + assert.Equal(t, 2, patchCount) +} diff --git a/internal/runtime/dummy.go b/internal/runtime/dummy.go new file mode 100644 index 0000000000..537a9fad79 --- /dev/null +++ b/internal/runtime/dummy.go @@ -0,0 +1,270 @@ +package runtime + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "gopkg.in/yaml.v3" + + "github.com/fullsend-ai/fullsend/internal/sandbox" + "github.com/fullsend-ai/fullsend/internal/ui" +) + +const behaviourScriptRelPath = "behaviour/current-scenario.yaml" +const behaviourResultsFile = "behaviour-results.json" + +// BehaviourOperation is a single scripted step for the dummy runtime. +type BehaviourOperation struct { + Description string `yaml:"description" json:"description"` + Op string `yaml:"op" json:"op"` + Args string `yaml:"args" json:"args"` + Content string `yaml:"content,omitempty" json:"content,omitempty"` +} + +// BehaviourScript is the YAML committed to .fullsend/behaviour/current-scenario.yaml. +type BehaviourScript struct { + Ops []BehaviourOperation `yaml:"ops"` +} + +// BehaviourOpResult records the outcome of one scripted operation. +type BehaviourOpResult struct { + Description string `json:"description"` + Success bool `json:"success"` + Error string `json:"error,omitempty"` +} + +// BehaviourResults is written to output/behaviour-results.json in the sandbox. +type BehaviourResults struct { + Operations []BehaviourOpResult `json:"operations"` +} + +// DummyRuntime executes scripted operations in the real OpenShell sandbox. +type DummyRuntime struct{} + +func (DummyRuntime) Name() string { return "dummy" } + +func (DummyRuntime) System() string { return "fullsend.dummy" } + +func (DummyRuntime) ConfigDir() string { return sandbox.SandboxWorkspace + "/.dummy" } + +func (DummyRuntime) WorkspaceDir() string { return sandbox.SandboxWorkspace } + +func (DummyRuntime) EnvExports() []string { return nil } + +func (DummyRuntime) Bootstrap(input BootstrapInput) error { + sandboxName := input.SandboxName() + mkdirCmd := fmt.Sprintf("mkdir -p %s/output %s/.dummy", sandbox.SandboxWorkspace, sandbox.SandboxWorkspace) + _, _, _, err := sandbox.Exec(sandboxName, mkdirCmd, 10*time.Second) + return err +} + +func (r DummyRuntime) Run(_ context.Context, params RunParams, printer *ui.Printer, _ time.Time, _ *RunMetrics) (int, error) { + scriptPath := filepath.Join(params.FullsendDir, behaviourScriptRelPath) + script, err := LoadBehaviourScript(scriptPath) + if err != nil { + return 1, err + } + + results, execErr := executeBehaviourScript(params.SandboxName, params.RepoDir, script) + if writeErr := writeBehaviourResults(params.SandboxName, results); writeErr != nil && execErr == nil { + execErr = writeErr + } + + if execErr != nil { + printer.StepWarn("Dummy runtime: " + execErr.Error()) + } + + // Non-zero exitCode mirrors ClaudeRuntime: run.go warns on non-zero exit but + // only aborts on a non-nil Go error (infrastructure failures). + exitCode := 0 + for _, res := range results.Operations { + if !res.Success { + exitCode = 1 + break + } + } + return exitCode, execErr +} + +func (r DummyRuntime) ClearIterationArtifacts(sandboxName string) error { + clearCmd := fmt.Sprintf("rm -rf %s/output/*", r.WorkspaceDir()) + _, _, _, err := sandbox.Exec(sandboxName, clearCmd, 10*time.Second) + return err +} + +func (DummyRuntime) ExtractTranscripts(_ string, _ string, _ string) error { return nil } + +func (DummyRuntime) ExtractDebugLog(_ string, _ string, _ string) error { return nil } + +func (DummyRuntime) ParseTranscriptErrors(_ string) []TranscriptError { return nil } + +func (DummyRuntime) ParseTranscriptFile(_ string) (TranscriptError, bool) { + return TranscriptError{}, false +} + +func (DummyRuntime) EmitTranscriptErrors(w io.Writer, summaries []TranscriptError) { + emitTranscriptErrors(w, summaries) +} + +// LoadBehaviourScript reads and parses a behaviour scenario script from disk. +func LoadBehaviourScript(path string) (*BehaviourScript, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading behaviour script %s: %w", path, err) + } + var script BehaviourScript + if err := yaml.Unmarshal(data, &script); err != nil { + return nil, fmt.Errorf("parsing behaviour script %s: %w", path, err) + } + if len(script.Ops) == 0 { + return nil, fmt.Errorf("behaviour script %s has no operations", path) + } + return &script, nil +} + +func executeBehaviourScript(sandboxName, repoDir string, script *BehaviourScript) (BehaviourResults, error) { + var results BehaviourResults + var firstErr error + for _, op := range script.Ops { + res := BehaviourOpResult{Description: op.Description} + if err := executeBehaviourOp(sandboxName, repoDir, op); err != nil { + res.Success = false + res.Error = err.Error() + if firstErr == nil { + firstErr = err + } + } else { + res.Success = true + } + results.Operations = append(results.Operations, res) + } + return results, firstErr +} + +func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) error { + switch op.Op { + case "read_file": + path := strings.TrimSpace(op.Args) + if path == "" { + return fmt.Errorf("read_file requires a path") + } + remotePath, err := resolveSandboxPath(repoDir, path) + if err != nil { + return err + } + cmd := fmt.Sprintf("test -r %s", shellQuote(remotePath)) + _, stderr, exitCode, err := sandbox.Exec(sandboxName, cmd, 30*time.Second) + if err != nil { + return fmt.Errorf("read_file exec: %w", err) + } + if exitCode != 0 { + return fmt.Errorf("read_file failed: %s", strings.TrimSpace(stderr)) + } + return nil + case "url_get": + url := strings.TrimSpace(op.Args) + if url == "" { + return fmt.Errorf("url_get requires a URL") + } + cmd := fmt.Sprintf("curl -sf %s -o /dev/null", shellQuote(url)) + _, stderr, exitCode, err := sandbox.Exec(sandboxName, cmd, 60*time.Second) + if err != nil { + return fmt.Errorf("url_get exec: %w", err) + } + if exitCode != 0 { + return fmt.Errorf("url_get failed: %s", strings.TrimSpace(stderr)) + } + return nil + case "write_fixture": + dest, content, err := resolveWriteFixture(op) + if err != nil { + return err + } + remoteDest, err := resolveSandboxPath(sandbox.SandboxWorkspace, dest) + if err != nil { + return err + } + parentDir := filepath.Dir(remoteDest) + mkdirCmd := fmt.Sprintf("mkdir -p %s", shellQuote(parentDir)) + if _, _, _, err := sandbox.Exec(sandboxName, mkdirCmd, 10*time.Second); err != nil { + return fmt.Errorf("write_fixture mkdir: %w", err) + } + tmp, err := os.CreateTemp("", "behaviour-fixture-*") + if err != nil { + return fmt.Errorf("write_fixture temp file: %w", err) + } + defer os.Remove(tmp.Name()) + if _, err := tmp.WriteString(content); err != nil { + tmp.Close() + return fmt.Errorf("write_fixture write temp: %w", err) + } + tmp.Close() + if err := sandbox.Upload(sandboxName, tmp.Name(), remoteDest); err != nil { + return fmt.Errorf("write_fixture upload: %w", err) + } + return nil + default: + return fmt.Errorf("unknown op %q", op.Op) + } +} + +func resolveWriteFixture(op BehaviourOperation) (dest string, content string, err error) { + parts := strings.SplitN(op.Args, ",", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("write_fixture args must be dest_path, fixture_path (fixture path is for test embedding; runtime uses op.content)") + } + dest = strings.TrimSpace(parts[0]) + if dest == "" { + return "", "", fmt.Errorf("write_fixture requires dest_path") + } + if op.Content != "" { + return dest, op.Content, nil + } + return "", "", fmt.Errorf("write_fixture requires embedded content in script") +} + +func resolveSandboxPath(base, rel string) (string, error) { + baseClean := filepath.Clean(base) + if filepath.IsAbs(rel) || strings.HasPrefix(rel, sandbox.SandboxWorkspace) { + clean := filepath.Clean(rel) + wsClean := filepath.Clean(sandbox.SandboxWorkspace) + if clean != wsClean && !strings.HasPrefix(clean, wsClean+string(os.PathSeparator)) { + return "", fmt.Errorf("path %q escapes sandbox workspace", rel) + } + return clean, nil + } + resolved := filepath.Clean(filepath.Join(baseClean, rel)) + if resolved != baseClean && !strings.HasPrefix(resolved, baseClean+string(os.PathSeparator)) { + return "", fmt.Errorf("path %q escapes base %q", rel, base) + } + return resolved, nil +} + +func writeBehaviourResults(sandboxName string, results BehaviourResults) error { + data, err := json.MarshalIndent(results, "", " ") + if err != nil { + return fmt.Errorf("marshaling behaviour results: %w", err) + } + tmp, err := os.CreateTemp("", "behaviour-results-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + tmp.Close() + remotePath := filepath.Join(sandbox.SandboxWorkspace, "output", behaviourResultsFile) + return sandbox.Upload(sandboxName, tmp.Name(), remotePath) +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} diff --git a/internal/runtime/dummy_test.go b/internal/runtime/dummy_test.go new file mode 100644 index 0000000000..9e935c7473 --- /dev/null +++ b/internal/runtime/dummy_test.go @@ -0,0 +1,332 @@ +package runtime + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/fullsend-ai/fullsend/internal/sandbox" + "github.com/fullsend-ai/fullsend/internal/ui" +) + +func TestLoadBehaviourScript(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "current-scenario.yaml") + content := `ops: + - description: Emit JSON + op: write_fixture + args: output/agent-result.json, fixtures/triage/sufficient.json + content: '{"action":"sufficient"}' +` + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + script, err := LoadBehaviourScript(path) + require.NoError(t, err) + require.Len(t, script.Ops, 1) + assert.Equal(t, "Emit JSON", script.Ops[0].Description) + assert.Equal(t, "write_fixture", script.Ops[0].Op) + assert.Contains(t, script.Ops[0].Content, "sufficient") +} + +func TestResolveWriteFixtureEmbeddedContent(t *testing.T) { + t.Parallel() + + dest, content, err := resolveWriteFixture(BehaviourOperation{ + Op: "write_fixture", + Args: "output/agent-result.json, fixtures/triage/sufficient.json", + Content: "hello", + }) + require.NoError(t, err) + assert.Equal(t, "output/agent-result.json", dest) + assert.Equal(t, "hello", content) +} + +func TestResolveWriteFixtureMissingContent(t *testing.T) { + t.Parallel() + + _, _, err := resolveWriteFixture(BehaviourOperation{ + Op: "write_fixture", + Args: "output/agent-result.json, fixtures/triage/sufficient.json", + }) + require.Error(t, err) +} + +func TestResolveSandboxPathWithinBase(t *testing.T) { + t.Parallel() + + base := filepath.Join(t.TempDir(), "workspace") + require.NoError(t, os.MkdirAll(base, 0o755)) + + got, err := resolveSandboxPath(base, "output/file.json") + require.NoError(t, err) + assert.Equal(t, filepath.Join(base, "output/file.json"), got) +} + +func TestResolveSandboxPathRejectsEscape(t *testing.T) { + t.Parallel() + + base := filepath.Join(t.TempDir(), "workspace") + require.NoError(t, os.MkdirAll(base, 0o755)) + + _, err := resolveSandboxPath(base, "../outside") + require.Error(t, err) + assert.Contains(t, err.Error(), "escapes base") +} + +func TestResolveSandboxPathRejectsAbsoluteOutsideWorkspace(t *testing.T) { + t.Parallel() + + _, err := resolveSandboxPath(sandbox.SandboxWorkspace, "/etc/passwd") + require.Error(t, err) + assert.Contains(t, err.Error(), "escapes sandbox workspace") +} + +func TestExecuteBehaviourOpUnknown(t *testing.T) { + t.Parallel() + + err := executeBehaviourOp("unused", t.TempDir(), BehaviourOperation{Op: "nope"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown op") +} + +func TestLoadBehaviourScript_MissingFile(t *testing.T) { + t.Parallel() + + _, err := LoadBehaviourScript(filepath.Join(t.TempDir(), "missing.yaml")) + require.Error(t, err) + assert.Contains(t, err.Error(), "reading behaviour script") +} + +func TestLoadBehaviourScript_InvalidYAML(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte(":\n- bad"), 0o644)) + + _, err := LoadBehaviourScript(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "parsing behaviour script") +} + +func TestLoadBehaviourScript_EmptyOps(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "empty.yaml") + require.NoError(t, os.WriteFile(path, []byte("ops: []\n"), 0o644)) + + _, err := LoadBehaviourScript(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "no operations") +} + +func TestResolveWriteFixture_BadArgs(t *testing.T) { + t.Parallel() + + _, _, err := resolveWriteFixture(BehaviourOperation{Op: "write_fixture", Args: "onlyone"}) + require.Error(t, err) +} + +func TestResolveWriteFixture_EmptyDest(t *testing.T) { + t.Parallel() + + _, _, err := resolveWriteFixture(BehaviourOperation{ + Op: "write_fixture", + Args: " ,fixtures/triage/sufficient.json", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "dest_path") +} + +func TestShellQuote(t *testing.T) { + t.Parallel() + + assert.Equal(t, "'hello'", shellQuote("hello")) + assert.Equal(t, "'don'\\''t'", shellQuote("don't")) +} + +func TestDummyRuntimeMetadata(t *testing.T) { + t.Parallel() + + rt := DummyRuntime{} + assert.Equal(t, "dummy", rt.Name()) + assert.Equal(t, "fullsend.dummy", rt.System()) + assert.Contains(t, rt.ConfigDir(), ".dummy") + assert.Equal(t, sandbox.SandboxWorkspace, rt.WorkspaceDir()) + assert.Nil(t, rt.EnvExports()) +} + +func TestDummyRuntimeNoopMethods(t *testing.T) { + t.Parallel() + + rt := DummyRuntime{} + assert.NoError(t, rt.ExtractTranscripts("", "", "")) + assert.NoError(t, rt.ExtractDebugLog("", "", "")) + assert.Nil(t, rt.ParseTranscriptErrors("")) + + var buf bytes.Buffer + rt.EmitTranscriptErrors(&buf, nil) +} + +func TestDummyRuntime_ParseTranscriptFile(t *testing.T) { + t.Parallel() + + rt := DummyRuntime{} + _, ok := rt.ParseTranscriptFile("/nonexistent/path.jsonl") + assert.False(t, ok) +} + +func TestExecuteBehaviourScript_ValidationFailures(t *testing.T) { + t.Parallel() + + script := &BehaviourScript{Ops: []BehaviourOperation{ + {Op: "read_file", Description: "missing path"}, + {Op: "url_get", Description: "missing url"}, + }} + results, err := executeBehaviourScript("unused", t.TempDir(), script) + require.Error(t, err) + require.Contains(t, err.Error(), "requires a path") + require.Len(t, results.Operations, 2) + assert.False(t, results.Operations[0].Success) + assert.Contains(t, results.Operations[0].Error, "requires a path") + assert.False(t, results.Operations[1].Success) + assert.Contains(t, results.Operations[1].Error, "requires a URL") +} + +func TestExecuteBehaviourOp_ReadFileEmptyPath(t *testing.T) { + t.Parallel() + + err := executeBehaviourOp("unused", t.TempDir(), BehaviourOperation{Op: "read_file"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a path") +} + +func TestExecuteBehaviourOp_URLGetEmpty(t *testing.T) { + t.Parallel() + + err := executeBehaviourOp("unused", t.TempDir(), BehaviourOperation{Op: "url_get"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a URL") +} + +func TestResolveSandboxPath_AbsoluteWithinWorkspace(t *testing.T) { + t.Parallel() + + ws := sandbox.SandboxWorkspace + rel := filepath.Join(ws, "output", "file.json") + got, err := resolveSandboxPath(ws, rel) + require.NoError(t, err) + assert.Equal(t, rel, got) +} + +type stubBootstrapInput struct { + sandboxName string +} + +func (s stubBootstrapInput) SandboxName() string { return s.sandboxName } +func (s stubBootstrapInput) AgentPath() string { return "" } +func (s stubBootstrapInput) AgentName() string { return "test" } +func (s stubBootstrapInput) SkillDirs() []string { return nil } +func (s stubBootstrapInput) PluginDirs() []string { return nil } + +func TestDummyRuntime_Bootstrap(t *testing.T) { + t.Parallel() + + rt := DummyRuntime{} + err := rt.Bootstrap(stubBootstrapInput{sandboxName: "nonexistent-sandbox"}) + require.Error(t, err) +} + +func TestDummyRuntime_RunMissingScript(t *testing.T) { + t.Parallel() + + rt := DummyRuntime{} + exit, err := rt.Run(context.Background(), RunParams{ + FullsendDir: t.TempDir(), + SandboxName: "unused", + RepoDir: t.TempDir(), + }, ui.New(io.Discard), time.Now(), nil) + assert.Equal(t, 1, exit) + require.Error(t, err) + assert.Contains(t, err.Error(), "behaviour script") +} + +func TestDummyRuntime_ClearIterationArtifacts(t *testing.T) { + t.Parallel() + + rt := DummyRuntime{} + err := rt.ClearIterationArtifacts("nonexistent-sandbox") + require.Error(t, err) +} + +func TestExecuteBehaviourOp_ReadFileExecFailure(t *testing.T) { + t.Parallel() + + base := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(base, "f.txt"), []byte("x"), 0o644)) + err := executeBehaviourOp("nonexistent-sandbox", base, BehaviourOperation{ + Op: "read_file", + Args: "f.txt", + }) + require.Error(t, err) +} + +func TestExecuteBehaviourOp_URLGetExecFailure(t *testing.T) { + t.Parallel() + + err := executeBehaviourOp("nonexistent-sandbox", t.TempDir(), BehaviourOperation{ + Op: "url_get", + Args: "https://example.com", + }) + require.Error(t, err) +} + +func TestExecuteBehaviourOp_WriteFixtureInvalidArgs(t *testing.T) { + t.Parallel() + + err := executeBehaviourOp("nonexistent-sandbox", t.TempDir(), BehaviourOperation{ + Op: "write_fixture", + Args: "only-one-part", + }) + require.Error(t, err) +} + +func TestWriteBehaviourResults(t *testing.T) { + t.Parallel() + + err := writeBehaviourResults("nonexistent-sandbox", BehaviourResults{ + Operations: []BehaviourOpResult{{Success: true, Description: "ok"}}, + }) + require.Error(t, err) +} + +func TestDummyRuntime_RunFailedOps(t *testing.T) { + t.Parallel() + + fullsendDir := t.TempDir() + scriptDir := filepath.Join(fullsendDir, "behaviour") + require.NoError(t, os.MkdirAll(scriptDir, 0o755)) + script := `ops: +- description: missing file + op: read_file + args: does-not-exist.txt +` + require.NoError(t, os.WriteFile(filepath.Join(scriptDir, "current-scenario.yaml"), []byte(script), 0o644)) + + rt := DummyRuntime{} + exit, err := rt.Run(context.Background(), RunParams{ + SandboxName: "nonexistent-sandbox", + RepoDir: t.TempDir(), + FullsendDir: fullsendDir, + }, ui.New(io.Discard), time.Now(), nil) + assert.Equal(t, 1, exit) + require.Error(t, err) +} diff --git a/internal/runtime/registry.go b/internal/runtime/registry.go new file mode 100644 index 0000000000..f201675700 --- /dev/null +++ b/internal/runtime/registry.go @@ -0,0 +1,41 @@ +package runtime + +import ( + "fmt" + "strings" + + "github.com/fullsend-ai/fullsend/internal/config" +) + +// Resolve returns the agent backend for the given runtime name. +func Resolve(name string) (Backend, error) { + switch name { + case "", "claude": + r := ClaudeRuntime{} + return Backend{Runtime: r, Transcripts: r}, nil + case "dummy": + // Selected only via explicit per-repo/org config (behaviour test orgs). + r := DummyRuntime{} + return Backend{Runtime: r, Transcripts: r}, nil + default: + return Backend{}, fmt.Errorf("unknown runtime %q: must be one of %s", name, strings.Join(config.ValidRuntimes(), ", ")) + } +} + +// ResolveFromConfig selects the runtime backend from org config defaults. +func ResolveFromConfig(cfg *config.OrgConfig) (Backend, error) { + rt := "claude" + if cfg != nil && cfg.Defaults.Runtime != "" { + rt = cfg.Defaults.Runtime + } + return Resolve(rt) +} + +// ResolveFromPerRepoConfig selects the runtime backend from per-repo config. +func ResolveFromPerRepoConfig(cfg *config.PerRepoConfig) (Backend, error) { + rt := "claude" + if cfg != nil && cfg.Runtime != "" { + rt = cfg.Runtime + } + return Resolve(rt) +} diff --git a/internal/runtime/registry_test.go b/internal/runtime/registry_test.go new file mode 100644 index 0000000000..a7c14a9cb3 --- /dev/null +++ b/internal/runtime/registry_test.go @@ -0,0 +1,50 @@ +package runtime + +import ( + "testing" + + "github.com/fullsend-ai/fullsend/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolve(t *testing.T) { + t.Parallel() + + claude, err := Resolve("claude") + require.NoError(t, err) + assert.Equal(t, "claude", claude.Runtime.Name()) + + dummy, err := Resolve("dummy") + require.NoError(t, err) + assert.Equal(t, "dummy", dummy.Runtime.Name()) + + _, err = Resolve("unknown") + require.Error(t, err) +} + +func TestResolveFromConfig(t *testing.T) { + t.Parallel() + + defaultBackend, err := ResolveFromConfig(nil) + require.NoError(t, err) + assert.Equal(t, "claude", defaultBackend.Runtime.Name()) + + cfg := &config.OrgConfig{Defaults: config.RepoDefaults{Runtime: "dummy"}} + dummyBackend, err := ResolveFromConfig(cfg) + require.NoError(t, err) + assert.Equal(t, "dummy", dummyBackend.Runtime.Name()) +} + +func TestResolveFromPerRepoConfig(t *testing.T) { + t.Parallel() + + defaultBackend, err := ResolveFromPerRepoConfig(nil) + require.NoError(t, err) + assert.Equal(t, "claude", defaultBackend.Runtime.Name()) + + cfg := &config.PerRepoConfig{Version: "1", Runtime: "dummy"} + dummyBackend, err := ResolveFromPerRepoConfig(cfg) + require.NoError(t, err) + assert.Equal(t, "dummy", dummyBackend.Runtime.Name()) +} diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 1342cfc7ef..49b02c6ca4 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -26,6 +26,7 @@ type RunParams struct { AgentBaseName string Model string RepoDir string + FullsendDir string PluginDirs []string Debug string Timeout time.Duration @@ -61,7 +62,7 @@ type Backend struct { Transcripts TranscriptHandler } -// Default returns the configured agent backend (Claude Code today). +// Default returns the Claude Code backend. Prefer ResolveFromConfig for org-aware selection. func Default() Backend { r := ClaudeRuntime{} return Backend{Runtime: r, Transcripts: r} From 5b4a7b4eb6226d34212d2b2816d5231171bc4dc2 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 12:25:38 +0300 Subject: [PATCH 02/22] ci(e2e): enroll pool test-repos in mint during behaviour install Vendored per-repo workflows need PER_REPO_WIF_REPOS for triage OIDC. Run fullsend mint enroll after github setup when E2E_GCP_PROJECT_ID is set. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/guides/dev/behaviour-testing.md | 4 +-- .../drivers/install/perrepo_github.go | 32 ++++++++++++++++++- .../drivers/install/perrepo_github_test.go | 9 ++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index 941ddc5823..e2c8091f1a 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -68,7 +68,7 @@ Use tags only for **exceptions** when a backend cannot run a scenario yet: `@ski make behaviour-test ``` -In CI, behaviour tests mint cross-org `e2e` installation tokens via OIDC (same as admin e2e). Pool orgs need only exist with mint-enrolled apps and a `test-repo`; the suite installs fullsend per-repo via `fullsend github setup` before scenarios run. +In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver runs `fullsend github setup` then `fullsend mint enroll /test-repo` when `E2E_GCP_PROJECT_ID` is set (CI). Runner env (defaults shown): @@ -76,7 +76,7 @@ Runner env (defaults shown): BEHAVIOUR_SCM=github BEHAVIOUR_CI=githubactions BEHAVIOUR_INSTALL_MODE=per-repo -E2E_GCP_PROJECT_ID=... # required in CI for per-repo inference secrets +E2E_GCP_PROJECT_ID=... # required in CI for mint enroll + per-repo inference secrets E2E_GCP_WIF_PROVIDER=... # required in CI for per-repo WIF secret ``` diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index 302209a391..caa87f78b5 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -16,7 +16,8 @@ import ( ) const ( - perRepoTestRepo = "test-repo" + perRepoTestRepo = "test-repo" + defaultMintRegion = "us-central1" // Vendored per-repo: issue_comment triggers the fullsend.yaml shim, which // workflow_calls reusable-dispatch → reusable-triage synchronously. perRepoTriageWorkflow = "fullsend.yaml" @@ -81,6 +82,10 @@ func (d *perRepoDriver) Install(ctx context.Context, org string) (State, error) return nil, fmt.Errorf("github setup %s: %w", target, err) } + if err := d.enrollRepoInMint(target); err != nil { + return nil, err + } + st := &perRepoState{org: org, repo: repo} if err := validatePerRepoPostInstall(ctx, d.client, org, repo); err != nil { return nil, err @@ -88,6 +93,31 @@ func (d *perRepoDriver) Install(ctx context.Context, org string) (State, error) return st, nil } +// perRepoMintEnrollArgs returns CLI args for enrolling a vendored per-repo install +// in mint (PER_REPO_WIF_REPOS + repo-scoped WIF provider). +func perRepoMintEnrollArgs(repoFullName, gcpProject string) []string { + return []string{ + "mint", "enroll", repoFullName, + "--project", gcpProject, + "--region", defaultMintRegion, + } +} + +func (d *perRepoDriver) enrollRepoInMint(repoFullName string) error { + project := strings.TrimSpace(d.e2eCfg.GCPProjectID) + if project == "" { + d.logf("[install] skipping mint enroll for %s (E2E_GCP_PROJECT_ID not set; repo must already be in PER_REPO_WIF_REPOS)", repoFullName) + return nil + } + + args := perRepoMintEnrollArgs(repoFullName, project) + d.logf("[install] running fullsend %s", strings.Join(args, " ")) + if _, err := admin.TryRunCLI(d.binary, d.token, args...); err != nil { + return fmt.Errorf("mint enroll %s: %w", repoFullName, err) + } + return nil +} + func (d *perRepoDriver) Teardown(ctx context.Context, org string, state State) error { repo := state.TestRepo() d.logf("[install] tearing down per-repo install on %s/%s", org, repo) diff --git a/e2e/behaviour/drivers/install/perrepo_github_test.go b/e2e/behaviour/drivers/install/perrepo_github_test.go index e50b97e15d..bce807249e 100644 --- a/e2e/behaviour/drivers/install/perrepo_github_test.go +++ b/e2e/behaviour/drivers/install/perrepo_github_test.go @@ -40,6 +40,15 @@ func TestValidatePerRepoPostInstall_MissingShim(t *testing.T) { assert.Contains(t, err.Error(), "fullsend.yaml") } +func TestPerRepoMintEnrollArgs(t *testing.T) { + args := perRepoMintEnrollArgs("halfsend-05/test-repo", "my-mint-project") + assert.Equal(t, []string{ + "mint", "enroll", "halfsend-05/test-repo", + "--project", "my-mint-project", + "--region", "us-central1", + }, args) +} + func TestValidatePerRepoPostInstall_WrongRuntime(t *testing.T) { client := forge.NewFakeClient() org, repo := "acme", "test-repo" From 8d2c52aa33573669e93470c7ec961c4ccb4b92d9 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 12:41:01 +0300 Subject: [PATCH 03/22] docs(e2e): document CI IAM for behaviour per-repo mint enroll Behaviour tests call mint enroll from CI; document the four GCP roles required on the hosted mint project and cross-link from behaviour guides. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/guides/dev/behaviour-drivers.md | 2 +- docs/guides/dev/behaviour-testing.md | 2 +- docs/guides/dev/e2e-testing.md | 42 ++++++++++++++++--- .../infrastructure/mint-administration.md | 2 + 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md index 8d7821aefc..86778677b9 100644 --- a/docs/guides/dev/behaviour-drivers.md +++ b/docs/guides/dev/behaviour-drivers.md @@ -31,7 +31,7 @@ The suite in `e2e/behaviour/suite_test.go` acquires a pool org, runs pre-install ### Install driver (v1 per-repo) -Uses `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with inference flags from `E2E_GCP_PROJECT_ID` and `E2E_GCP_WIF_PROVIDER`. Pool orgs must already have shared GitHub Apps and mint enrollment; the driver does not run `fullsend admin install`. +Uses `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with inference flags from `E2E_GCP_PROJECT_ID` and `E2E_GCP_WIF_PROVIDER`. When `E2E_GCP_PROJECT_ID` is set (CI), the driver then runs `fullsend mint enroll /test-repo` so vendored workflows can mint same-org triage tokens. Pool orgs must already have shared GitHub Apps and org-level mint enrollment; the driver does not run `fullsend admin install`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment) for CI service-account IAM on the mint project. Teardown removes shim workflows, stale branches, and open fullsend PRs on `test-repo` via shared helpers in `e2e/admin/cleanup.go`. diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index e2c8091f1a..a4c341ef1f 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -68,7 +68,7 @@ Use tags only for **exceptions** when a backend cannot run a scenario yet: `@ski make behaviour-test ``` -In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver runs `fullsend github setup` then `fullsend mint enroll /test-repo` when `E2E_GCP_PROJECT_ID` is set (CI). +In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver runs `fullsend github setup` then `fullsend mint enroll /test-repo` when `E2E_GCP_PROJECT_ID` is set (CI). The CI service account needs mint-admin IAM on that project — see [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). Runner env (defaults shown): diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md index d9f2d970bb..c9cb17b4bd 100644 --- a/docs/guides/dev/e2e-testing.md +++ b/docs/guides/dev/e2e-testing.md @@ -15,7 +15,7 @@ Playwright or stored sessions. Before running e2e locally or in CI: -1. **Pool orgs** (`halfsend-01` … `halfsend-06`) provisioned per [Pool org provisioning](#pool-org-provisioning) below +1. **Pool orgs** (`halfsend-01` … `halfsend-12`) provisioned per [Pool org provisioning](#pool-org-provisioning) below 2. **Mint** deployed with `e2e` role enrolled and `ALLOWED_ORGS` including `fullsend-ai` 3. **CI only:** pool orgs with `FULLSEND_FOREIGN_E2E_REPOS` authorizing `fullsend-ai/fullsend` 4. **Local only:** `gh auth login` (or `GH_TOKEN` / `GITHUB_TOKEN`) with admin access on pool orgs @@ -36,12 +36,12 @@ Optional environment variables: | `GH_TOKEN` / `GITHUB_TOKEN` | Override token source for local runs | | `FULLSEND_MINT_URL` | Override mint endpoint (default: hosted public mint, same as `fullsend admin --mint-url`) | | `E2E_LOCK_TIMEOUT` | Max wait for a free pool org (default 10m) | -| `E2E_GCP_PROJECT_ID` | GCP project for inference-related setup (if needed) | +| `E2E_GCP_PROJECT_ID` | GCP project for inference setup and behaviour per-repo mint enroll (must be the mint project when using the hosted mint) | Behaviour tests use the same pool orgs but install via `fullsend github setup` (per-repo) instead of `fullsend admin install`. See [behaviour-testing.md](behaviour-testing.md) and [behaviour-drivers.md](behaviour-drivers.md). Tests acquire an exclusive lock on one org from the pool (`halfsend-01` … -`halfsend-06`) — see [ADR 0040](../../ADRs/0040-org-pool-for-parallel-e2e-tests.md). +`halfsend-12`) — see [ADR 0040](../../ADRs/0040-org-pool-for-parallel-e2e-tests.md). ## CI runs @@ -57,10 +57,42 @@ Required repository secrets: |--------|---------| | `E2E_GCP_WIF_PROVIDER` | GCP WIF provider (inference / auxiliary GCP access) | | `E2E_GCP_SERVICE_ACCOUNT` | GCP service account for WIF | -| `E2E_GCP_PROJECT_ID` | GCP project ID | +| `E2E_GCP_PROJECT_ID` | GCP project ID (mint project for hosted mint: `it-gcp-konflux-dev-fullsend`) | Mint URL uses the hosted public endpoint by default (same as `fullsend admin --mint-url`). Override with org/repo variable `FULLSEND_MINT_URL` if needed; no separate e2e secret. +### Behaviour tests and per-repo mint enrollment + +Behaviour tests install fullsend in **per-repo** mode (`fullsend github setup`) and run `fullsend mint enroll /test-repo` so vendored reusable workflows can mint same-org `triage` tokens (`PER_REPO_WIF_REPOS`). This is the first CI workflow that exercises per-repo mint enrollment; admin e2e only needs org-level cross-org `e2e` mint auth. + +The CI service account (`E2E_GCP_SERVICE_ACCOUNT`, impersonated via `E2E_GCP_WIF_PROVIDER`) must have these roles on the **mint GCP project** (`E2E_GCP_PROJECT_ID`): + +| IAM role | Purpose | +|----------|---------| +| `roles/cloudfunctions.viewer` | Discover mint (`fullsend-mint`) during `mint enroll` | +| `roles/run.admin` | Update Cloud Run env vars (`ALLOWED_ORGS`, `PER_REPO_WIF_REPOS`) | +| `roles/iam.workloadIdentityPoolAdmin` | Create/update repo-scoped WIF providers | +| `roles/resourcemanager.projectIamAdmin` | Grant `roles/aiplatform.user` to per-repo WIF principals | + +Hosted mint example (project `it-gcp-konflux-dev-fullsend`, CI SA `github-fullsend-ai-fullsend-ci@it-gcp-konflux-dev-fullsend.iam.gserviceaccount.com`): + +```bash +export GCP_PROJECT=it-gcp-konflux-dev-fullsend +export E2E_SA=github-fullsend-ai-fullsend-ci@${GCP_PROJECT}.iam.gserviceaccount.com + +for ROLE in \ + roles/cloudfunctions.viewer \ + roles/run.admin \ + roles/iam.workloadIdentityPoolAdmin \ + roles/resourcemanager.projectIamAdmin; do + gcloud projects add-iam-policy-binding "$GCP_PROJECT" \ + --member="serviceAccount:${E2E_SA}" \ + --role="$ROLE" +done +``` + +`mint enroll` is idempotent — behaviour install can re-run safely when a pool org's `test-repo` is already enrolled. + ## Pool org provisioning Each pool org must be provisioned before e2e can use it: @@ -84,7 +116,7 @@ fullsend admin foreign list --org halfsend-01 # expect e2e → fullsend-ai/fullsend ``` -Existing pool orgs (`halfsend-01` … `halfsend-06`) need a one-time operator pass: install the e2e app (if missing) and run: +Existing pool orgs (`halfsend-01` … `halfsend-12`) need a one-time operator pass: install the e2e app (if missing) and run: ```bash fullsend admin foreign allow --org halfsend-NN --role e2e --caller fullsend-ai/fullsend diff --git a/docs/guides/infrastructure/mint-administration.md b/docs/guides/infrastructure/mint-administration.md index 56c07a1106..57a87aa22a 100644 --- a/docs/guides/infrastructure/mint-administration.md +++ b/docs/guides/infrastructure/mint-administration.md @@ -72,6 +72,8 @@ Pass this URL as `--mint-url` when running `fullsend github setup`, or set the ` `roles/owner` covers all of the above for users with broad access. + **Behaviour / e2e CI:** When CI runs `fullsend mint enroll owner/repo` (per-repo behaviour tests), the GitHub Actions service account impersonated via WIF needs the per-repo `mint enroll` roles on the mint project — not just inference roles. See [e2e-testing.md](../dev/e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). + An administrator can grant all required roles with a single script: ```bash From 89629a10df9fa23f98f85e7f8928dd2f83f6e02b Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 12:53:26 +0300 Subject: [PATCH 04/22] ci(e2e): use hosted mint project for behaviour mint enroll E2E_GCP_PROJECT_ID targets inference, not the hosted mint deployment. Resolve mint enroll --project via E2E_GCP_MINT_PROJECT_ID or default to it-gcp-konflux-dev-fullsend when using the public mint URL. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/guides/dev/behaviour-testing.md | 3 +- docs/guides/dev/e2e-testing.md | 9 ++-- e2e/admin/auth.go | 21 +++++++++ e2e/admin/auth_test.go | 43 +++++++++++++++++++ .../drivers/install/perrepo_github.go | 4 +- 5 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 e2e/admin/auth_test.go diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index a4c341ef1f..1716080ee3 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -76,7 +76,8 @@ Runner env (defaults shown): BEHAVIOUR_SCM=github BEHAVIOUR_CI=githubactions BEHAVIOUR_INSTALL_MODE=per-repo -E2E_GCP_PROJECT_ID=... # required in CI for mint enroll + per-repo inference secrets +E2E_GCP_PROJECT_ID=... # inference project for per-repo WIF secrets +E2E_GCP_MINT_PROJECT_ID=... # optional; defaults to hosted mint project when using public mint URL E2E_GCP_WIF_PROVIDER=... # required in CI for per-repo WIF secret ``` diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md index c9cb17b4bd..76ec8a4bf0 100644 --- a/docs/guides/dev/e2e-testing.md +++ b/docs/guides/dev/e2e-testing.md @@ -57,7 +57,8 @@ Required repository secrets: |--------|---------| | `E2E_GCP_WIF_PROVIDER` | GCP WIF provider (inference / auxiliary GCP access) | | `E2E_GCP_SERVICE_ACCOUNT` | GCP service account for WIF | -| `E2E_GCP_PROJECT_ID` | GCP project ID (mint project for hosted mint: `it-gcp-konflux-dev-fullsend`) | +| `E2E_GCP_PROJECT_ID` | GCP project ID for inference secrets (`github setup --inference-project`) | +| `E2E_GCP_MINT_PROJECT_ID` | Optional override for `mint enroll --project` (default: hosted mint project when using public mint URL) | Mint URL uses the hosted public endpoint by default (same as `fullsend admin --mint-url`). Override with org/repo variable `FULLSEND_MINT_URL` if needed; no separate e2e secret. @@ -65,7 +66,7 @@ Mint URL uses the hosted public endpoint by default (same as `fullsend admin --m Behaviour tests install fullsend in **per-repo** mode (`fullsend github setup`) and run `fullsend mint enroll /test-repo` so vendored reusable workflows can mint same-org `triage` tokens (`PER_REPO_WIF_REPOS`). This is the first CI workflow that exercises per-repo mint enrollment; admin e2e only needs org-level cross-org `e2e` mint auth. -The CI service account (`E2E_GCP_SERVICE_ACCOUNT`, impersonated via `E2E_GCP_WIF_PROVIDER`) must have these roles on the **mint GCP project** (`E2E_GCP_PROJECT_ID`): +The CI service account (`E2E_GCP_SERVICE_ACCOUNT`, impersonated via `E2E_GCP_WIF_PROVIDER`) must have these roles on the **mint GCP project** — not necessarily the same project as `E2E_GCP_PROJECT_ID` (inference). Behaviour install resolves the mint project via `E2E_GCP_MINT_PROJECT_ID`, or defaults to `it-gcp-konflux-dev-fullsend` when using the hosted mint URL: | IAM role | Purpose | |----------|---------| @@ -74,11 +75,11 @@ The CI service account (`E2E_GCP_SERVICE_ACCOUNT`, impersonated via `E2E_GCP_WIF | `roles/iam.workloadIdentityPoolAdmin` | Create/update repo-scoped WIF providers | | `roles/resourcemanager.projectIamAdmin` | Grant `roles/aiplatform.user` to per-repo WIF principals | -Hosted mint example (project `it-gcp-konflux-dev-fullsend`, CI SA `github-fullsend-ai-fullsend-ci@it-gcp-konflux-dev-fullsend.iam.gserviceaccount.com`): +Hosted mint example (project `it-gcp-konflux-dev-fullsend`). Grant roles on the **same** service account stored in the `E2E_GCP_SERVICE_ACCOUNT` repository secret (commonly `github-fullsend-ai-fullsen-966@…` or `github-fullsend-ai-fullsend-ci@…`): ```bash export GCP_PROJECT=it-gcp-konflux-dev-fullsend -export E2E_SA=github-fullsend-ai-fullsend-ci@${GCP_PROJECT}.iam.gserviceaccount.com +export E2E_SA="" for ROLE in \ roles/cloudfunctions.viewer \ diff --git a/e2e/admin/auth.go b/e2e/admin/auth.go index b288115a48..7a337b32f5 100644 --- a/e2e/admin/auth.go +++ b/e2e/admin/auth.go @@ -50,6 +50,27 @@ func resolveMintURL() string { return cli.DefaultMintURL } +// DefaultHostedMintGCPProject is the GCP project hosting the public mint service. +// See docs/guides/infrastructure/mint-administration.md. +const DefaultHostedMintGCPProject = "it-gcp-konflux-dev-fullsend" + +// MintEnrollProjectID returns the GCP project for `fullsend mint enroll`. +// Inference may use a different project via E2E_GCP_PROJECT_ID; the hosted mint +// always lives in DefaultHostedMintGCPProject unless E2E_GCP_MINT_PROJECT_ID is set. +func MintEnrollProjectID(cfg EnvConfig) string { + if p := strings.TrimSpace(os.Getenv("E2E_GCP_MINT_PROJECT_ID")); p != "" { + return p + } + mintURL := strings.TrimSpace(cfg.MintURL) + if mintURL == "" { + mintURL = cli.DefaultMintURL + } + if mintURL == cli.DefaultMintURL { + return DefaultHostedMintGCPProject + } + return strings.TrimSpace(cfg.GCPProjectID) +} + // resolveE2EToken mints a cross-org e2e installation token for targetOrg. // Repos are omitted so the token covers the full installation (needed to // create and operate on e2e-lock and .fullsend at runtime). diff --git a/e2e/admin/auth_test.go b/e2e/admin/auth_test.go new file mode 100644 index 0000000000..76bb46ea5c --- /dev/null +++ b/e2e/admin/auth_test.go @@ -0,0 +1,43 @@ +//go:build e2e || behaviour + +package admin + +import ( + "os" + "testing" + + "github.com/fullsend-ai/fullsend/internal/cli" + "github.com/stretchr/testify/assert" +) + +func TestMintEnrollProjectID(t *testing.T) { + t.Setenv("E2E_GCP_MINT_PROJECT_ID", "") + cfg := EnvConfig{ + MintURL: cli.DefaultMintURL, + GCPProjectID: "inference-only-project", + } + assert.Equal(t, DefaultHostedMintGCPProject, MintEnrollProjectID(cfg)) + + t.Setenv("E2E_GCP_MINT_PROJECT_ID", "override-mint-project") + assert.Equal(t, "override-mint-project", MintEnrollProjectID(cfg)) + + t.Setenv("E2E_GCP_MINT_PROJECT_ID", "") + cfg.MintURL = "https://mint.example.com" + assert.Equal(t, "inference-only-project", MintEnrollProjectID(cfg)) +} + +func TestMintEnrollProjectID_EmptyWithoutHostedMint(t *testing.T) { + t.Setenv("E2E_GCP_MINT_PROJECT_ID", "") + cfg := EnvConfig{ + MintURL: "https://mint.example.com", + GCPProjectID: "", + } + assert.Empty(t, MintEnrollProjectID(cfg)) +} + +func TestMintEnrollProjectID_RespectsEnvOverride(t *testing.T) { + t.Setenv("E2E_GCP_MINT_PROJECT_ID", "from-env") + cfg := EnvConfig{MintURL: cli.DefaultMintURL} + assert.Equal(t, "from-env", MintEnrollProjectID(cfg)) + _ = os.Unsetenv("E2E_GCP_MINT_PROJECT_ID") +} diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index caa87f78b5..2bbb7b069b 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -104,9 +104,9 @@ func perRepoMintEnrollArgs(repoFullName, gcpProject string) []string { } func (d *perRepoDriver) enrollRepoInMint(repoFullName string) error { - project := strings.TrimSpace(d.e2eCfg.GCPProjectID) + project := admin.MintEnrollProjectID(d.e2eCfg) if project == "" { - d.logf("[install] skipping mint enroll for %s (E2E_GCP_PROJECT_ID not set; repo must already be in PER_REPO_WIF_REPOS)", repoFullName) + d.logf("[install] skipping mint enroll for %s (no mint GCP project; set E2E_GCP_MINT_PROJECT_ID or E2E_GCP_PROJECT_ID, or use hosted mint URL)", repoFullName) return nil } From 49dd246b68c1c60f745c4936326afdab883084ed Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 12:57:53 +0300 Subject: [PATCH 05/22] ci(e2e): log GCP principal during behaviour mint enroll Aids CI diagnosis when mint enroll fails with IAM permission errors. Signed-off-by: Barak Korren Co-authored-by: Cursor --- .../drivers/install/perrepo_github.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index 2bbb7b069b..51d305f690 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -4,7 +4,9 @@ package install import ( "context" + "encoding/json" "fmt" + "os" "path/filepath" "strings" @@ -111,6 +113,7 @@ func (d *perRepoDriver) enrollRepoInMint(repoFullName string) error { } args := perRepoMintEnrollArgs(repoFullName, project) + d.logf("[install] mint enroll project=%s gcp_principal=%s", project, gcpCredentialPrincipal()) d.logf("[install] running fullsend %s", strings.Join(args, " ")) if _, err := admin.TryRunCLI(d.binary, d.token, args...); err != nil { return fmt.Errorf("mint enroll %s: %w", repoFullName, err) @@ -118,6 +121,37 @@ func (d *perRepoDriver) enrollRepoInMint(repoFullName string) error { return nil } +// gcpCredentialPrincipal returns a short label for the active GCP credential +// (service account email or external account) to aid CI debugging. +func gcpCredentialPrincipal() string { + path := strings.TrimSpace(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) + if path == "" { + return "unknown" + } + data, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return "unreadable" + } + var creds struct { + ClientEmail string `json:"client_email"` + Type string `json:"type"` + ServiceURL string `json:"service_account_impersonation_url"` + } + if err := json.Unmarshal(data, &creds); err != nil { + return "unparsed" + } + if creds.ClientEmail != "" { + return creds.ClientEmail + } + if creds.Type != "" { + return creds.Type + } + if creds.ServiceURL != "" { + return creds.ServiceURL + } + return "unknown" +} + func (d *perRepoDriver) Teardown(ctx context.Context, org string, state State) error { repo := state.TestRepo() d.logf("[install] tearing down per-repo install on %s/%s", org, repo) From fd17c07e6d1c2bc5364250c8b6f7b2b9aa24a0ec Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 13:01:26 +0300 Subject: [PATCH 06/22] ci(e2e): log WIF provider during behaviour mint enroll Signed-off-by: Barak Korren Co-authored-by: Cursor --- e2e/behaviour/drivers/install/perrepo_github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index 51d305f690..7a50b4e1d3 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -113,7 +113,7 @@ func (d *perRepoDriver) enrollRepoInMint(repoFullName string) error { } args := perRepoMintEnrollArgs(repoFullName, project) - d.logf("[install] mint enroll project=%s gcp_principal=%s", project, gcpCredentialPrincipal()) + d.logf("[install] mint enroll project=%s wif_provider=%s gcp_principal=%s", project, d.e2eCfg.WIFProvider, gcpCredentialPrincipal()) d.logf("[install] running fullsend %s", strings.Join(args, " ")) if _, err := admin.TryRunCLI(d.binary, d.token, args...); err != nil { return fmt.Errorf("mint enroll %s: %w", repoFullName, err) From be8413967490dc7119370381e06cf84f2153a49c Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 13:09:30 +0300 Subject: [PATCH 07/22] fix(mint): discover Gen2 mint via Cloud Run for WIF CI mint enroll used the Cloud Functions API for discovery, but e2e behaviour CI authenticates as a WIF external account with run.admin only. Fall back to Cloud Run service discovery when cloudfunctions.functions.get is denied. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/guides/dev/e2e-testing.md | 4 +- internal/dispatch/gcf/fakeclient.go | 10 +++ internal/dispatch/gcf/gcp.go | 31 +++++++ internal/dispatch/gcf/provisioner.go | 105 ++++++++++++++-------- internal/dispatch/gcf/provisioner_test.go | 29 ++++-- 5 files changed, 135 insertions(+), 44 deletions(-) diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md index 76ec8a4bf0..acb6d9c479 100644 --- a/docs/guides/dev/e2e-testing.md +++ b/docs/guides/dev/e2e-testing.md @@ -66,7 +66,9 @@ Mint URL uses the hosted public endpoint by default (same as `fullsend admin --m Behaviour tests install fullsend in **per-repo** mode (`fullsend github setup`) and run `fullsend mint enroll /test-repo` so vendored reusable workflows can mint same-org `triage` tokens (`PER_REPO_WIF_REPOS`). This is the first CI workflow that exercises per-repo mint enrollment; admin e2e only needs org-level cross-org `e2e` mint auth. -The CI service account (`E2E_GCP_SERVICE_ACCOUNT`, impersonated via `E2E_GCP_WIF_PROVIDER`) must have these roles on the **mint GCP project** — not necessarily the same project as `E2E_GCP_PROJECT_ID` (inference). Behaviour install resolves the mint project via `E2E_GCP_MINT_PROJECT_ID`, or defaults to `it-gcp-konflux-dev-fullsend` when using the hosted mint URL: +The CI service account (`E2E_GCP_SERVICE_ACCOUNT`, impersonated via `E2E_GCP_WIF_PROVIDER`) must have these roles on the **mint GCP project** — not necessarily the same project as `E2E_GCP_PROJECT_ID` (inference). Behaviour install resolves the mint project via `E2E_GCP_MINT_PROJECT_ID`, or defaults to `it-gcp-konflux-dev-fullsend` when using the hosted mint URL. + +In practice the e2e workflow often authenticates as a **WIF external account** (not a service-account key). Grant the roles below to the value of `E2E_GCP_SERVICE_ACCOUNT` *and/or* the WIF `principalSet` for `fullsend-ai/fullsend` on the pool named in `E2E_GCP_WIF_PROVIDER` (commonly `fullsend-pool/providers/github-oidc`). Gen2 mint discovery uses the Cloud Run API (`run.services.get`), not Cloud Functions — `roles/run.admin` is required; `roles/cloudfunctions.viewer` alone is insufficient for WIF principals. | IAM role | Purpose | |----------|---------| diff --git a/internal/dispatch/gcf/fakeclient.go b/internal/dispatch/gcf/fakeclient.go index b7c6a83a61..a9905a9365 100644 --- a/internal/dispatch/gcf/fakeclient.go +++ b/internal/dispatch/gcf/fakeclient.go @@ -181,6 +181,16 @@ func (f *fakeGCFClient) GetFunction(_ context.Context, _, _, _ string) (*Functio } return f.functionInfo, nil } +func (f *fakeGCFClient) GetCloudRunServiceURI(_ context.Context, _, _, _ string) (string, error) { + f.calls = append(f.calls, "GetCloudRunServiceURI") + if err := f.errs["GetCloudRunServiceURI"]; err != nil { + return "", err + } + if f.functionInfo != nil && f.functionInfo.URI != "" { + return f.functionInfo.URI, nil + } + return f.functionURL, nil +} func (f *fakeGCFClient) UploadFunctionSource(_ context.Context, _, _ string, _ []byte) (json.RawMessage, error) { f.calls = append(f.calls, "UploadFunctionSource") if err := f.errs["UploadFunctionSource"]; err != nil { diff --git a/internal/dispatch/gcf/gcp.go b/internal/dispatch/gcf/gcp.go index ca61103ec7..a28f893326 100644 --- a/internal/dispatch/gcf/gcp.go +++ b/internal/dispatch/gcf/gcp.go @@ -124,6 +124,9 @@ type GCFClient interface { // Cloud Functions v2 GetFunction(ctx context.Context, projectID, region, functionName string) (*FunctionInfo, error) + // GetCloudRunServiceURI returns the public URI of the Cloud Run service + // backing a Gen2 Cloud Function (same name as the function). + GetCloudRunServiceURI(ctx context.Context, projectID, region, serviceName string) (string, error) UploadFunctionSource(ctx context.Context, projectID, region string, sourceZip []byte) (storageSource json.RawMessage, err error) CreateFunction(ctx context.Context, projectID, region, functionName string, cfg FunctionConfig) (string, error) UpdateFunction(ctx context.Context, projectID, region, functionName string, cfg FunctionConfig) (string, error) @@ -976,6 +979,34 @@ func (c *LiveGCFClient) GetFunction(ctx context.Context, projectID, region, func }, nil } +// GetCloudRunServiceURI returns the public URI of a Cloud Run service. +func (c *LiveGCFClient) GetCloudRunServiceURI(ctx context.Context, projectID, region, serviceName string) (string, error) { + serviceURL := fmt.Sprintf("https://run.googleapis.com/v2/projects/%s/locations/%s/services/%s", + url.PathEscape(projectID), url.PathEscape(region), url.PathEscape(serviceName)) + + resp, err := c.Client.DoRequest(ctx, http.MethodGet, serviceURL, "") + if err != nil { + return "", fmt.Errorf("getting Cloud Run service: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return "", nil + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + return "", fmt.Errorf("unexpected status %d getting Cloud Run service: %s", resp.StatusCode, gcp.ExtractErrorMessage(body)) + } + + var service struct { + URI string `json:"uri"` + } + if err := json.NewDecoder(resp.Body).Decode(&service); err != nil { + return "", fmt.Errorf("decoding Cloud Run service: %w", err) + } + return service.URI, nil +} + // UploadFunctionSource generates a signed upload URL and uploads the source zip. // Returns the storage source URI for use in CreateFunction. func (c *LiveGCFClient) UploadFunctionSource(ctx context.Context, projectID, region string, sourceZip []byte) (json.RawMessage, error) { diff --git a/internal/dispatch/gcf/provisioner.go b/internal/dispatch/gcf/provisioner.go index 171aef43a2..8a24cdf5d8 100644 --- a/internal/dispatch/gcf/provisioner.go +++ b/internal/dispatch/gcf/provisioner.go @@ -324,40 +324,81 @@ type MintDiscovery struct { PerRepoWIFRepos []string } -// DiscoverMint fetches the mint function once and returns its URL and -// ROLE_APP_IDS in a single API call. Returns ErrFunctionNotFound (wrapped) -// if the function does not exist. +// DiscoverMint fetches the mint service once and returns its URL and +// ROLE_APP_IDS. Gen2 mint is backed by Cloud Run; when the Cloud Functions +// API is unavailable (common for WIF principals with run.admin only), discovery +// falls back to the Cloud Run API. func (p *Provisioner) DiscoverMint(ctx context.Context) (*MintDiscovery, error) { - fn, err := p.gcpAPI.GetFunction(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) + fn, cfErr := p.gcpAPI.GetFunction(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) + if cfErr == nil && fn != nil && fn.URI != "" { + return mintDiscoveryFromEnvVars(fn.URI, fn.EnvVars), nil + } + if cfErr != nil && !isCloudFunctionsPermissionDenied(cfErr) { + return nil, fmt.Errorf("checking mint function: %w", cfErr) + } + + d, runErr := p.discoverMintFromCloudRun(ctx) + if runErr != nil { + if cfErr != nil { + return nil, fmt.Errorf("checking mint function: %w", cfErr) + } + return nil, runErr + } + return d, nil +} + +func isCloudFunctionsPermissionDenied(err error) bool { + return err != nil && strings.Contains(err.Error(), "cloudfunctions.functions.get") +} + +func (p *Provisioner) discoverMintFromCloudRun(ctx context.Context) (*MintDiscovery, error) { + uri, err := p.gcpAPI.GetCloudRunServiceURI(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) if err != nil { - return nil, fmt.Errorf("checking mint function: %w", err) + return nil, fmt.Errorf("checking mint Cloud Run service: %w", err) } - if fn == nil || fn.URI == "" { + if uri == "" { return nil, fmt.Errorf("%w: %s in project %s region %s", ErrFunctionNotFound, functionName, p.cfg.ProjectID, p.cfg.Region) } - result := &MintDiscovery{URL: fn.URI} - if fn.EnvVars != nil { - if raw := fn.EnvVars["ROLE_APP_IDS"]; raw != "" { - var m map[string]string - if err := json.Unmarshal([]byte(raw), &m); err != nil { - log.Printf("warning: malformed ROLE_APP_IDS in mint function: %v", err) - } else { - result.RoleAppIDs = m - } + envVars, err := p.gcpAPI.GetServiceTrafficEnvVars(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) + if err != nil { + return nil, fmt.Errorf("reading mint Cloud Run env vars: %w", err) + } + return mintDiscoveryFromEnvVars(uri, envVars), nil +} + +func mintDiscoveryFromEnvVars(uri string, envVars map[string]string) *MintDiscovery { + result := &MintDiscovery{URL: uri} + if envVars == nil { + return result + } + if raw := envVars["ROLE_APP_IDS"]; raw != "" { + var m map[string]string + if err := json.Unmarshal([]byte(raw), &m); err != nil { + log.Printf("warning: malformed ROLE_APP_IDS in mint function: %v", err) + } else { + result.RoleAppIDs = m } - if raw := fn.EnvVars["PER_REPO_WIF_REPOS"]; raw != "" { - for _, entry := range strings.Split(raw, ",") { - entry = strings.TrimSpace(entry) - if entry != "" { - result.PerRepoWIFRepos = append(result.PerRepoWIFRepos, entry) - } + } + if raw := envVars["PER_REPO_WIF_REPOS"]; raw != "" { + for _, entry := range strings.Split(raw, ",") { + entry = strings.TrimSpace(entry) + if entry != "" { + result.PerRepoWIFRepos = append(result.PerRepoWIFRepos, entry) } - sort.Strings(result.PerRepoWIFRepos) } + sort.Strings(result.PerRepoWIFRepos) } - return result, nil + return result +} + +func (p *Provisioner) resolveMintURI(ctx context.Context) (string, error) { + d, err := p.DiscoverMint(ctx) + if err != nil { + return "", err + } + return d.URL, nil } // GetFunctionURL returns the URL of the deployed mint function. @@ -393,16 +434,16 @@ func (p *Provisioner) GetExistingRoleAppIDs(ctx context.Context) (map[string]str func (p *Provisioner) EnsureOrgInMint(ctx context.Context, expectedURL string, org string) error { org = strings.ToLower(org) - fn, err := p.gcpAPI.GetFunction(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) + mintURI, err := p.resolveMintURI(ctx) if err != nil { return fmt.Errorf("getting mint function: %w", err) } - if fn == nil { + if mintURI == "" { return fmt.Errorf("mint function %q not found in project %s region %s", functionName, p.cfg.ProjectID, p.cfg.Region) } - if fn.URI != expectedURL { - return fmt.Errorf("mint URL mismatch: expected %q but function has %q", expectedURL, fn.URI) + if mintURI != expectedURL { + return fmt.Errorf("mint URL mismatch: expected %q but function has %q", expectedURL, mintURI) } trafficEnvVars, err := p.gcpAPI.GetServiceTrafficEnvVars(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) @@ -465,16 +506,6 @@ func (p *Provisioner) RegisterPerRepoWIF(ctx context.Context, repo string) error return fmt.Errorf("repo name cannot contain commas, got %q", repo) } - fn, err := p.gcpAPI.GetFunction(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) - if err != nil { - return fmt.Errorf("getting mint function: %w", err) - } - if fn == nil { - return fmt.Errorf("mint function not found") - } - - // Read env vars from the traffic-serving revision to avoid stale data - // on partial failure or historical divergence (same fix as EnsureOrgInMint). trafficEnvVars, err := p.gcpAPI.GetServiceTrafficEnvVars(ctx, p.cfg.ProjectID, p.cfg.Region, functionName) if err != nil { return fmt.Errorf("reading traffic-serving env vars: %w", err) diff --git a/internal/dispatch/gcf/provisioner_test.go b/internal/dispatch/gcf/provisioner_test.go index 4d2467eb1f..24f0356f4f 100644 --- a/internal/dispatch/gcf/provisioner_test.go +++ b/internal/dispatch/gcf/provisioner_test.go @@ -2270,7 +2270,7 @@ func TestEnsureOrgInMint_NilReturn(t *testing.T) { p := NewProvisioner(Config{ProjectID: "proj1", Region: "us-central1"}, fake) err := p.EnsureOrgInMint(context.Background(), "https://mint.example.com", "acme-corp") require.Error(t, err) - assert.Contains(t, err.Error(), "not found in project") + assert.Contains(t, err.Error(), "mint function not found") } func TestEnsureOrgInMint_LowercasesOrg(t *testing.T) { @@ -2463,14 +2463,14 @@ func TestRegisterPerRepoWIF_Idempotent(t *testing.T) { assert.NotContains(t, fake.calls, "UpdateServiceEnvVars") } -func TestRegisterPerRepoWIF_FunctionNotFound(t *testing.T) { +func TestRegisterPerRepoWIF_ServiceNotFound(t *testing.T) { fake := newFakeGCFClient() - fake.functionInfo = nil + fake.errs["GetServiceTrafficEnvVars"] = fmt.Errorf("unexpected status 404 getting Cloud Run service") p := NewProvisioner(Config{ProjectID: "proj1", Region: "us-central1"}, fake) err := p.RegisterPerRepoWIF(context.Background(), "acme-corp/my-service") require.Error(t, err) - assert.Contains(t, err.Error(), "mint function not found") + assert.Contains(t, err.Error(), "reading traffic-serving env vars") } func TestRegisterPerRepoWIF_LowercasesRepo(t *testing.T) { @@ -2520,12 +2520,12 @@ func TestRegisterPerRepoWIF_NilEnvVars(t *testing.T) { func TestRegisterPerRepoWIF_GetFunctionError(t *testing.T) { fake := newFakeGCFClient() - fake.errs["GetFunction"] = fmt.Errorf("permission denied") + fake.errs["GetServiceTrafficEnvVars"] = fmt.Errorf("permission denied") p := NewProvisioner(Config{ProjectID: "proj1", Region: "us-central1"}, fake) err := p.RegisterPerRepoWIF(context.Background(), "acme-corp/my-service") require.Error(t, err) - assert.Contains(t, err.Error(), "getting mint function") + assert.Contains(t, err.Error(), "reading traffic-serving env vars") } func TestRegisterPerRepoWIF_PartialFailureSurfacesRevision(t *testing.T) { @@ -3270,3 +3270,20 @@ func TestRemoveRoleFromMint_UpdateEnvVarsError(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "updating mint env vars") } + +func TestDiscoverMint_FallsBackToCloudRunOnCFForbidden(t *testing.T) { + fake := newFakeGCFClient() + fake.errs["GetFunction"] = fmt.Errorf("unexpected status 403 checking function: Permission 'cloudfunctions.functions.get' denied") + fake.functionInfo = &FunctionInfo{URI: "https://mint.example.com"} + fake.trafficEnvVars = map[string]string{ + "ROLE_APP_IDS": `{"triage":"123"}`, + } + + p := NewProvisioner(Config{ProjectID: "proj1", Region: "us-central1"}, fake) + d, err := p.DiscoverMint(context.Background()) + require.NoError(t, err) + assert.Equal(t, "https://mint.example.com", d.URL) + assert.Equal(t, map[string]string{"triage": "123"}, d.RoleAppIDs) + assert.Contains(t, fake.calls, "GetCloudRunServiceURI") + assert.Contains(t, fake.calls, "GetServiceTrafficEnvVars") +} From b85e28f77355d443211ef0fe1f6ba9b43a01a20e Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 13:12:22 +0300 Subject: [PATCH 08/22] fix(mint): surface Cloud Run discovery errors from DiscoverMint Signed-off-by: Barak Korren Co-authored-by: Cursor --- internal/dispatch/gcf/provisioner.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/dispatch/gcf/provisioner.go b/internal/dispatch/gcf/provisioner.go index 8a24cdf5d8..0044ac040b 100644 --- a/internal/dispatch/gcf/provisioner.go +++ b/internal/dispatch/gcf/provisioner.go @@ -339,9 +339,6 @@ func (p *Provisioner) DiscoverMint(ctx context.Context) (*MintDiscovery, error) d, runErr := p.discoverMintFromCloudRun(ctx) if runErr != nil { - if cfErr != nil { - return nil, fmt.Errorf("checking mint function: %w", cfErr) - } return nil, runErr } return d, nil From 200b7e46fdcd30d3cff75ed0190a2bca2acd2251 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 15:04:18 +0300 Subject: [PATCH 09/22] ci(e2e): drop CI mint enroll from behaviour install driver Pool halfsend-*/test-repo repos are enrolled once on the hosted mint project by a GCP admin. Inference and mint live in different projects, so CI no longer needs mint-admin IAM during install. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/guides/dev/behaviour-drivers.md | 2 +- docs/guides/dev/behaviour-testing.md | 3 +- docs/guides/dev/e2e-testing.md | 32 ++------- .../infrastructure/mint-administration.md | 2 +- .../drivers/install/perrepo_github.go | 66 +------------------ .../drivers/install/perrepo_github_test.go | 9 --- 6 files changed, 11 insertions(+), 103 deletions(-) diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md index 86778677b9..9e288a5e65 100644 --- a/docs/guides/dev/behaviour-drivers.md +++ b/docs/guides/dev/behaviour-drivers.md @@ -31,7 +31,7 @@ The suite in `e2e/behaviour/suite_test.go` acquires a pool org, runs pre-install ### Install driver (v1 per-repo) -Uses `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with inference flags from `E2E_GCP_PROJECT_ID` and `E2E_GCP_WIF_PROVIDER`. When `E2E_GCP_PROJECT_ID` is set (CI), the driver then runs `fullsend mint enroll /test-repo` so vendored workflows can mint same-org triage tokens. Pool orgs must already have shared GitHub Apps and org-level mint enrollment; the driver does not run `fullsend admin install`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment) for CI service-account IAM on the mint project. +Uses `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with inference flags from `E2E_GCP_PROJECT_ID` and `E2E_GCP_WIF_PROVIDER`. Pool orgs must already have shared GitHub Apps, org-level mint enrollment, and per-repo mint enrollment for `test-repo` (one-time GCP admin step). The driver does not run `fullsend admin install` or `fullsend mint enroll`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). Teardown removes shim workflows, stale branches, and open fullsend PRs on `test-repo` via shared helpers in `e2e/admin/cleanup.go`. diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index 1716080ee3..320181c578 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -68,7 +68,7 @@ Use tags only for **exceptions** when a backend cannot run a scenario yet: `@ski make behaviour-test ``` -In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver runs `fullsend github setup` then `fullsend mint enroll /test-repo` when `E2E_GCP_PROJECT_ID` is set (CI). The CI service account needs mint-admin IAM on that project — see [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). +In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`) on the hosted mint project. Pool `test-repo` repos are enrolled once by a GCP admin — not during CI install. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). Runner env (defaults shown): @@ -77,7 +77,6 @@ BEHAVIOUR_SCM=github BEHAVIOUR_CI=githubactions BEHAVIOUR_INSTALL_MODE=per-repo E2E_GCP_PROJECT_ID=... # inference project for per-repo WIF secrets -E2E_GCP_MINT_PROJECT_ID=... # optional; defaults to hosted mint project when using public mint URL E2E_GCP_WIF_PROVIDER=... # required in CI for per-repo WIF secret ``` diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md index acb6d9c479..aab6f0abc3 100644 --- a/docs/guides/dev/e2e-testing.md +++ b/docs/guides/dev/e2e-testing.md @@ -36,7 +36,7 @@ Optional environment variables: | `GH_TOKEN` / `GITHUB_TOKEN` | Override token source for local runs | | `FULLSEND_MINT_URL` | Override mint endpoint (default: hosted public mint, same as `fullsend admin --mint-url`) | | `E2E_LOCK_TIMEOUT` | Max wait for a free pool org (default 10m) | -| `E2E_GCP_PROJECT_ID` | GCP project for inference setup and behaviour per-repo mint enroll (must be the mint project when using the hosted mint) | +| `E2E_GCP_PROJECT_ID` | GCP project for inference setup (`github setup --inference-project`) | Behaviour tests use the same pool orgs but install via `fullsend github setup` (per-repo) instead of `fullsend admin install`. See [behaviour-testing.md](behaviour-testing.md) and [behaviour-drivers.md](behaviour-drivers.md). @@ -58,43 +58,25 @@ Required repository secrets: | `E2E_GCP_WIF_PROVIDER` | GCP WIF provider (inference / auxiliary GCP access) | | `E2E_GCP_SERVICE_ACCOUNT` | GCP service account for WIF | | `E2E_GCP_PROJECT_ID` | GCP project ID for inference secrets (`github setup --inference-project`) | -| `E2E_GCP_MINT_PROJECT_ID` | Optional override for `mint enroll --project` (default: hosted mint project when using public mint URL) | Mint URL uses the hosted public endpoint by default (same as `fullsend admin --mint-url`). Override with org/repo variable `FULLSEND_MINT_URL` if needed; no separate e2e secret. ### Behaviour tests and per-repo mint enrollment -Behaviour tests install fullsend in **per-repo** mode (`fullsend github setup`) and run `fullsend mint enroll /test-repo` so vendored reusable workflows can mint same-org `triage` tokens (`PER_REPO_WIF_REPOS`). This is the first CI workflow that exercises per-repo mint enrollment; admin e2e only needs org-level cross-org `e2e` mint auth. +Behaviour tests install fullsend in **per-repo** mode (`fullsend github setup`). Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; that requires per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver does **not** run `mint enroll` — pool org `test-repo` repos must be enrolled once by a GCP admin on the hosted mint project. -The CI service account (`E2E_GCP_SERVICE_ACCOUNT`, impersonated via `E2E_GCP_WIF_PROVIDER`) must have these roles on the **mint GCP project** — not necessarily the same project as `E2E_GCP_PROJECT_ID` (inference). Behaviour install resolves the mint project via `E2E_GCP_MINT_PROJECT_ID`, or defaults to `it-gcp-konflux-dev-fullsend` when using the hosted mint URL. +Inference (`E2E_GCP_PROJECT_ID`) and mint (`it-gcp-konflux-dev-fullsend` for the hosted mint) may be different GCP projects. CI only needs inference WIF secrets for `github setup`; mint-admin IAM on the CI service account is not required for behaviour tests. -In practice the e2e workflow often authenticates as a **WIF external account** (not a service-account key). Grant the roles below to the value of `E2E_GCP_SERVICE_ACCOUNT` *and/or* the WIF `principalSet` for `fullsend-ai/fullsend` on the pool named in `E2E_GCP_WIF_PROVIDER` (commonly `fullsend-pool/providers/github-oidc`). Gen2 mint discovery uses the Cloud Run API (`run.services.get`), not Cloud Functions — `roles/run.admin` is required; `roles/cloudfunctions.viewer` alone is insufficient for WIF principals. - -| IAM role | Purpose | -|----------|---------| -| `roles/cloudfunctions.viewer` | Discover mint (`fullsend-mint`) during `mint enroll` | -| `roles/run.admin` | Update Cloud Run env vars (`ALLOWED_ORGS`, `PER_REPO_WIF_REPOS`) | -| `roles/iam.workloadIdentityPoolAdmin` | Create/update repo-scoped WIF providers | -| `roles/resourcemanager.projectIamAdmin` | Grant `roles/aiplatform.user` to per-repo WIF principals | - -Hosted mint example (project `it-gcp-konflux-dev-fullsend`). Grant roles on the **same** service account stored in the `E2E_GCP_SERVICE_ACCOUNT` repository secret (commonly `github-fullsend-ai-fullsen-966@…` or `github-fullsend-ai-fullsend-ci@…`): +One-time enrollment for all pool orgs (idempotent): ```bash export GCP_PROJECT=it-gcp-konflux-dev-fullsend -export E2E_SA="" - -for ROLE in \ - roles/cloudfunctions.viewer \ - roles/run.admin \ - roles/iam.workloadIdentityPoolAdmin \ - roles/resourcemanager.projectIamAdmin; do - gcloud projects add-iam-policy-binding "$GCP_PROJECT" \ - --member="serviceAccount:${E2E_SA}" \ - --role="$ROLE" +for i in $(seq -w 1 12); do + fullsend mint enroll "halfsend-${i}/test-repo" --project="$GCP_PROJECT" --region=us-central1 done ``` -`mint enroll` is idempotent — behaviour install can re-run safely when a pool org's `test-repo` is already enrolled. +Re-run enrollment when adding a new pool org or after mint infrastructure changes that drop `PER_REPO_WIF_REPOS` entries. See [mint-administration.md](../infrastructure/mint-administration.md) for required operator IAM. ## Pool org provisioning diff --git a/docs/guides/infrastructure/mint-administration.md b/docs/guides/infrastructure/mint-administration.md index 57a87aa22a..e9c5b86d6d 100644 --- a/docs/guides/infrastructure/mint-administration.md +++ b/docs/guides/infrastructure/mint-administration.md @@ -72,7 +72,7 @@ Pass this URL as `--mint-url` when running `fullsend github setup`, or set the ` `roles/owner` covers all of the above for users with broad access. - **Behaviour / e2e CI:** When CI runs `fullsend mint enroll owner/repo` (per-repo behaviour tests), the GitHub Actions service account impersonated via WIF needs the per-repo `mint enroll` roles on the mint project — not just inference roles. See [e2e-testing.md](../dev/e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). + **Behaviour / e2e pool orgs:** Per-repo behaviour tests need `halfsend-NN/test-repo` enrolled on the hosted mint (`PER_REPO_WIF_REPOS`). Run `fullsend mint enroll owner/repo` once per pool org — not from CI. See [e2e-testing.md](../dev/e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). An administrator can grant all required roles with a single script: diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index 7a50b4e1d3..302209a391 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -4,9 +4,7 @@ package install import ( "context" - "encoding/json" "fmt" - "os" "path/filepath" "strings" @@ -18,8 +16,7 @@ import ( ) const ( - perRepoTestRepo = "test-repo" - defaultMintRegion = "us-central1" + perRepoTestRepo = "test-repo" // Vendored per-repo: issue_comment triggers the fullsend.yaml shim, which // workflow_calls reusable-dispatch → reusable-triage synchronously. perRepoTriageWorkflow = "fullsend.yaml" @@ -84,10 +81,6 @@ func (d *perRepoDriver) Install(ctx context.Context, org string) (State, error) return nil, fmt.Errorf("github setup %s: %w", target, err) } - if err := d.enrollRepoInMint(target); err != nil { - return nil, err - } - st := &perRepoState{org: org, repo: repo} if err := validatePerRepoPostInstall(ctx, d.client, org, repo); err != nil { return nil, err @@ -95,63 +88,6 @@ func (d *perRepoDriver) Install(ctx context.Context, org string) (State, error) return st, nil } -// perRepoMintEnrollArgs returns CLI args for enrolling a vendored per-repo install -// in mint (PER_REPO_WIF_REPOS + repo-scoped WIF provider). -func perRepoMintEnrollArgs(repoFullName, gcpProject string) []string { - return []string{ - "mint", "enroll", repoFullName, - "--project", gcpProject, - "--region", defaultMintRegion, - } -} - -func (d *perRepoDriver) enrollRepoInMint(repoFullName string) error { - project := admin.MintEnrollProjectID(d.e2eCfg) - if project == "" { - d.logf("[install] skipping mint enroll for %s (no mint GCP project; set E2E_GCP_MINT_PROJECT_ID or E2E_GCP_PROJECT_ID, or use hosted mint URL)", repoFullName) - return nil - } - - args := perRepoMintEnrollArgs(repoFullName, project) - d.logf("[install] mint enroll project=%s wif_provider=%s gcp_principal=%s", project, d.e2eCfg.WIFProvider, gcpCredentialPrincipal()) - d.logf("[install] running fullsend %s", strings.Join(args, " ")) - if _, err := admin.TryRunCLI(d.binary, d.token, args...); err != nil { - return fmt.Errorf("mint enroll %s: %w", repoFullName, err) - } - return nil -} - -// gcpCredentialPrincipal returns a short label for the active GCP credential -// (service account email or external account) to aid CI debugging. -func gcpCredentialPrincipal() string { - path := strings.TrimSpace(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")) - if path == "" { - return "unknown" - } - data, err := os.ReadFile(filepath.Clean(path)) - if err != nil { - return "unreadable" - } - var creds struct { - ClientEmail string `json:"client_email"` - Type string `json:"type"` - ServiceURL string `json:"service_account_impersonation_url"` - } - if err := json.Unmarshal(data, &creds); err != nil { - return "unparsed" - } - if creds.ClientEmail != "" { - return creds.ClientEmail - } - if creds.Type != "" { - return creds.Type - } - if creds.ServiceURL != "" { - return creds.ServiceURL - } - return "unknown" -} - func (d *perRepoDriver) Teardown(ctx context.Context, org string, state State) error { repo := state.TestRepo() d.logf("[install] tearing down per-repo install on %s/%s", org, repo) diff --git a/e2e/behaviour/drivers/install/perrepo_github_test.go b/e2e/behaviour/drivers/install/perrepo_github_test.go index bce807249e..e50b97e15d 100644 --- a/e2e/behaviour/drivers/install/perrepo_github_test.go +++ b/e2e/behaviour/drivers/install/perrepo_github_test.go @@ -40,15 +40,6 @@ func TestValidatePerRepoPostInstall_MissingShim(t *testing.T) { assert.Contains(t, err.Error(), "fullsend.yaml") } -func TestPerRepoMintEnrollArgs(t *testing.T) { - args := perRepoMintEnrollArgs("halfsend-05/test-repo", "my-mint-project") - assert.Equal(t, []string{ - "mint", "enroll", "halfsend-05/test-repo", - "--project", "my-mint-project", - "--region", "us-central1", - }, args) -} - func TestValidatePerRepoPostInstall_WrongRuntime(t *testing.T) { client := forge.NewFakeClient() org, repo := "acme", "test-repo" From 34d53a897d5dabd489afda5a84a404a9dfe920b6 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Thu, 2 Jul 2026 16:36:46 +0300 Subject: [PATCH 10/22] ci(e2e): provision repo-scoped inference WIF in behaviour install Run fullsend inference provision before github setup so pool test-repo workflows get a WIF provider scoped to halfsend-NN/test-repo instead of the CI-wide E2E_GCP_WIF_PROVIDER. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/guides/dev/behaviour-drivers.md | 2 +- docs/guides/dev/behaviour-testing.md | 6 +- docs/guides/dev/e2e-testing.md | 9 ++- .../drivers/install/perrepo_github.go | 55 +++++++++++++++++-- .../drivers/install/perrepo_github_test.go | 23 ++++++++ 5 files changed, 86 insertions(+), 9 deletions(-) diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md index 9e288a5e65..619d6cb3a2 100644 --- a/docs/guides/dev/behaviour-drivers.md +++ b/docs/guides/dev/behaviour-drivers.md @@ -31,7 +31,7 @@ The suite in `e2e/behaviour/suite_test.go` acquires a pool org, runs pre-install ### Install driver (v1 per-repo) -Uses `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with inference flags from `E2E_GCP_PROJECT_ID` and `E2E_GCP_WIF_PROVIDER`. Pool orgs must already have shared GitHub Apps, org-level mint enrollment, and per-repo mint enrollment for `test-repo` (one-time GCP admin step). The driver does not run `fullsend admin install` or `fullsend mint enroll`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). +Uses `fullsend inference provision /test-repo` then `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with the repo-scoped WIF provider from provision (`E2E_GCP_PROJECT_ID`). Pool orgs must already have shared GitHub Apps, org-level mint enrollment, and per-repo mint enrollment for `test-repo` (one-time GCP admin step on the hosted mint project). The driver does not run `fullsend admin install` or `fullsend mint enroll`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). Teardown removes shim workflows, stale branches, and open fullsend PRs on `test-repo` via shared helpers in `e2e/admin/cleanup.go`. diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index 320181c578..7909063ddc 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -68,7 +68,7 @@ Use tags only for **exceptions** when a backend cannot run a scenario yet: `@ski make behaviour-test ``` -In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`) on the hosted mint project. Pool `test-repo` repos are enrolled once by a GCP admin — not during CI install. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). +In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`) on the hosted mint project. Pool `test-repo` repos are enrolled once by a GCP admin — not during CI install. The install driver provisions repo-scoped inference WIF via `fullsend inference provision` before `github setup`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). Runner env (defaults shown): @@ -76,8 +76,8 @@ Runner env (defaults shown): BEHAVIOUR_SCM=github BEHAVIOUR_CI=githubactions BEHAVIOUR_INSTALL_MODE=per-repo -E2E_GCP_PROJECT_ID=... # inference project for per-repo WIF secrets -E2E_GCP_WIF_PROVIDER=... # required in CI for per-repo WIF secret +E2E_GCP_PROJECT_ID=... # inference project; install runs inference provision per pool repo +E2E_GCP_WIF_PROVIDER=... # CI job GCP auth (not written to pool test-repo secrets) ``` Triage scenarios apply the `ready-for-triage` label (not `/fs-triage` comments) because the per-repo shim ignores `issue_comment` events from bot users and CI uses minted e2e installation tokens. diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md index aab6f0abc3..31be41df93 100644 --- a/docs/guides/dev/e2e-testing.md +++ b/docs/guides/dev/e2e-testing.md @@ -65,7 +65,14 @@ Mint URL uses the hosted public endpoint by default (same as `fullsend admin --m Behaviour tests install fullsend in **per-repo** mode (`fullsend github setup`). Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; that requires per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver does **not** run `mint enroll` — pool org `test-repo` repos must be enrolled once by a GCP admin on the hosted mint project. -Inference (`E2E_GCP_PROJECT_ID`) and mint (`it-gcp-konflux-dev-fullsend` for the hosted mint) may be different GCP projects. CI only needs inference WIF secrets for `github setup`; mint-admin IAM on the CI service account is not required for behaviour tests. +Inference (`E2E_GCP_PROJECT_ID`) and mint (`it-gcp-konflux-dev-fullsend` for the hosted mint) may be different GCP projects. The behaviour install driver runs `fullsend inference provision /test-repo` using CI credentials on the inference project (same access model as admin e2e), then passes the repo-scoped WIF provider to `github setup`. `E2E_GCP_WIF_PROVIDER` authenticates the CI job itself; it is not written to pool org repos. + +The CI service account needs inference-provision IAM on `E2E_GCP_PROJECT_ID`: + +| IAM role | Purpose | +|----------|---------| +| `roles/iam.workloadIdentityPoolAdmin` | Create/update repo-scoped inference WIF providers | +| `roles/resourcemanager.projectIamAdmin` | Grant `roles/aiplatform.user` to repo WIF principals | One-time enrollment for all pool orgs (idempotent): diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index 302209a391..a493b0fb22 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -4,6 +4,7 @@ package install import ( "context" + "encoding/json" "fmt" "path/filepath" "strings" @@ -70,10 +71,11 @@ func (d *perRepoDriver) Install(ctx context.Context, org string) (State, error) "--runtime", "dummy", } if project := strings.TrimSpace(d.e2eCfg.GCPProjectID); project != "" { - args = append(args, "--inference-project", project) - } - if wif := strings.TrimSpace(d.e2eCfg.WIFProvider); wif != "" { - args = append(args, "--inference-wif-provider", wif) + wifProvider, err := d.provisionPerRepoInference(target, project) + if err != nil { + return nil, err + } + args = append(args, "--inference-project", project, "--inference-wif-provider", wifProvider) } d.logf("[install] running fullsend %s", strings.Join(args, " ")) @@ -88,6 +90,51 @@ func (d *perRepoDriver) Install(ctx context.Context, org string) (State, error) return st, nil } +// provisionPerRepoInference creates repo-scoped inference WIF for target and +// returns the provider resource name for github setup. Idempotent. +func (d *perRepoDriver) provisionPerRepoInference(target, project string) (string, error) { + provisionArgs := []string{"inference", "provision", target, "--project", project} + d.logf("[install] running fullsend %s", strings.Join(provisionArgs, " ")) + if _, err := admin.TryRunCLI(d.binary, d.token, provisionArgs...); err != nil { + return "", fmt.Errorf("inference provision %s: %w", target, err) + } + + statusArgs := []string{"inference", "status", target, "--project", project, "--format", "json"} + d.logf("[install] running fullsend %s", strings.Join(statusArgs, " ")) + out, err := admin.TryRunCLI(d.binary, d.token, statusArgs...) + if err != nil { + return "", fmt.Errorf("inference status %s: %w", target, err) + } + + wifProvider, err := parseInferenceStatusWIFProvider(out) + if err != nil { + return "", fmt.Errorf("inference status %s: %w", target, err) + } + d.logf("[install] repo-scoped inference WIF provider: %s", wifProvider) + return wifProvider, nil +} + +func parseInferenceStatusWIFProvider(output string) (string, error) { + start := strings.Index(output, "{") + if start < 0 { + return "", fmt.Errorf("no JSON in output") + } + var status struct { + Status string `json:"status"` + WIFProvider string `json:"FULLSEND_GCP_WIF_PROVIDER"` + } + if err := json.Unmarshal([]byte(output[start:]), &status); err != nil { + return "", fmt.Errorf("parse JSON: %w", err) + } + if status.WIFProvider == "" { + return "", fmt.Errorf("missing FULLSEND_GCP_WIF_PROVIDER (status=%q)", status.Status) + } + if status.Status != "healthy" { + return "", fmt.Errorf("expected healthy status, got %q", status.Status) + } + return status.WIFProvider, nil +} + func (d *perRepoDriver) Teardown(ctx context.Context, org string, state State) error { repo := state.TestRepo() d.logf("[install] tearing down per-repo install on %s/%s", org, repo) diff --git a/e2e/behaviour/drivers/install/perrepo_github_test.go b/e2e/behaviour/drivers/install/perrepo_github_test.go index e50b97e15d..ea0c4b97a6 100644 --- a/e2e/behaviour/drivers/install/perrepo_github_test.go +++ b/e2e/behaviour/drivers/install/perrepo_github_test.go @@ -40,6 +40,29 @@ func TestValidatePerRepoPostInstall_MissingShim(t *testing.T) { assert.Contains(t, err.Error(), "fullsend.yaml") } +func TestParseInferenceStatusWIFProvider_OK(t *testing.T) { + out := `{ + "status": "healthy", + "FULLSEND_GCP_PROJECT_ID": "my-project", + "FULLSEND_GCP_WIF_PROVIDER": "projects/123/locations/global/workloadIdentityPools/fullsend-inference/providers/gh-halfsend-01-test-repo" +}` + got, err := parseInferenceStatusWIFProvider(out) + require.NoError(t, err) + assert.Equal(t, "projects/123/locations/global/workloadIdentityPools/fullsend-inference/providers/gh-halfsend-01-test-repo", got) +} + +func TestParseInferenceStatusWIFProvider_NoJSON(t *testing.T) { + _, err := parseInferenceStatusWIFProvider("no json here") + require.Error(t, err) + assert.Contains(t, err.Error(), "no JSON") +} + +func TestParseInferenceStatusWIFProvider_Unhealthy(t *testing.T) { + _, err := parseInferenceStatusWIFProvider(`{"status":"unhealthy","FULLSEND_GCP_WIF_PROVIDER":"projects/1/locations/global/workloadIdentityPools/p/providers/x"}`) + require.Error(t, err) + assert.Contains(t, err.Error(), "healthy") +} + func TestValidatePerRepoPostInstall_WrongRuntime(t *testing.T) { client := forge.NewFakeClient() org, repo := "acme", "test-repo" From 591934a8836bc3aba2e44af6374509ff2f7e1553 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Fri, 3 Jul 2026 00:16:33 +0300 Subject: [PATCH 11/22] ci(e2e): use dummy runtime config path in per-repo triage Pass fullsend-dir=.fullsend for per-repo reusable-triage so fullsend run loads .fullsend/config.yaml (runtime: dummy) and writes behaviour-results.json. Fall back to .fullsend/config.yaml in run when the primary config path is missing, and download artifacts from the nested triage agent workflow run. Signed-off-by: Barak Korren Co-authored-by: Cursor --- .github/workflows/reusable-triage.yml | 1 + e2e/behaviour/steps/artifacts.go | 45 ++++++++++++++++++++++----- internal/cli/run.go | 4 +++ internal/cli/run_test.go | 16 ++++++++++ 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable-triage.yml b/.github/workflows/reusable-triage.yml index 23ea79f241..17718b1856 100644 --- a/.github/workflows/reusable-triage.yml +++ b/.github/workflows/reusable-triage.yml @@ -161,6 +161,7 @@ jobs: with: agent: triage version: ${{ inputs.fullsend_version }} + fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} status-repo: ${{ inputs.source_repo }} status-number: ${{ fromJSON(inputs.event_payload).issue.number }} diff --git a/e2e/behaviour/steps/artifacts.go b/e2e/behaviour/steps/artifacts.go index d7980fb4e2..b08ed057b5 100644 --- a/e2e/behaviour/steps/artifacts.go +++ b/e2e/behaviour/steps/artifacts.go @@ -47,19 +47,44 @@ func ensureArtifacts(w *world.World) error { if err != nil { return err } - if w.WorkflowRun != nil { - if err := w.CI.DownloadArtifacts(ctx, w.Org, w.Install.TriageWorkflowRepo(), w.WorkflowRun.ID, dest); err == nil { - if _, findErr := artifacts.FindBehaviourResults(dest); findErr == nil { - w.ArtifactDir = dest - return nil - } + + tryDownloadRun := func(runID int) error { + if err := w.CI.DownloadArtifacts(ctx, w.Org, w.Install.TriageWorkflowRepo(), runID, dest); err != nil { + return err } + if _, findErr := artifacts.FindBehaviourResults(dest); findErr != nil { + return findErr + } + return nil + } + + resetDest := func() error { _ = os.RemoveAll(dest) dest, err = prepareArtifactDir() - if err != nil { + return err + } + + if w.WorkflowRun != nil { + if err := tryDownloadRun(w.WorkflowRun.ID); err == nil { + w.ArtifactDir = dest + return nil + } + if err := resetDest(); err != nil { + return err + } + } + + // Reusable triage uploads artifacts on the nested agent workflow run, not the shim. + if agentRun, err := w.CI.FindCompletedWorkflowRun(ctx, w.Org, w.Install.TriageWorkflowRepo(), w.Install.AgentWorkflowFile(), w.ScenarioStart); err == nil && agentRun != nil { + if err := tryDownloadRun(agentRun.ID); err == nil { + w.ArtifactDir = dest + return nil + } + if err := resetDest(); err != nil { return err } } + if err := w.CI.DownloadNamedArtifactAfter(ctx, w.Org, w.Install.TriageWorkflowRepo(), w.Install.AgentArtifactName(), w.ScenarioStart, dest); err != nil { _ = os.RemoveAll(dest) return err @@ -78,7 +103,11 @@ func prepareArtifactDir() (string, error) { if err := os.MkdirAll(ciArtifactDir, 0o755); err != nil { return "", fmt.Errorf("creating behaviour artifact dir: %w", err) } - return ciArtifactDir, nil + sub, err := os.MkdirTemp(ciArtifactDir, "run-*") + if err != nil { + return "", fmt.Errorf("creating behaviour artifact subdir: %w", err) + } + return sub, nil } return os.MkdirTemp("", "behaviour-artifacts-*") } diff --git a/internal/cli/run.go b/internal/cli/run.go index f6dccb5fb7..196ba45e52 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -154,6 +154,10 @@ func isOrgConfigData(data []byte) bool { func backendFromConfigFile(path string) (agentruntime.Backend, error) { data, readErr := os.ReadFile(path) + if readErr != nil && os.IsNotExist(readErr) { + alt := filepath.Join(filepath.Dir(path), ".fullsend", "config.yaml") + data, readErr = os.ReadFile(alt) + } if readErr == nil { backend, resolveErr := resolveBackendFromConfigData(data) if resolveErr != nil { diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index e38fb5d3a3..988be4f886 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3056,6 +3056,22 @@ func TestBackendFromConfigFile_PerRepoConfig(t *testing.T) { assert.Equal(t, "dummy", backend.Runtime.Name()) } +func TestBackendFromConfigFile_PerRepoNestedConfig(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + cfg := config.NewPerRepoConfig(config.PerRepoDefaultRoles(), "acme/test-repo") + cfg.Runtime = "dummy" + data, err := cfg.Marshal() + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".fullsend"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, ".fullsend", "config.yaml"), data, 0o644)) + + backend, err := backendFromConfigFile(filepath.Join(dir, "config.yaml")) + require.NoError(t, err) + assert.Equal(t, "dummy", backend.Runtime.Name()) +} + func TestBackendFromConfigFile_ReadError(t *testing.T) { t.Parallel() From 14fcc692168a66b8ef2d8c934a64138f0740e888 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Mon, 6 Jul 2026 08:45:03 +0300 Subject: [PATCH 12/22] ci(e2e): fix ADR 0065 numbering and gofmt after main merge Renumber behaviour-tests ADR to 0065 to avoid collision with 0063-polling-based-work-discovery from main. Format internal/forge/fake.go. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/ADRs/0003-org-config-repo-convention.md | 2 +- ...rs.md => 0065-behaviour-tests-with-gherkin-and-drivers.md} | 4 ++-- docs/architecture.md | 2 +- docs/guides/dev/behaviour-testing.md | 2 +- docs/problems/testing-agents.md | 2 +- internal/forge/fake.go | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename docs/ADRs/{0063-behaviour-tests-with-gherkin-and-drivers.md => 0065-behaviour-tests-with-gherkin-and-drivers.md} (95%) diff --git a/docs/ADRs/0003-org-config-repo-convention.md b/docs/ADRs/0003-org-config-repo-convention.md index 526fd27b7e..4772699c5b 100644 --- a/docs/ADRs/0003-org-config-repo-convention.md +++ b/docs/ADRs/0003-org-config-repo-convention.md @@ -212,7 +212,7 @@ version: 1 # Schema version for future evolution intent_repo: /features # or /intent architecture_repo: /architecture -# Agent runtime defaults (see ADR 0063 for behaviour-test dummy runtime) +# Agent runtime defaults (see ADR 0065 for behaviour-test dummy runtime) defaults: runtime: claude # claude or dummy (dummy: behaviour test orgs only) diff --git a/docs/ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md b/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md similarity index 95% rename from docs/ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md rename to docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md index 991fc97a91..d3a778e8b3 100644 --- a/docs/ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md +++ b/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md @@ -1,5 +1,5 @@ --- -title: "63. Behaviour tests with Gherkin and pluggable drivers" +title: "65. Behaviour tests with Gherkin and pluggable drivers" status: Accepted relates_to: - agent-infrastructure @@ -10,7 +10,7 @@ topics: - behaviour-tests --- -# 63. Behaviour tests with Gherkin and pluggable drivers +# 65. Behaviour tests with Gherkin and pluggable drivers Date: 2026-06-07 diff --git a/docs/architecture.md b/docs/architecture.md index f68332af48..8d5bbce2d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -131,7 +131,7 @@ This is the thing that actually reasons and acts. Everything else in this docume ### Behaviour testing -End-to-end **behaviour tests** under `e2e/behaviour/` validate deterministic platform code — dispatch routing, harness loading, sandbox policy, SCM mutations — with the LLM layer removed via the dummy runtime. Tests exercise real GitHub and GitHub Actions through pluggable SCM and CI drivers; Gherkin scenarios stay install-mode agnostic while runner env vars select backends. This coverage is **orthogonal** to LLM and instruction testing in [testing-agents.md](problems/testing-agents.md). See [ADR 0063](ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md). +End-to-end **behaviour tests** under `e2e/behaviour/` validate deterministic platform code — dispatch routing, harness loading, sandbox policy, SCM mutations — with the LLM layer removed via the dummy runtime. Tests exercise real GitHub and GitHub Actions through pluggable SCM and CI drivers; Gherkin scenarios stay install-mode agnostic while runner env vars select backends. This coverage is **orthogonal** to LLM and instruction testing in [testing-agents.md](problems/testing-agents.md). See [ADR 0065](ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md). **Open questions:** diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index 7909063ddc..bd7ae6fafe 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -82,4 +82,4 @@ E2E_GCP_WIF_PROVIDER=... # CI job GCP auth (not written to pool test-repo s Triage scenarios apply the `ready-for-triage` label (not `/fs-triage` comments) because the per-repo shim ignores `issue_comment` events from bot users and CI uses minted e2e installation tokens. -See [behaviour-drivers.md](behaviour-drivers.md) for driver configuration and [ADR 0063](../../ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md) for the decision record. +See [behaviour-drivers.md](behaviour-drivers.md) for driver configuration and [ADR 0065](../../ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md) for the decision record. diff --git a/docs/problems/testing-agents.md b/docs/problems/testing-agents.md index ff5f7615d0..73f50c4c89 100644 --- a/docs/problems/testing-agents.md +++ b/docs/problems/testing-agents.md @@ -8,7 +8,7 @@ Testing application code is a solved problem with mature tooling: unit tests, in Today, if someone modifies a review agent's instructions, the only verification is human review of the prose change. There is no automated way to confirm the agent still behaves correctly after the modification. This is the equivalent of shipping code changes with no test suite — something we would never accept for application code. -**Behaviour tests are orthogonal.** [ADR 0063](../ADRs/0063-behaviour-tests-with-gherkin-and-drivers.md) describes Gherkin end-to-end tests under `e2e/behaviour/` that validate deterministic platform code (workflows, harness, sandbox policy, post-scripts) with inference explicitly removed via the dummy runtime. They do not evaluate instructions, prompts, or models, and they do not replace golden-set or statistical LLM evals described below. +**Behaviour tests are orthogonal.** [ADR 0065](../ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md) describes Gherkin end-to-end tests under `e2e/behaviour/` that validate deterministic platform code (workflows, harness, sandbox policy, post-scripts) with inference explicitly removed via the dummy runtime. They do not evaluate instructions, prompts, or models, and they do not replace golden-set or statistical LLM evals described below. ## What makes agent testing hard diff --git a/internal/forge/fake.go b/internal/forge/fake.go index 085a266bde..62b88e88eb 100644 --- a/internal/forge/fake.go +++ b/internal/forge/fake.go @@ -111,7 +111,7 @@ type FakeClient struct { // Pre-populated data Repos []Repository - OrgRepos map[string][]Repository // per-org repos; when set, ListOrgRepos uses this instead of Repos + OrgRepos map[string][]Repository // per-org repos; when set, ListOrgRepos uses this instead of Repos FileContents map[string][]byte // key: "owner/repo/path" WorkflowRuns map[string]*WorkflowRun // key: "owner/repo/workflow" RecentWorkflowRuns map[string][]WorkflowRun // key: "owner/repo" From 4c760fd8b01c949be0a2df373e0db89003e60541 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Mon, 6 Jul 2026 09:43:17 +0300 Subject: [PATCH 13/22] fix(runtime): treat dummy scripted op failures as exit code only Scripted operation failures (e.g. sandbox-blocked url_get) must not abort fullsend run before behaviour-results.json is extracted. Add a harness write_fixture to the sandbox scenario so triage validation still passes. Signed-off-by: Barak Korren Co-authored-by: Cursor --- e2e/behaviour/features/triage/triage.feature | 5 +++-- internal/runtime/dummy.go | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/e2e/behaviour/features/triage/triage.feature b/e2e/behaviour/features/triage/triage.feature index c0c31463ec..d2c1097386 100644 --- a/e2e/behaviour/features/triage/triage.feature +++ b/e2e/behaviour/features/triage/triage.feature @@ -17,8 +17,9 @@ Feature: Manual triage via ready-for-triage label Scenario: Sandbox blocks disallowed outbound URL Given the enrolled test repository And a dummy agent that would: - | description | op | args | - | Search for foo | url_get | https://www.google.com/search?q=foo | + | description | op | args | + | Search for foo | url_get | https://www.google.com/search?q=foo | + | Satisfy validation | write_fixture | output/agent-result.json, fixtures/triage/sufficient.json | And an issue When the issue is labeled "ready-for-triage" Then the triage workflow completes successfully diff --git a/internal/runtime/dummy.go b/internal/runtime/dummy.go index 537a9fad79..b5f1e1961c 100644 --- a/internal/runtime/dummy.go +++ b/internal/runtime/dummy.go @@ -71,13 +71,13 @@ func (r DummyRuntime) Run(_ context.Context, params RunParams, printer *ui.Print return 1, err } - results, execErr := executeBehaviourScript(params.SandboxName, params.RepoDir, script) - if writeErr := writeBehaviourResults(params.SandboxName, results); writeErr != nil && execErr == nil { - execErr = writeErr + results, scriptErr := executeBehaviourScript(params.SandboxName, params.RepoDir, script) + if writeErr := writeBehaviourResults(params.SandboxName, results); writeErr != nil { + return 1, writeErr } - if execErr != nil { - printer.StepWarn("Dummy runtime: " + execErr.Error()) + if scriptErr != nil { + printer.StepWarn("Dummy runtime: " + scriptErr.Error()) } // Non-zero exitCode mirrors ClaudeRuntime: run.go warns on non-zero exit but @@ -89,7 +89,7 @@ func (r DummyRuntime) Run(_ context.Context, params RunParams, printer *ui.Print break } } - return exitCode, execErr + return exitCode, nil } func (r DummyRuntime) ClearIterationArtifacts(sandboxName string) error { From 95e3190ba58ea5dd62eaf206512408ee9feacbe1 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Mon, 6 Jul 2026 09:54:21 +0300 Subject: [PATCH 14/22] test: cover dummy Run op-failure exit path for codecov patch Signed-off-by: Barak Korren Co-authored-by: Cursor --- internal/runtime/dummy.go | 5 ++++- internal/runtime/dummy_test.go | 30 ++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/internal/runtime/dummy.go b/internal/runtime/dummy.go index b5f1e1961c..cfbbd4ff56 100644 --- a/internal/runtime/dummy.go +++ b/internal/runtime/dummy.go @@ -19,6 +19,9 @@ import ( const behaviourScriptRelPath = "behaviour/current-scenario.yaml" const behaviourResultsFile = "behaviour-results.json" +// writeBehaviourResultsFn is the production writer; tests may replace it. +var writeBehaviourResultsFn = writeBehaviourResults + // BehaviourOperation is a single scripted step for the dummy runtime. type BehaviourOperation struct { Description string `yaml:"description" json:"description"` @@ -72,7 +75,7 @@ func (r DummyRuntime) Run(_ context.Context, params RunParams, printer *ui.Print } results, scriptErr := executeBehaviourScript(params.SandboxName, params.RepoDir, script) - if writeErr := writeBehaviourResults(params.SandboxName, results); writeErr != nil { + if writeErr := writeBehaviourResultsFn(params.SandboxName, results); writeErr != nil { return 1, writeErr } diff --git a/internal/runtime/dummy_test.go b/internal/runtime/dummy_test.go index 9e935c7473..2b9fff5f01 100644 --- a/internal/runtime/dummy_test.go +++ b/internal/runtime/dummy_test.go @@ -309,8 +309,9 @@ func TestWriteBehaviourResults(t *testing.T) { } func TestDummyRuntime_RunFailedOps(t *testing.T) { - t.Parallel() - + orig := writeBehaviourResultsFn + writeBehaviourResultsFn = writeBehaviourResults + t.Cleanup(func() { writeBehaviourResultsFn = orig }) fullsendDir := t.TempDir() scriptDir := filepath.Join(fullsendDir, "behaviour") require.NoError(t, os.MkdirAll(scriptDir, 0o755)) @@ -330,3 +331,28 @@ func TestDummyRuntime_RunFailedOps(t *testing.T) { assert.Equal(t, 1, exit) require.Error(t, err) } + +func TestDummyRuntime_RunOpFailureReturnsNilGoError(t *testing.T) { + orig := writeBehaviourResultsFn + writeBehaviourResultsFn = func(string, BehaviourResults) error { return nil } + t.Cleanup(func() { writeBehaviourResultsFn = orig }) + + fullsendDir := t.TempDir() + scriptDir := filepath.Join(fullsendDir, "behaviour") + require.NoError(t, os.MkdirAll(scriptDir, 0o755)) + script := `ops: +- description: blocked fetch + op: url_get + args: https://example.com/blocked +` + require.NoError(t, os.WriteFile(filepath.Join(scriptDir, "current-scenario.yaml"), []byte(script), 0o644)) + + rt := DummyRuntime{} + exit, err := rt.Run(context.Background(), RunParams{ + SandboxName: "nonexistent-sandbox", + RepoDir: t.TempDir(), + FullsendDir: fullsendDir, + }, ui.New(io.Discard), time.Now(), nil) + assert.Equal(t, 1, exit) + require.NoError(t, err) +} From c395ea9a44d66bf9e4d73f56b8221e2f33f7172f Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Mon, 6 Jul 2026 10:06:50 +0300 Subject: [PATCH 15/22] test: cover fake workflow helpers for codecov patch Signed-off-by: Barak Korren Co-authored-by: Cursor --- internal/forge/fake_test.go | 53 +++++++++++++++++++++++++++++++ internal/runtime/registry_test.go | 3 ++ 2 files changed, 56 insertions(+) diff --git a/internal/forge/fake_test.go b/internal/forge/fake_test.go index 377f2fe1ab..f85126aaa7 100644 --- a/internal/forge/fake_test.go +++ b/internal/forge/fake_test.go @@ -1015,3 +1015,56 @@ func TestFakeClient_CommitFilesErrSeq(t *testing.T) { require.ErrorIs(t, err, ErrNonFastForward) }) } + +func TestFakeClient_GetRepo(t *testing.T) { + ctx := context.Background() + fc := &FakeClient{ + Repos: []Repository{{Name: "repo", FullName: "org/repo"}}, + } + + repo, err := fc.GetRepo(ctx, "org", "repo") + require.NoError(t, err) + assert.Equal(t, "org/repo", repo.FullName) + + _, err = fc.GetRepo(ctx, "org", "missing") + require.ErrorIs(t, err, ErrNotFound) +} + +func TestFakeClient_GetOrgPlan(t *testing.T) { + ctx := context.Background() + fc := &FakeClient{} + + plan, err := fc.GetOrgPlan(ctx, "org") + require.NoError(t, err) + assert.Equal(t, "free", plan) + + fc.OrgPlan = "enterprise" + plan, err = fc.GetOrgPlan(ctx, "org") + require.NoError(t, err) + assert.Equal(t, "enterprise", plan) +} + +func TestFakeClient_ListWorkflowRuns(t *testing.T) { + ctx := context.Background() + run := WorkflowRun{ID: 42, Status: "completed", Conclusion: "success"} + fc := &FakeClient{ + WorkflowRuns: map[string]*WorkflowRun{ + "org/repo/ci.yml": &run, + }, + } + + runs, err := fc.ListWorkflowRuns(ctx, "org", "repo", "ci.yml") + require.NoError(t, err) + require.Len(t, runs, 1) + assert.Equal(t, 42, runs[0].ID) + + runs, err = fc.ListWorkflowRuns(ctx, "org", "repo", "missing.yml") + require.NoError(t, err) + assert.Nil(t, runs) +} + +func TestFakeClient_DispatchWorkflow(t *testing.T) { + ctx := context.Background() + fc := &FakeClient{} + require.NoError(t, fc.DispatchWorkflow(ctx, "org", "repo", "ci.yml", "main", map[string]string{"k": "v"})) +} diff --git a/internal/runtime/registry_test.go b/internal/runtime/registry_test.go index a7c14a9cb3..c298aa6582 100644 --- a/internal/runtime/registry_test.go +++ b/internal/runtime/registry_test.go @@ -47,4 +47,7 @@ func TestResolveFromPerRepoConfig(t *testing.T) { dummyBackend, err := ResolveFromPerRepoConfig(cfg) require.NoError(t, err) assert.Equal(t, "dummy", dummyBackend.Runtime.Name()) + + _, err = ResolveFromPerRepoConfig(&config.PerRepoConfig{Version: "1", Runtime: "invalid"}) + require.Error(t, err) } From 69a35e048c7b55a2e01120b8023ea1b3ae949188 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Mon, 6 Jul 2026 10:16:38 +0300 Subject: [PATCH 16/22] test: expand dummy runtime and github setup coverage Signed-off-by: Barak Korren Co-authored-by: Cursor --- internal/cli/github_test.go | 23 +++++++++++ internal/runtime/dummy.go | 16 +++++--- internal/runtime/dummy_test.go | 71 ++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 5 deletions(-) diff --git a/internal/cli/github_test.go b/internal/cli/github_test.go index 299db82532..3fbdd32334 100644 --- a/internal/cli/github_test.go +++ b/internal/cli/github_test.go @@ -982,3 +982,26 @@ func TestRunGitHubSetupPerRepo_RuntimeInConfig(t *testing.T) { require.NoError(t, err) assert.Equal(t, "dummy", cfg.Runtime) } + +func TestRunGitHubSetupPerRepo_InvalidRuntime(t *testing.T) { + t.Setenv("GH_TOKEN", "test-token") + client := forge.NewFakeClient() + client.AuthenticatedUser = "acme" + client.Repos = []forge.Repository{{FullName: "acme/widget", DefaultBranch: "main"}} + client.TokenScopes = []string{"repo", "workflow"} + client.Secrets = map[string]bool{ + "acme/widget/FULLSEND_GCP_PROJECT_ID": true, + "acme/widget/FULLSEND_GCP_WIF_PROVIDER": true, + } + printer := ui.New(&discardWriter{}) + + err := runGitHubSetupPerRepo(context.Background(), client, printer, githubSetupConfig{ + target: "acme/widget", + mintURL: "https://mint-test-abc123.run.app", + inferenceRegion: "global", + agents: strings.Join(config.PerRepoDefaultRoles(), ","), + runtime: "invalid-runtime", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid runtime") +} diff --git a/internal/runtime/dummy.go b/internal/runtime/dummy.go index cfbbd4ff56..02b96faf32 100644 --- a/internal/runtime/dummy.go +++ b/internal/runtime/dummy.go @@ -22,6 +22,12 @@ const behaviourResultsFile = "behaviour-results.json" // writeBehaviourResultsFn is the production writer; tests may replace it. var writeBehaviourResultsFn = writeBehaviourResults +// sandboxExecFn and sandboxUploadFn are production defaults; tests may replace them. +var ( + sandboxExecFn = sandbox.Exec + sandboxUploadFn = sandbox.Upload +) + // BehaviourOperation is a single scripted step for the dummy runtime. type BehaviourOperation struct { Description string `yaml:"description" json:"description"` @@ -162,7 +168,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro return err } cmd := fmt.Sprintf("test -r %s", shellQuote(remotePath)) - _, stderr, exitCode, err := sandbox.Exec(sandboxName, cmd, 30*time.Second) + _, stderr, exitCode, err := sandboxExecFn(sandboxName, cmd, 30*time.Second) if err != nil { return fmt.Errorf("read_file exec: %w", err) } @@ -176,7 +182,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro return fmt.Errorf("url_get requires a URL") } cmd := fmt.Sprintf("curl -sf %s -o /dev/null", shellQuote(url)) - _, stderr, exitCode, err := sandbox.Exec(sandboxName, cmd, 60*time.Second) + _, stderr, exitCode, err := sandboxExecFn(sandboxName, cmd, 60*time.Second) if err != nil { return fmt.Errorf("url_get exec: %w", err) } @@ -195,7 +201,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro } parentDir := filepath.Dir(remoteDest) mkdirCmd := fmt.Sprintf("mkdir -p %s", shellQuote(parentDir)) - if _, _, _, err := sandbox.Exec(sandboxName, mkdirCmd, 10*time.Second); err != nil { + if _, _, _, err := sandboxExecFn(sandboxName, mkdirCmd, 10*time.Second); err != nil { return fmt.Errorf("write_fixture mkdir: %w", err) } tmp, err := os.CreateTemp("", "behaviour-fixture-*") @@ -208,7 +214,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro return fmt.Errorf("write_fixture write temp: %w", err) } tmp.Close() - if err := sandbox.Upload(sandboxName, tmp.Name(), remoteDest); err != nil { + if err := sandboxUploadFn(sandboxName, tmp.Name(), remoteDest); err != nil { return fmt.Errorf("write_fixture upload: %w", err) } return nil @@ -265,7 +271,7 @@ func writeBehaviourResults(sandboxName string, results BehaviourResults) error { } tmp.Close() remotePath := filepath.Join(sandbox.SandboxWorkspace, "output", behaviourResultsFile) - return sandbox.Upload(sandboxName, tmp.Name(), remotePath) + return sandboxUploadFn(sandboxName, tmp.Name(), remotePath) } func shellQuote(s string) string { diff --git a/internal/runtime/dummy_test.go b/internal/runtime/dummy_test.go index 2b9fff5f01..9d2b06e073 100644 --- a/internal/runtime/dummy_test.go +++ b/internal/runtime/dummy_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "time" @@ -356,3 +357,73 @@ func TestDummyRuntime_RunOpFailureReturnsNilGoError(t *testing.T) { assert.Equal(t, 1, exit) require.NoError(t, err) } + +func TestExecuteBehaviourOp_URLGetNonZeroExit(t *testing.T) { + orig := sandboxExecFn + sandboxExecFn = func(_ string, cmd string, _ time.Duration) (string, string, int, error) { + if strings.Contains(cmd, "curl") { + return "", "blocked by sandbox policy", 22, nil + } + return "", "", 0, nil + } + t.Cleanup(func() { sandboxExecFn = orig }) + + err := executeBehaviourOp("sandbox", t.TempDir(), BehaviourOperation{ + Op: "url_get", + Args: "https://www.google.com/search?q=foo", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "url_get failed") + assert.Contains(t, err.Error(), "blocked by sandbox policy") +} + +func TestExecuteBehaviourOp_ReadFileNonZeroExit(t *testing.T) { + orig := sandboxExecFn + sandboxExecFn = func(_ string, _ string, _ time.Duration) (string, string, int, error) { + return "", "missing file", 1, nil + } + t.Cleanup(func() { sandboxExecFn = orig }) + + err := executeBehaviourOp("sandbox", t.TempDir(), BehaviourOperation{ + Op: "read_file", + Args: "output/missing.json", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "read_file failed") +} + +func TestExecuteBehaviourOp_WriteFixtureSuccess(t *testing.T) { + origExec := sandboxExecFn + origUpload := sandboxUploadFn + sandboxExecFn = func(_ string, _ string, _ time.Duration) (string, string, int, error) { + return "", "", 0, nil + } + sandboxUploadFn = func(_, _, _ string) error { return nil } + t.Cleanup(func() { + sandboxExecFn = origExec + sandboxUploadFn = origUpload + }) + + err := executeBehaviourOp("sandbox", t.TempDir(), BehaviourOperation{ + Op: "write_fixture", + Args: "output/agent-result.json, fixtures/triage/sufficient.json", + Content: `{"action":"sufficient"}`, + }) + require.NoError(t, err) +} + +func TestWriteBehaviourResultsSuccess(t *testing.T) { + orig := sandboxUploadFn + var uploaded bool + sandboxUploadFn = func(_, _, _ string) error { + uploaded = true + return nil + } + t.Cleanup(func() { sandboxUploadFn = orig }) + + err := writeBehaviourResults("sandbox", BehaviourResults{ + Operations: []BehaviourOpResult{{Description: "ok", Success: true}}, + }) + require.NoError(t, err) + assert.True(t, uploaded) +} From 2dc14f6759ee09af52f2732c95a6132b7ed34b84 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Tue, 7 Jul 2026 09:39:48 +0300 Subject: [PATCH 17/22] ci(e2e): address behaviour PR review findings Wire Runtime through OrgConfigFromPerRepo, harden dummy runtime and CI drivers, improve forge retry/artifact handling, acknowledge infra trade-offs in ADR 0065, and restore ADR 0003 example with cross-reference. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/ADRs/0003-org-config-repo-convention.md | 8 +- ...ehaviour-tests-with-gherkin-and-drivers.md | 2 + docs/guides/dev/behaviour-drivers.md | 2 +- .../drivers/ci/githubactions/githubactions.go | 55 +++++++++--- .../ci/githubactions/githubactions_test.go | 86 ++++++++++++++----- e2e/behaviour/suite_test.go | 4 +- internal/cli/run.go | 24 ++++-- internal/cli/run_test.go | 17 +++- internal/config/config.go | 2 +- internal/config/config_test.go | 2 + internal/forge/github/github.go | 32 ++++++- internal/runtime/dummy.go | 39 +++++++-- internal/runtime/dummy_test.go | 24 +++++- 13 files changed, 236 insertions(+), 61 deletions(-) diff --git a/docs/ADRs/0003-org-config-repo-convention.md b/docs/ADRs/0003-org-config-repo-convention.md index 4772699c5b..e39e0b279e 100644 --- a/docs/ADRs/0003-org-config-repo-convention.md +++ b/docs/ADRs/0003-org-config-repo-convention.md @@ -212,11 +212,7 @@ version: 1 # Schema version for future evolution intent_repo: /features # or /intent architecture_repo: /architecture -# Agent runtime defaults (see ADR 0065 for behaviour-test dummy runtime) -defaults: - runtime: claude # claude or dummy (dummy: behaviour test orgs only) - -# Legacy harness/model settings (separate from defaults.runtime backend selection) +# Agent runtime defaults runtime: harness: claude-code # or opencode model: claude-sonnet-4-6 @@ -232,6 +228,8 @@ sandbox: filesystem: ephemeral ``` +Backend selection for behaviour tests (`defaults.runtime` / per-repo `runtime: dummy`) is documented in [ADR 0065](0065-behaviour-tests-with-gherkin-and-drivers.md), not in the org config example above. + Per-repo overrides in `repos/.yaml` can override org defaults (within the bounds of org-wide guardrails that cannot be weakened). diff --git a/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md b/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md index d3a778e8b3..699591a11c 100644 --- a/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md +++ b/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md @@ -38,3 +38,5 @@ Runtime selection is shared with production via `defaults.runtime` in org `confi - Behaviour orgs are provisioned at suite start with `--runtime dummy`; production orgs must not use dummy unintentionally. - Adding GitLab or Tekton requires new drivers and runner env values, not feature file rewrites. - Dummy runtime op vocabulary stays minimal; new ops require runtime + docs updates when scenarios need them. +- Behaviour tests depend on live external infrastructure: GitHub API, GitHub Actions runners, GCP WIF/mint, and the shared halfsend org pool. Transient outages, API rate limits, or pool org state corruption can fail the suite; CI distinguishes infrastructure failures from regressions via workflow logs and artifact inspection, but there is no offline fallback. +- Behaviour tests share the halfsend org pool and lock mechanism with admin e2e tests (`e2e.yml` runs both jobs). Lock hold time scales with scenario count; pool size was doubled to absorb the additional load and can be increased again if contention appears. diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md index 619d6cb3a2..e9f0d1c28d 100644 --- a/docs/guides/dev/behaviour-drivers.md +++ b/docs/guides/dev/behaviour-drivers.md @@ -15,7 +15,7 @@ v1 reference implementations: - `e2e/behaviour/drivers/scm/github/` - `e2e/behaviour/drivers/ci/githubactions/` -- `e2e/behaviour/drivers/install/github/perrepo/` (`BEHAVIOUR_INSTALL_MODE=per-repo`) +- `e2e/behaviour/drivers/install/perrepo_github.go` (`BEHAVIOUR_INSTALL_MODE=per-repo`) ## Runner configuration diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions.go b/e2e/behaviour/drivers/ci/githubactions/githubactions.go index dee5ac6321..cf1b2216ce 100644 --- a/e2e/behaviour/drivers/ci/githubactions/githubactions.go +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions.go @@ -54,6 +54,9 @@ func (d *Driver) WaitForWorkflow(ctx context.Context, owner, repo, workflowFile continue } if candidate := selectWorkflowRun(runs, after, event); candidate != nil { + if candidate.Status == "completed" && candidate.Conclusion != "success" { + return nil, fmt.Errorf("workflow %s run %d concluded with %q during dispatch", workflowFile, candidate.ID, candidate.Conclusion) + } triageRun = candidate break } @@ -80,7 +83,7 @@ func (d *Driver) WaitForWorkflow(ctx context.Context, owner, repo, workflowFile if run.Conclusion == "success" { return run, nil } - if replacement := selectWorkflowRun(latestRuns(ctx, d, owner, repo, workflowFile), after, event); replacement != nil && replacement.ID > triageRun.ID { + if replacement := selectSuccessfulWorkflowRun(latestRuns(ctx, d, owner, repo, workflowFile), after, event); replacement != nil && replacement.ID > triageRun.ID { triageRun = replacement continue } @@ -98,17 +101,29 @@ func latestRuns(ctx context.Context, d *Driver, owner, repo, workflowFile string return runs } -// selectWorkflowRun returns the newest workflow run after triggerTime that is -// still active or completed successfully. Skipped/cancelled/failed runs are -// ignored so a concurrent stale dispatch does not mask the real triage run. -// When event is non-empty, only runs with a matching GitHub event are considered. +// selectWorkflowRun returns the newest workflow run after triggerTime that matches +// the optional event filter. Callers decide how to handle non-success conclusions. func selectWorkflowRun(runs []forge.WorkflowRun, triggerTime time.Time, event string) *forge.WorkflowRun { var best *forge.WorkflowRun for _, run := range runs { if !workflowRunMatches(run, triggerTime, event) { continue } - if run.Status == "completed" && run.Conclusion != "success" { + if best == nil || run.ID > best.ID { + r := run + best = &r + } + } + return best +} + +func selectSuccessfulWorkflowRun(runs []forge.WorkflowRun, triggerTime time.Time, event string) *forge.WorkflowRun { + var best *forge.WorkflowRun + for _, run := range runs { + if !workflowRunMatches(run, triggerTime, event) { + continue + } + if run.Status != "completed" || run.Conclusion != "success" { continue } if best == nil || run.ID > best.ID { @@ -296,18 +311,25 @@ func selectRepositoryArtifactAfter(arts []forge.RepositoryArtifact, name string, } func extractArtifactZip(name string, zipData []byte, destDir string) error { - artDir := filepath.Join(destDir, name) + safeName := filepath.Base(name) + if safeName == "" || safeName == "." { + safeName = "artifact" + } + artDir := filepath.Join(destDir, safeName) if err := os.MkdirAll(artDir, 0o755); err != nil { return err } zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) if err != nil { - rawPath := filepath.Join(destDir, name+".bin") - return os.WriteFile(rawPath, zipData, 0o644) + return fmt.Errorf("parse artifact zip %q: %w", safeName, err) } + const perFileLimit = 10 << 20 for _, f := range zr.File { + if f.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("artifact zip %q contains symlink entry %q", safeName, f.Name) + } outPath := filepath.Join(artDir, f.Name) if !strings.HasPrefix(filepath.Clean(outPath), filepath.Clean(artDir)+string(os.PathSeparator)) { continue @@ -321,10 +343,10 @@ func extractArtifactZip(name string, zipData []byte, destDir string) error { if err != nil { return err } - data, err := io.ReadAll(io.LimitReader(rc, 10<<20)) + data, err := readLimited(rc, perFileLimit) rc.Close() if err != nil { - return err + return fmt.Errorf("read artifact entry %q: %w", f.Name, err) } if err := os.WriteFile(outPath, data, 0o644); err != nil { return err @@ -332,3 +354,14 @@ func extractArtifactZip(name string, zipData []byte, destDir string) error { } return nil } + +func readLimited(r io.Reader, limit int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("entry exceeds %d byte limit", limit) + } + return data, nil +} diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go index c98a8e8d0d..339307da41 100644 --- a/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go @@ -1,40 +1,86 @@ package githubactions import ( + "archive/zip" + "bytes" + "os" + "path/filepath" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/fullsend-ai/fullsend/internal/forge" ) -func TestSelectWorkflowRun_FiltersByEvent(t *testing.T) { +func TestSelectWorkflowRun_ReturnsFailedRun(t *testing.T) { t.Parallel() - after := time.Date(2026, 7, 1, 11, 33, 0, 0, time.UTC) - runs := []forge.WorkflowRun{ - {ID: 1, Event: "issues", Status: "completed", Conclusion: "success", CreatedAt: "2026-07-01T11:33:09Z"}, - {ID: 2, Event: "issue_comment", Status: "completed", Conclusion: "skipped", CreatedAt: "2026-07-01T11:33:09Z"}, - {ID: 3, Event: "issue_comment", Status: "in_progress", Conclusion: "", CreatedAt: "2026-07-01T11:33:10Z"}, - } - got := selectWorkflowRun(runs, after, "issue_comment") - if got == nil || got.ID != 3 { - t.Fatalf("selectWorkflowRun() = %#v, want issue_comment run 3", got) + after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + runs := []forge.WorkflowRun{ + {ID: 1, Status: "completed", Conclusion: "failure", Event: "issues", CreatedAt: "2026-01-01T01:00:00Z"}, } - got = selectWorkflowRun(runs, after, "issues") - if got == nil || got.ID != 1 { - t.Fatalf("selectWorkflowRun(issues) = %#v, want run 1", got) - } + got := selectWorkflowRun(runs, after, "issues") + require.NotNil(t, got) + assert.Equal(t, 1, got.ID) + assert.Equal(t, "failure", got.Conclusion) } -func TestSelectWorkflowRun_IgnoresFailedBeforeSuccess(t *testing.T) { +func TestSelectSuccessfulWorkflowRun_SkipsFailed(t *testing.T) { t.Parallel() - after := time.Date(2026, 7, 1, 11, 33, 0, 0, time.UTC) + + after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) runs := []forge.WorkflowRun{ - {ID: 10, Event: "issue_comment", Status: "completed", Conclusion: "skipped", CreatedAt: "2026-07-01T11:33:09Z"}, + {ID: 1, Status: "completed", Conclusion: "failure", Event: "issues", CreatedAt: "2026-01-01T01:00:00Z"}, + {ID: 2, Status: "completed", Conclusion: "success", Event: "issues", CreatedAt: "2026-01-01T02:00:00Z"}, } - if got := selectWorkflowRun(runs, after, "issue_comment"); got != nil { - t.Fatalf("selectWorkflowRun() = %#v, want nil for skipped-only runs", got) - } + got := selectSuccessfulWorkflowRun(runs, after, "issues") + require.NotNil(t, got) + assert.Equal(t, 2, got.ID) +} + +func TestExtractArtifactZip_RejectsCorruptZip(t *testing.T) { + t.Parallel() + + dest := t.TempDir() + err := extractArtifactZip("artifact", []byte("not-a-zip"), dest) + require.Error(t, err) + assert.Contains(t, err.Error(), "parse artifact zip") +} + +func TestExtractArtifactZip_RejectsSymlink(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + hdr := &zip.FileHeader{Name: "link", Method: zip.Store} + hdr.SetMode(os.ModeSymlink | 0o755) + _, err := zw.CreateHeader(hdr) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + dest := t.TempDir() + err = extractArtifactZip("../escape", buf.Bytes(), dest) + require.Error(t, err) + assert.Contains(t, err.Error(), "symlink") +} + +func TestExtractArtifactZip_SanitizesName(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("ok.txt") + require.NoError(t, err) + _, err = w.Write([]byte("hello")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + dest := t.TempDir() + require.NoError(t, extractArtifactZip("../../weird/name", buf.Bytes(), dest)) + _, err = os.Stat(filepath.Join(dest, "name", "ok.txt")) + require.NoError(t, err) } diff --git a/e2e/behaviour/suite_test.go b/e2e/behaviour/suite_test.go index 0a4c0e8db3..52b88e9ce1 100644 --- a/e2e/behaviour/suite_test.go +++ b/e2e/behaviour/suite_test.go @@ -40,6 +40,9 @@ func TestBehaviourSuite(t *testing.T) { t.Fatalf("acquiring org: %v", err) } client := admin.NewLiveClient(token) + t.Cleanup(func() { + admin.ReleaseLock(context.Background(), client, org, runID, t) + }) binary := admin.BuildCLIBinary(t) installDriver, err := install.NewDriver(cfg, e2eCfg, client, token, binary, t.Logf) @@ -59,7 +62,6 @@ func TestBehaviourSuite(t *testing.T) { if teardownErr := installDriver.Teardown(teardownCtx, org, installState); teardownErr != nil { t.Logf("install teardown: %v", teardownErr) } - admin.ReleaseLock(teardownCtx, client, org, runID, t) }) testRepo := installState.TestRepo() diff --git a/internal/cli/run.go b/internal/cli/run.go index 4842781a93..fcfad48b72 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "io" "math" @@ -106,25 +107,30 @@ func writeMetricsJSON(dir string, m aggregateMetrics) error { return os.WriteFile(filepath.Join(dir, metricsFile), append(data, '\n'), 0o644) } +var ( + errParsingConfigRuntime = errors.New("parsing config for runtime selection") + errResolvingRuntime = errors.New("resolving runtime") +) + func resolveBackendFromConfigData(orgConfigData []byte) (agentruntime.Backend, error) { if isOrgConfigData(orgConfigData) { orgCfg, orgErr := config.ParseOrgConfig(orgConfigData) if orgErr != nil { - return agentruntime.Backend{}, fmt.Errorf("parsing config for runtime selection: %w", orgErr) + return agentruntime.Backend{}, fmt.Errorf("%w: %w", errParsingConfigRuntime, orgErr) } backend, resolveErr := agentruntime.ResolveFromConfig(orgCfg) if resolveErr != nil { - return agentruntime.Backend{}, fmt.Errorf("resolving runtime: %w", resolveErr) + return agentruntime.Backend{}, fmt.Errorf("%w: %w", errResolvingRuntime, resolveErr) } return backend, nil } perRepoCfg, perRepoErr := config.ParsePerRepoConfig(orgConfigData) if perRepoErr != nil { - return agentruntime.Backend{}, fmt.Errorf("parsing config for runtime selection: %w", perRepoErr) + return agentruntime.Backend{}, fmt.Errorf("%w: %w", errParsingConfigRuntime, perRepoErr) } backend, resolveErr := agentruntime.ResolveFromPerRepoConfig(perRepoCfg) if resolveErr != nil { - return agentruntime.Backend{}, fmt.Errorf("resolving runtime: %w", resolveErr) + return agentruntime.Backend{}, fmt.Errorf("%w: %w", errResolvingRuntime, resolveErr) } return backend, nil } @@ -168,7 +174,7 @@ func backendFromConfigFile(path string) (agentruntime.Backend, error) { if os.IsNotExist(readErr) { return agentruntime.Default(), nil } - return agentruntime.Backend{}, fmt.Errorf("reading org config for runtime selection: %w", readErr) + return agentruntime.Backend{}, fmt.Errorf("reading config.yaml for runtime selection: %w", readErr) } func newRunCmd() *cobra.Command { @@ -815,12 +821,12 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep backend, backendErr := backendFromConfigFile(orgConfigPath) if backendErr != nil { switch { - case strings.Contains(backendErr.Error(), "parsing config for runtime selection"): - printer.StepFail("Failed to parse org config") - case strings.Contains(backendErr.Error(), "resolving runtime"): + case errors.Is(backendErr, errParsingConfigRuntime): + printer.StepFail("Failed to parse config.yaml") + case errors.Is(backendErr, errResolvingRuntime): printer.StepFail("Failed to resolve runtime") default: - printer.StepFail("Failed to load org config") + printer.StepFail("Failed to load config.yaml") } return backendErr } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index f91f9930db..666ee3ef40 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3285,7 +3285,7 @@ func TestBackendFromConfigFile_ReadError(t *testing.T) { _, err := backendFromConfigFile(path) require.Error(t, err) - assert.Contains(t, err.Error(), "reading org config for runtime selection") + assert.Contains(t, err.Error(), "reading config.yaml for runtime selection") } func TestBackendFromConfigFile_ResolveError(t *testing.T) { @@ -3309,3 +3309,18 @@ func TestIsOrgConfigData_InvalidYAML(t *testing.T) { assert.False(t, isOrgConfigData([]byte("not: [valid"))) } + +func TestIsOrgConfigData_HeaderlessPerRepoByStructure(t *testing.T) { + t.Parallel() + + // Hand-edited per-repo config without the header comment. + data := []byte("version: \"1\"\nroles:\n - triage\n") + assert.False(t, isOrgConfigData(data)) +} + +func TestIsOrgConfigData_HeaderlessOrgByStructure(t *testing.T) { + t.Parallel() + + data := []byte("version: \"1\"\ndefaults:\n roles:\n - triage\nrepos:\n widget:\n enabled: true\n") + assert.True(t, isOrgConfigData(data)) +} diff --git a/internal/config/config.go b/internal/config/config.go index c534c6903c..dfb77faac7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -462,7 +462,7 @@ func OrgConfigFromPerRepo(pr *PerRepoConfig) *OrgConfig { return &OrgConfig{ Version: pr.Version, KillSwitch: pr.KillSwitch, - Defaults: RepoDefaults{Roles: pr.Roles}, + Defaults: RepoDefaults{Roles: pr.Roles, Runtime: pr.Runtime}, Agents: pr.Agents, AllowedRemoteResources: pr.AllowedRemoteResources, CreateIssues: pr.CreateIssues, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f496456e01..876dbfc46e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1717,6 +1717,7 @@ func TestOrgConfigFromPerRepo(t *testing.T) { Version: "1", KillSwitch: true, Roles: []string{"triage", "coder"}, + Runtime: "dummy", Agents: []AgentEntry{ {Name: "lint", Source: "harness/lint.yaml"}, }, @@ -1736,6 +1737,7 @@ func TestOrgConfigFromPerRepo(t *testing.T) { require.NotNil(t, org.CreateIssues) assert.Equal(t, []string{"org/repo"}, org.CreateIssues.AllowTargets.Repos) assert.Equal(t, []string{"triage", "coder"}, org.Defaults.Roles) + assert.Equal(t, "dummy", org.Defaults.Runtime) assert.Nil(t, org.Repos) assert.Empty(t, org.Dispatch.Platform) } diff --git a/internal/forge/github/github.go b/internal/forge/github/github.go index 46363c49d0..8072831b0d 100644 --- a/internal/forge/github/github.go +++ b/internal/forge/github/github.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "log" "math" "math/rand/v2" "net/http" @@ -743,9 +744,23 @@ func (c *LiveClient) CommitFiles(ctx context.Context, owner, repo, message strin } func (c *LiveClient) commitFilesWithRetry(ctx context.Context, owner, repo, branch, message string, files []forge.TreeFile) (bool, error) { - changed, err := c.commitFilesTo(ctx, owner, repo, branch, message, files) - if err != nil && forge.IsNonFastForward(err) { + const maxAttempts = 3 + var changed bool + var err error + for attempt := 0; attempt < maxAttempts; attempt++ { + if attempt > 0 { + jitter := time.Duration(100+rand.IntN(400)) * time.Millisecond + select { + case <-ctx.Done(): + return false, ctx.Err() + case <-time.After(jitter): + } + log.Printf("retrying commit to %s/%s@%s (attempt %d/%d): %v", owner, repo, branch, attempt+1, maxAttempts, err) + } changed, err = c.commitFilesTo(ctx, owner, repo, branch, message, files) + if err == nil || !forge.IsNonFastForward(err) { + break + } } return changed, err } @@ -2627,13 +2642,24 @@ func (c *LiveClient) DownloadWorkflowRunArtifact(ctx context.Context, owner, rep if err := checkStatus(resp, http.StatusOK); err != nil { return nil, fmt.Errorf("download workflow artifact %d: %w", artifactID, err) } - data, err := io.ReadAll(io.LimitReader(resp.Body, 50<<20)) + data, err := readLimitedBytes(resp.Body, 50<<20) if err != nil { return nil, fmt.Errorf("read workflow artifact %d: %w", artifactID, err) } return data, nil } +func readLimitedBytes(r io.Reader, limit int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("response exceeds %d byte limit", limit) + } + return data, nil +} + // ListRepositoryArtifacts returns recent artifacts stored for a repository. func (c *LiveClient) ListRepositoryArtifacts(ctx context.Context, owner, repo string, perPage int) ([]forge.RepositoryArtifact, error) { if perPage <= 0 { diff --git a/internal/runtime/dummy.go b/internal/runtime/dummy.go index 02b96faf32..d219951fd1 100644 --- a/internal/runtime/dummy.go +++ b/internal/runtime/dummy.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "net/url" "os" "path/filepath" "strings" @@ -69,18 +70,18 @@ func (DummyRuntime) EnvExports() []string { return nil } func (DummyRuntime) Bootstrap(input BootstrapInput) error { sandboxName := input.SandboxName() mkdirCmd := fmt.Sprintf("mkdir -p %s/output %s/.dummy", sandbox.SandboxWorkspace, sandbox.SandboxWorkspace) - _, _, _, err := sandbox.Exec(sandboxName, mkdirCmd, 10*time.Second) + _, _, _, err := sandboxExecFn(sandboxName, mkdirCmd, 10*time.Second) return err } -func (r DummyRuntime) Run(_ context.Context, params RunParams, printer *ui.Printer, _ time.Time, _ *RunMetrics) (int, error) { +func (r DummyRuntime) Run(ctx context.Context, params RunParams, printer *ui.Printer, _ time.Time, _ *RunMetrics) (int, error) { scriptPath := filepath.Join(params.FullsendDir, behaviourScriptRelPath) script, err := LoadBehaviourScript(scriptPath) if err != nil { return 1, err } - results, scriptErr := executeBehaviourScript(params.SandboxName, params.RepoDir, script) + results, scriptErr := executeBehaviourScript(ctx, params.SandboxName, params.RepoDir, script) if writeErr := writeBehaviourResultsFn(params.SandboxName, results); writeErr != nil { return 1, writeErr } @@ -103,7 +104,7 @@ func (r DummyRuntime) Run(_ context.Context, params RunParams, printer *ui.Print func (r DummyRuntime) ClearIterationArtifacts(sandboxName string) error { clearCmd := fmt.Sprintf("rm -rf %s/output/*", r.WorkspaceDir()) - _, _, _, err := sandbox.Exec(sandboxName, clearCmd, 10*time.Second) + _, _, _, err := sandboxExecFn(sandboxName, clearCmd, 10*time.Second) return err } @@ -137,10 +138,13 @@ func LoadBehaviourScript(path string) (*BehaviourScript, error) { return &script, nil } -func executeBehaviourScript(sandboxName, repoDir string, script *BehaviourScript) (BehaviourResults, error) { +func executeBehaviourScript(ctx context.Context, sandboxName, repoDir string, script *BehaviourScript) (BehaviourResults, error) { var results BehaviourResults var firstErr error for _, op := range script.Ops { + if err := ctx.Err(); err != nil { + return results, fmt.Errorf("behaviour script cancelled: %w", err) + } res := BehaviourOpResult{Description: op.Description} if err := executeBehaviourOp(sandboxName, repoDir, op); err != nil { res.Success = false @@ -177,11 +181,14 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro } return nil case "url_get": - url := strings.TrimSpace(op.Args) - if url == "" { + rawURL := strings.TrimSpace(op.Args) + if rawURL == "" { return fmt.Errorf("url_get requires a URL") } - cmd := fmt.Sprintf("curl -sf %s -o /dev/null", shellQuote(url)) + if err := validateHTTPURL(rawURL); err != nil { + return err + } + cmd := fmt.Sprintf("curl -sf -- %s -o /dev/null", shellQuote(rawURL)) _, stderr, exitCode, err := sandboxExecFn(sandboxName, cmd, 60*time.Second) if err != nil { return fmt.Errorf("url_get exec: %w", err) @@ -223,6 +230,22 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro } } +func validateHTTPURL(raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("url_get invalid URL: %w", err) + } + switch u.Scheme { + case "http", "https": + default: + return fmt.Errorf("url_get requires http or https scheme, got %q", u.Scheme) + } + if u.Host == "" { + return fmt.Errorf("url_get requires a host") + } + return nil +} + func resolveWriteFixture(op BehaviourOperation) (dest string, content string, err error) { parts := strings.SplitN(op.Args, ",", 2) if len(parts) != 2 { diff --git a/internal/runtime/dummy_test.go b/internal/runtime/dummy_test.go index 9d2b06e073..478bbade51 100644 --- a/internal/runtime/dummy_test.go +++ b/internal/runtime/dummy_test.go @@ -192,7 +192,7 @@ func TestExecuteBehaviourScript_ValidationFailures(t *testing.T) { {Op: "read_file", Description: "missing path"}, {Op: "url_get", Description: "missing url"}, }} - results, err := executeBehaviourScript("unused", t.TempDir(), script) + results, err := executeBehaviourScript(context.Background(), "unused", t.TempDir(), script) require.Error(t, err) require.Contains(t, err.Error(), "requires a path") require.Len(t, results.Operations, 2) @@ -427,3 +427,25 @@ func TestWriteBehaviourResultsSuccess(t *testing.T) { require.NoError(t, err) assert.True(t, uploaded) } + +func TestValidateHTTPURL(t *testing.T) { + t.Parallel() + + require.NoError(t, validateHTTPURL("https://example.com/path")) + require.Error(t, validateHTTPURL("file:///etc/passwd")) + require.Error(t, validateHTTPURL("")) + require.Error(t, validateHTTPURL("://missing")) +} + +func TestExecuteBehaviourScript_CancelledContext(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + script := &BehaviourScript{Ops: []BehaviourOperation{ + {Op: "read_file", Args: "x.txt", Description: "read"}, + }} + _, err := executeBehaviourScript(ctx, "sandbox", t.TempDir(), script) + require.Error(t, err) + assert.Contains(t, err.Error(), "cancelled") +} From 6dc0c32eec66d84310a91fda76207a2a066eb9b8 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Tue, 7 Jul 2026 20:34:44 +0300 Subject: [PATCH 18/22] ci(e2e): address maruiz93 review feedback Move ParseRepo to forge-neutral scm package, log runtime selection source, download named artifacts in fallback tiers, stream zip extraction via temp file, harden zip path handling, use json.Decoder for CLI output, and skip redundant artifact polls when the repository snapshot is unchanged. Signed-off-by: Barak Korren Co-authored-by: Cursor --- e2e/behaviour/drivers/ci/driver.go | 1 + .../drivers/ci/githubactions/githubactions.go | 69 +++++++++++++++++-- .../ci/githubactions/githubactions_test.go | 27 ++++++++ .../drivers/install/perrepo_github.go | 2 +- e2e/behaviour/drivers/scm/github/github.go | 8 +-- e2e/behaviour/drivers/scm/parse.go | 15 ++++ e2e/behaviour/drivers/scm/parse_test.go | 20 ++++++ e2e/behaviour/steps/artifacts.go | 2 +- e2e/behaviour/steps/triage.go | 4 +- internal/cli/run.go | 17 +++-- internal/cli/run_test.go | 12 ++-- internal/forge/fake_test.go | 66 ++++++++++++++++++ 12 files changed, 216 insertions(+), 27 deletions(-) create mode 100644 e2e/behaviour/drivers/scm/parse.go create mode 100644 e2e/behaviour/drivers/scm/parse_test.go diff --git a/e2e/behaviour/drivers/ci/driver.go b/e2e/behaviour/drivers/ci/driver.go index 7ca154fbd4..9044907214 100644 --- a/e2e/behaviour/drivers/ci/driver.go +++ b/e2e/behaviour/drivers/ci/driver.go @@ -14,5 +14,6 @@ type Driver interface { AssertNoWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) error GetRunLogs(ctx context.Context, owner, repo string, runID int) (string, error) DownloadArtifacts(ctx context.Context, owner, repo string, runID int, destDir string) error + DownloadNamedArtifactFromRun(ctx context.Context, owner, repo string, runID int, artifactName string, destDir string) error DownloadNamedArtifactAfter(ctx context.Context, owner, repo, artifactName string, after time.Time, destDir string) error } diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions.go b/e2e/behaviour/drivers/ci/githubactions/githubactions.go index cf1b2216ce..24740be899 100644 --- a/e2e/behaviour/drivers/ci/githubactions/githubactions.go +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions.go @@ -2,7 +2,6 @@ package githubactions import ( "archive/zip" - "bytes" "context" "fmt" "io" @@ -252,6 +251,24 @@ func (d *Driver) GetRunLogs(ctx context.Context, owner, repo string, runID int) return d.Client.GetWorkflowRunLogs(ctx, owner, repo, runID) } +func (d *Driver) DownloadNamedArtifactFromRun(ctx context.Context, owner, repo string, runID int, artifactName string, destDir string) error { + artifacts, err := d.Client.ListWorkflowRunArtifacts(ctx, owner, repo, runID) + if err != nil { + return err + } + for _, art := range artifacts { + if art.Name != artifactName { + continue + } + zipData, err := d.Client.DownloadWorkflowRunArtifact(ctx, owner, repo, art.ID) + if err != nil { + return err + } + return extractArtifactZip(art.Name, zipData, destDir) + } + return fmt.Errorf("artifact %q not found on workflow run %d", artifactName, runID) +} + func (d *Driver) DownloadArtifacts(ctx context.Context, owner, repo string, runID int, destDir string) error { artifacts, err := d.Client.ListWorkflowRunArtifacts(ctx, owner, repo, runID) if err != nil { @@ -271,11 +288,23 @@ func (d *Driver) DownloadArtifacts(ctx context.Context, owner, repo string, runI func (d *Driver) DownloadNamedArtifactAfter(ctx context.Context, owner, repo, artifactName string, after time.Time, destDir string) error { deadline := time.Now().Add(artifactRunWait) + var lastNewestCreatedAt string for time.Now().Before(deadline) { arts, err := d.Client.ListRepositoryArtifacts(ctx, owner, repo, 100) if err != nil { return err } + newestCreatedAt := newestRepositoryArtifactCreatedAt(arts) + if newestCreatedAt != "" && newestCreatedAt == lastNewestCreatedAt { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(artifactRunPoll): + } + continue + } + lastNewestCreatedAt = newestCreatedAt + if art := selectRepositoryArtifactAfter(arts, artifactName, after); art != nil { zipData, err := d.Client.DownloadWorkflowRunArtifact(ctx, owner, repo, art.ID) if err != nil { @@ -292,6 +321,16 @@ func (d *Driver) DownloadNamedArtifactAfter(ctx context.Context, owner, repo, ar return fmt.Errorf("artifact %q not found after %s", artifactName, after.Format(time.RFC3339)) } +func newestRepositoryArtifactCreatedAt(arts []forge.RepositoryArtifact) string { + var newest string + for _, art := range arts { + if art.CreatedAt > newest { + newest = art.CreatedAt + } + } + return newest +} + func selectRepositoryArtifactAfter(arts []forge.RepositoryArtifact, name string, after time.Time) *forge.RepositoryArtifact { var best *forge.RepositoryArtifact for _, art := range arts { @@ -311,6 +350,21 @@ func selectRepositoryArtifactAfter(arts []forge.RepositoryArtifact, name string, } func extractArtifactZip(name string, zipData []byte, destDir string) error { + tmp, err := os.CreateTemp("", "behaviour-artifact-*.zip") + if err != nil { + return fmt.Errorf("create temp artifact zip: %w", err) + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + + if _, err := tmp.Write(zipData); err != nil { + tmp.Close() + return fmt.Errorf("write temp artifact zip: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close temp artifact zip: %w", err) + } + safeName := filepath.Base(name) if safeName == "" || safeName == "." { safeName = "artifact" @@ -320,10 +374,11 @@ func extractArtifactZip(name string, zipData []byte, destDir string) error { return err } - zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) + zr, err := zip.OpenReader(tmpPath) if err != nil { return fmt.Errorf("parse artifact zip %q: %w", safeName, err) } + defer zr.Close() const perFileLimit = 10 << 20 for _, f := range zr.File { @@ -332,13 +387,17 @@ func extractArtifactZip(name string, zipData []byte, destDir string) error { } outPath := filepath.Join(artDir, f.Name) if !strings.HasPrefix(filepath.Clean(outPath), filepath.Clean(artDir)+string(os.PathSeparator)) { - continue + return fmt.Errorf("artifact zip %q contains path traversal entry %q", safeName, f.Name) } if f.FileInfo().IsDir() { - _ = os.MkdirAll(outPath, 0o755) + if err := os.MkdirAll(outPath, 0o755); err != nil { + return fmt.Errorf("create artifact dir %q: %w", f.Name, err) + } continue } - _ = os.MkdirAll(filepath.Dir(outPath), 0o755) + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + return fmt.Errorf("create artifact parent dir for %q: %w", f.Name, err) + } rc, err := f.Open() if err != nil { return err diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go index 339307da41..1208aa241e 100644 --- a/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go @@ -68,6 +68,23 @@ func TestExtractArtifactZip_RejectsSymlink(t *testing.T) { assert.Contains(t, err.Error(), "symlink") } +func TestExtractArtifactZip_RejectsPathTraversal(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("../escape.txt") + require.NoError(t, err) + _, err = w.Write([]byte("nope")) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + dest := t.TempDir() + err = extractArtifactZip("artifact", buf.Bytes(), dest) + require.Error(t, err) + assert.Contains(t, err.Error(), "path traversal") +} + func TestExtractArtifactZip_SanitizesName(t *testing.T) { t.Parallel() @@ -84,3 +101,13 @@ func TestExtractArtifactZip_SanitizesName(t *testing.T) { _, err = os.Stat(filepath.Join(dest, "name", "ok.txt")) require.NoError(t, err) } + +func TestNewestRepositoryArtifactCreatedAt(t *testing.T) { + t.Parallel() + + arts := []forge.RepositoryArtifact{ + {CreatedAt: "2026-01-01T00:00:00Z"}, + {CreatedAt: "2026-01-02T00:00:00Z"}, + } + assert.Equal(t, "2026-01-02T00:00:00Z", newestRepositoryArtifactCreatedAt(arts)) +} diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index a493b0fb22..5af5d6fca1 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -123,7 +123,7 @@ func parseInferenceStatusWIFProvider(output string) (string, error) { Status string `json:"status"` WIFProvider string `json:"FULLSEND_GCP_WIF_PROVIDER"` } - if err := json.Unmarshal([]byte(output[start:]), &status); err != nil { + if err := json.NewDecoder(strings.NewReader(output[start:])).Decode(&status); err != nil { return "", fmt.Errorf("parse JSON: %w", err) } if status.WIFProvider == "" { diff --git a/e2e/behaviour/drivers/scm/github/github.go b/e2e/behaviour/drivers/scm/github/github.go index 1bd80efaca..3b85e595d4 100644 --- a/e2e/behaviour/drivers/scm/github/github.go +++ b/e2e/behaviour/drivers/scm/github/github.go @@ -2,8 +2,6 @@ package github import ( "context" - "fmt" - "strings" "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm" "github.com/fullsend-ai/fullsend/internal/forge" @@ -49,9 +47,5 @@ func (d *Driver) GetIssue(ctx context.Context, owner, repo string, number int) ( // ParseRepo splits "owner/repo" into owner and repo name. func ParseRepo(fullName string) (owner, repo string, err error) { - parts := strings.Split(fullName, "/") - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return "", "", fmt.Errorf("invalid repository %q: expected owner/repo", fullName) - } - return parts[0], parts[1], nil + return scm.ParseRepo(fullName) } diff --git a/e2e/behaviour/drivers/scm/parse.go b/e2e/behaviour/drivers/scm/parse.go new file mode 100644 index 0000000000..53963e60ee --- /dev/null +++ b/e2e/behaviour/drivers/scm/parse.go @@ -0,0 +1,15 @@ +package scm + +import ( + "fmt" + "strings" +) + +// ParseRepo splits "owner/repo" into owner and repo name. +func ParseRepo(fullName string) (owner, repo string, err error) { + parts := strings.Split(fullName, "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid repository %q: expected owner/repo", fullName) + } + return parts[0], parts[1], nil +} diff --git a/e2e/behaviour/drivers/scm/parse_test.go b/e2e/behaviour/drivers/scm/parse_test.go new file mode 100644 index 0000000000..2a7e6fe413 --- /dev/null +++ b/e2e/behaviour/drivers/scm/parse_test.go @@ -0,0 +1,20 @@ +package scm + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseRepo(t *testing.T) { + t.Parallel() + + owner, repo, err := ParseRepo("acme/widget") + require.NoError(t, err) + assert.Equal(t, "acme", owner) + assert.Equal(t, "widget", repo) + + _, _, err = ParseRepo("invalid") + require.Error(t, err) +} diff --git a/e2e/behaviour/steps/artifacts.go b/e2e/behaviour/steps/artifacts.go index b08ed057b5..88c392d1c2 100644 --- a/e2e/behaviour/steps/artifacts.go +++ b/e2e/behaviour/steps/artifacts.go @@ -49,7 +49,7 @@ func ensureArtifacts(w *world.World) error { } tryDownloadRun := func(runID int) error { - if err := w.CI.DownloadArtifacts(ctx, w.Org, w.Install.TriageWorkflowRepo(), runID, dest); err != nil { + if err := w.CI.DownloadNamedArtifactFromRun(ctx, w.Org, w.Install.TriageWorkflowRepo(), runID, w.Install.AgentArtifactName(), dest); err != nil { return err } if _, findErr := artifacts.FindBehaviourResults(dest); findErr != nil { diff --git a/e2e/behaviour/steps/triage.go b/e2e/behaviour/steps/triage.go index 841e246d62..ec360003cc 100644 --- a/e2e/behaviour/steps/triage.go +++ b/e2e/behaviour/steps/triage.go @@ -7,7 +7,7 @@ import ( "github.com/cucumber/godog" - scmgh "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm/github" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm" "github.com/fullsend-ai/fullsend/e2e/behaviour/world" ) @@ -39,7 +39,7 @@ func givenEnrolledTestRepository(w *world.World) error { } func givenEnrolledRepository(w *world.World, fullName string) error { - owner, repo, err := scmgh.ParseRepo(fullName) + owner, repo, err := scm.ParseRepo(fullName) if err != nil { return err } diff --git a/internal/cli/run.go b/internal/cli/run.go index 9b36923f98..3d2d5ee004 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -186,23 +186,27 @@ func isOrgConfigData(data []byte) bool { return probe.Dispatch != nil || probe.Defaults != nil || len(probe.Repos) > 0 } -func backendFromConfigFile(path string) (agentruntime.Backend, error) { +func backendFromConfigFile(path string) (agentruntime.Backend, string, error) { data, readErr := os.ReadFile(path) + source := path if readErr != nil && os.IsNotExist(readErr) { alt := filepath.Join(filepath.Dir(path), ".fullsend", "config.yaml") data, readErr = os.ReadFile(alt) + if readErr == nil { + source = alt + } } if readErr == nil { backend, resolveErr := resolveBackendFromConfigData(data) if resolveErr != nil { - return agentruntime.Backend{}, resolveErr + return agentruntime.Backend{}, source, resolveErr } - return backend, nil + return backend, source, nil } if os.IsNotExist(readErr) { - return agentruntime.Default(), nil + return agentruntime.Default(), "default (config not found)", nil } - return agentruntime.Backend{}, fmt.Errorf("reading config.yaml for runtime selection: %w", readErr) + return agentruntime.Backend{}, source, fmt.Errorf("reading config.yaml for runtime selection: %w", readErr) } func newRunCmd() *cobra.Command { @@ -851,7 +855,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // 7. Bootstrap sandbox. var backend agentruntime.Backend orgConfigPath = filepath.Join(absFullsendDir, "config.yaml") - backend, backendErr := backendFromConfigFile(orgConfigPath) + backend, configSource, backendErr := backendFromConfigFile(orgConfigPath) if backendErr != nil { switch { case errors.Is(backendErr, errParsingConfigRuntime): @@ -863,6 +867,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep } return backendErr } + fmt.Fprintf(os.Stderr, "runtime: selected %q from %s\n", backend.Runtime.Name(), configSource) rt := backend.Runtime tx := backend.Transcripts bootstrapStart := time.Now() diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 36e8ee0c16..1ec766ab8b 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3558,8 +3558,9 @@ func TestIsOrgConfigData(t *testing.T) { func TestBackendFromConfigFile_MissingUsesDefault(t *testing.T) { t.Parallel() - backend, err := backendFromConfigFile(filepath.Join(t.TempDir(), "missing.yaml")) + backend, source, err := backendFromConfigFile(filepath.Join(t.TempDir(), "missing.yaml")) require.NoError(t, err) + assert.Equal(t, "default (config not found)", source) assert.Equal(t, "claude", backend.Runtime.Name()) } @@ -3574,7 +3575,7 @@ func TestBackendFromConfigFile_PerRepoConfig(t *testing.T) { path := filepath.Join(dir, "config.yaml") require.NoError(t, os.WriteFile(path, data, 0o644)) - backend, err := backendFromConfigFile(path) + backend, _, err := backendFromConfigFile(path) require.NoError(t, err) assert.Equal(t, "dummy", backend.Runtime.Name()) } @@ -3590,8 +3591,9 @@ func TestBackendFromConfigFile_PerRepoNestedConfig(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(dir, ".fullsend"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(dir, ".fullsend", "config.yaml"), data, 0o644)) - backend, err := backendFromConfigFile(filepath.Join(dir, "config.yaml")) + backend, source, err := backendFromConfigFile(filepath.Join(dir, "config.yaml")) require.NoError(t, err) + assert.Contains(t, source, ".fullsend") assert.Equal(t, "dummy", backend.Runtime.Name()) } @@ -3604,7 +3606,7 @@ func TestBackendFromConfigFile_ReadError(t *testing.T) { path := filepath.Join(t.TempDir(), "config.yaml") require.NoError(t, os.Mkdir(path, 0o755)) - _, err := backendFromConfigFile(path) + _, _, err := backendFromConfigFile(path) require.Error(t, err) assert.Contains(t, err.Error(), "reading config.yaml for runtime selection") } @@ -3620,7 +3622,7 @@ func TestBackendFromConfigFile_ResolveError(t *testing.T) { path := filepath.Join(dir, "config.yaml") require.NoError(t, os.WriteFile(path, data, 0o644)) - _, err = backendFromConfigFile(path) + _, _, err = backendFromConfigFile(path) require.Error(t, err) assert.Contains(t, err.Error(), "resolving runtime") } diff --git a/internal/forge/fake_test.go b/internal/forge/fake_test.go index fc2dfe46df..a18b56adc5 100644 --- a/internal/forge/fake_test.go +++ b/internal/forge/fake_test.go @@ -686,6 +686,10 @@ func TestFakeClient_ErrorInjection(t *testing.T) { _, err := fc.ListDirectoryContents(ctx, "o", "r", "p", "main", false) return err }}, + {"ListRepositoryFiles", func(fc *FakeClient) error { + _, err := fc.ListRepositoryFiles(ctx, "o", "r") + return err + }}, {"GetFileContentAtRef", func(fc *FakeClient) error { _, err := fc.GetFileContentAtRef(ctx, "o", "r", "p", "main") return err @@ -767,6 +771,7 @@ func TestFakeClient_ThreadSafety(t *testing.T) { _, _ = fc.GetOrgVariableRepos(ctx, "o", "n") _ = fc.DeleteIssueComment(ctx, "o", "r", 1) _, _ = fc.ListDirectoryContents(ctx, "o", "r", "p", "main", false) + _, _ = fc.ListRepositoryFiles(ctx, "o", "r") _, _ = fc.GetFileContentAtRef(ctx, "o", "r", "p", "main") _ = fc.DeleteRepoSecret(ctx, "o", "r", "n") _, _ = fc.ListRepoVariables(ctx, "o", "r") @@ -1149,3 +1154,64 @@ func TestFakeClient_DispatchWorkflow(t *testing.T) { fc := &FakeClient{} require.NoError(t, fc.DispatchWorkflow(ctx, "org", "repo", "ci.yml", "main", map[string]string{"k": "v"})) } + +func TestIsTreeTruncated(t *testing.T) { + assert.True(t, IsTreeTruncated(ErrTreeTruncated)) + assert.True(t, IsTreeTruncated(errors.Join(errors.New("wrapper"), ErrTreeTruncated))) + assert.False(t, IsTreeTruncated(errors.New("some other error"))) + assert.False(t, IsTreeTruncated(nil)) +} + +func TestFakeClient_ListRepositoryFiles(t *testing.T) { + ctx := context.Background() + + t.Run("returns files matching owner/repo prefix", func(t *testing.T) { + fc := &FakeClient{ + FileContents: map[string][]byte{ + "org/repo/file1.go": []byte("a"), + "org/repo/dir/f2": []byte("b"), + "org/other/nope": []byte("c"), + }, + } + paths, err := fc.ListRepositoryFiles(ctx, "org", "repo") + require.NoError(t, err) + assert.ElementsMatch(t, []string{"file1.go", "dir/f2"}, paths) + }) + + t.Run("returns nil for empty repo", func(t *testing.T) { + fc := &FakeClient{FileContents: map[string][]byte{}} + paths, err := fc.ListRepositoryFiles(ctx, "org", "repo") + require.NoError(t, err) + assert.Empty(t, paths) + }) + + t.Run("returns injected error", func(t *testing.T) { + fc := &FakeClient{ + Errors: map[string]error{ + "ListRepositoryFiles": ErrTreeTruncated, + }, + } + _, err := fc.ListRepositoryFiles(ctx, "org", "repo") + require.ErrorIs(t, err, ErrTreeTruncated) + }) +} + +func TestFakeClient_ListRepositoryFiles_ConcurrentSafe(t *testing.T) { + fc := &FakeClient{ + FileContents: map[string][]byte{ + "org/repo/a": []byte("a"), + "org/repo/b": []byte("b"), + }, + } + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := fc.ListRepositoryFiles(context.Background(), "org", "repo") + assert.NoError(t, err) + }() + } + wg.Wait() +} From ddffbe50c9a70451e676922e18ab9b3a90b9175a Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Tue, 7 Jul 2026 20:51:16 +0300 Subject: [PATCH 19/22] ci(e2e): re-trigger CI after e2e flake Signed-off-by: Barak Korren Co-authored-by: Cursor From 5c4cd4fa9650d90b49de0d225285b543974c9029 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Wed, 8 Jul 2026 08:11:42 +0300 Subject: [PATCH 20/22] ci(e2e): address Review Squad findings on behaviour suite Pin godog concurrency to 1 with an explicit guard, move dummy runtime test hooks onto DummyRuntime fields, harden artifact zip extraction and inference status JSON parsing, and wrap AddIssueLabels in the GitHub API labels object. Signed-off-by: Barak Korren Co-authored-by: Cursor --- .../drivers/ci/githubactions/githubactions.go | 6 ++ .../ci/githubactions/githubactions_test.go | 21 ++++++ .../drivers/install/perrepo_github.go | 9 ++- .../drivers/install/perrepo_github_test.go | 11 ++- e2e/behaviour/suite_test.go | 16 ++-- internal/forge/github/github.go | 2 +- internal/forge/github/github_test.go | 8 +- internal/runtime/dummy.go | 68 +++++++++++------ internal/runtime/dummy_test.go | 75 ++++++++----------- 9 files changed, 139 insertions(+), 77 deletions(-) diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions.go b/e2e/behaviour/drivers/ci/githubactions/githubactions.go index 24740be899..c5d33d1dd9 100644 --- a/e2e/behaviour/drivers/ci/githubactions/githubactions.go +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions.go @@ -381,6 +381,8 @@ func extractArtifactZip(name string, zipData []byte, destDir string) error { defer zr.Close() const perFileLimit = 10 << 20 + const totalExtractLimit = 100 << 20 + var totalExtracted int64 for _, f := range zr.File { if f.Mode()&os.ModeSymlink != 0 { return fmt.Errorf("artifact zip %q contains symlink entry %q", safeName, f.Name) @@ -407,6 +409,10 @@ func extractArtifactZip(name string, zipData []byte, destDir string) error { if err != nil { return fmt.Errorf("read artifact entry %q: %w", f.Name, err) } + totalExtracted += int64(len(data)) + if totalExtracted > totalExtractLimit { + return fmt.Errorf("artifact zip %q exceeds %d byte aggregate extraction limit", safeName, totalExtractLimit) + } if err := os.WriteFile(outPath, data, 0o644); err != nil { return err } diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go index 1208aa241e..fd1541a256 100644 --- a/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions_test.go @@ -3,6 +3,7 @@ package githubactions import ( "archive/zip" "bytes" + "fmt" "os" "path/filepath" "testing" @@ -102,6 +103,26 @@ func TestExtractArtifactZip_SanitizesName(t *testing.T) { require.NoError(t, err) } +func TestExtractArtifactZip_RejectsAggregateLimit(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + const chunk = 10 << 20 // per-file limit + for i := 0; i < 11; i++ { + w, err := zw.Create(fmt.Sprintf("part-%d.bin", i)) + require.NoError(t, err) + _, err = w.Write(bytes.Repeat([]byte("x"), chunk)) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + + dest := t.TempDir() + err := extractArtifactZip("artifact", buf.Bytes(), dest) + require.Error(t, err) + assert.Contains(t, err.Error(), "aggregate extraction limit") +} + func TestNewestRepositoryArtifactCreatedAt(t *testing.T) { t.Parallel() diff --git a/e2e/behaviour/drivers/install/perrepo_github.go b/e2e/behaviour/drivers/install/perrepo_github.go index 5af5d6fca1..040d54635c 100644 --- a/e2e/behaviour/drivers/install/perrepo_github.go +++ b/e2e/behaviour/drivers/install/perrepo_github.go @@ -115,9 +115,14 @@ func (d *perRepoDriver) provisionPerRepoInference(target, project string) (strin } func parseInferenceStatusWIFProvider(output string) (string, error) { - start := strings.Index(output, "{") + statusKey := `"status":` + keyIdx := strings.Index(output, statusKey) + if keyIdx < 0 { + return "", fmt.Errorf("no JSON status object in output") + } + start := strings.LastIndex(output[:keyIdx], "{") if start < 0 { - return "", fmt.Errorf("no JSON in output") + return "", fmt.Errorf("no JSON status object in output") } var status struct { Status string `json:"status"` diff --git a/e2e/behaviour/drivers/install/perrepo_github_test.go b/e2e/behaviour/drivers/install/perrepo_github_test.go index ea0c4b97a6..b2fa87de49 100644 --- a/e2e/behaviour/drivers/install/perrepo_github_test.go +++ b/e2e/behaviour/drivers/install/perrepo_github_test.go @@ -54,7 +54,16 @@ func TestParseInferenceStatusWIFProvider_OK(t *testing.T) { func TestParseInferenceStatusWIFProvider_NoJSON(t *testing.T) { _, err := parseInferenceStatusWIFProvider("no json here") require.Error(t, err) - assert.Contains(t, err.Error(), "no JSON") + assert.Contains(t, err.Error(), "no JSON status object") +} + +func TestParseInferenceStatusWIFProvider_IgnoresLeadingNoise(t *testing.T) { + out := `Running inference status... +log line with { brace noise +{"status":"healthy","FULLSEND_GCP_WIF_PROVIDER":"projects/1/locations/global/workloadIdentityPools/p/providers/x"}` + got, err := parseInferenceStatusWIFProvider(out) + require.NoError(t, err) + assert.Equal(t, "projects/1/locations/global/workloadIdentityPools/p/providers/x", got) } func TestParseInferenceStatusWIFProvider_Unhealthy(t *testing.T) { diff --git a/e2e/behaviour/suite_test.go b/e2e/behaviour/suite_test.go index 52b88e9ce1..b03bb11c7c 100644 --- a/e2e/behaviour/suite_test.go +++ b/e2e/behaviour/suite_test.go @@ -25,6 +25,9 @@ func TestBehaviourSuite(t *testing.T) { if testing.Short() { t.Skip("skipping behaviour tests in short mode") } + if c := os.Getenv("GODOG_CONCURRENCY"); c != "" && c != "1" { + t.Fatalf("behaviour suite does not support GODOG_CONCURRENCY=%q (shared World); use serial runs until parallel support lands", c) + } cfg := env.LoadRunnerConfig() if err := cfg.Validate(); err != nil { @@ -78,16 +81,17 @@ func TestBehaviourSuite(t *testing.T) { RepoFull: org + "/" + testRepo, } - // World is shared across scenarios; godog runs serially by default. Do not - // pass --concurrency without giving each scenario its own World in Before. + // World is shared across scenarios; pin concurrency to 1 until each scenario + // gets its own World and behaviour script path (see follow-up issue). suite := godog.TestSuite{ Name: "behaviour", ScenarioInitializer: func(sc *godog.ScenarioContext) { initializeScenario(sc, w) }, Options: &godog.Options{ - Format: "pretty", - Paths: []string{"features"}, - TestingT: t, - Tags: os.Getenv("GODOG_TAGS"), + Format: "pretty", + Paths: []string{"features"}, + TestingT: t, + Tags: os.Getenv("GODOG_TAGS"), + Concurrency: 1, }, } if st := suite.Run(); st != 0 { diff --git a/internal/forge/github/github.go b/internal/forge/github/github.go index 497a0fac0e..524dacf1b7 100644 --- a/internal/forge/github/github.go +++ b/internal/forge/github/github.go @@ -2172,7 +2172,7 @@ func (c *LiveClient) AddIssueLabels(ctx context.Context, owner, repo string, num if len(labels) == 0 { return nil } - resp, err := c.post(ctx, fmt.Sprintf("/repos/%s/%s/issues/%d/labels", owner, repo, number), labels) + resp, err := c.post(ctx, fmt.Sprintf("/repos/%s/%s/issues/%d/labels", owner, repo, number), map[string][]string{"labels": labels}) if err != nil { return fmt.Errorf("add issue labels: %w", err) } diff --git a/internal/forge/github/github_test.go b/internal/forge/github/github_test.go index f60ea60404..ab149ae137 100644 --- a/internal/forge/github/github_test.go +++ b/internal/forge/github/github_test.go @@ -2842,9 +2842,11 @@ func TestAddIssueLabels(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "POST", r.Method) assert.Equal(t, "/repos/org/repo/issues/7/labels", r.URL.Path) - var labels []string - require.NoError(t, json.NewDecoder(r.Body).Decode(&labels)) - assert.Equal(t, []string{"ready-for-triage"}, labels) + var body struct { + Labels []string `json:"labels"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + assert.Equal(t, []string{"ready-for-triage"}, body.Labels) w.WriteHeader(http.StatusOK) })) defer srv.Close() diff --git a/internal/runtime/dummy.go b/internal/runtime/dummy.go index d219951fd1..d842bac6d7 100644 --- a/internal/runtime/dummy.go +++ b/internal/runtime/dummy.go @@ -20,14 +20,11 @@ import ( const behaviourScriptRelPath = "behaviour/current-scenario.yaml" const behaviourResultsFile = "behaviour-results.json" -// writeBehaviourResultsFn is the production writer; tests may replace it. -var writeBehaviourResultsFn = writeBehaviourResults +type sandboxExecFunc func(sandboxName, cmd string, timeout time.Duration) (stdout, stderr string, exitCode int, err error) -// sandboxExecFn and sandboxUploadFn are production defaults; tests may replace them. -var ( - sandboxExecFn = sandbox.Exec - sandboxUploadFn = sandbox.Upload -) +type sandboxUploadFunc func(sandboxName, localPath, remotePath string) error + +type writeBehaviourResultsFunc func(sandboxName string, results BehaviourResults) error // BehaviourOperation is a single scripted step for the dummy runtime. type BehaviourOperation struct { @@ -55,7 +52,34 @@ type BehaviourResults struct { } // DummyRuntime executes scripted operations in the real OpenShell sandbox. -type DummyRuntime struct{} +// ExecFn, UploadFn, and WriteResultsFn are optional test overrides; production +// uses sandbox.Exec, sandbox.Upload, and writeBehaviourResults. +type DummyRuntime struct { + ExecFn sandboxExecFunc + UploadFn sandboxUploadFunc + WriteResultsFn writeBehaviourResultsFunc +} + +func (r DummyRuntime) execFn() sandboxExecFunc { + if r.ExecFn != nil { + return r.ExecFn + } + return sandbox.Exec +} + +func (r DummyRuntime) uploadFn() sandboxUploadFunc { + if r.UploadFn != nil { + return r.UploadFn + } + return sandbox.Upload +} + +func (r DummyRuntime) writeResultsFn() writeBehaviourResultsFunc { + if r.WriteResultsFn != nil { + return r.WriteResultsFn + } + return r.writeBehaviourResults +} func (DummyRuntime) Name() string { return "dummy" } @@ -67,10 +91,10 @@ func (DummyRuntime) WorkspaceDir() string { return sandbox.SandboxWorkspace } func (DummyRuntime) EnvExports() []string { return nil } -func (DummyRuntime) Bootstrap(input BootstrapInput) error { +func (r DummyRuntime) Bootstrap(input BootstrapInput) error { sandboxName := input.SandboxName() mkdirCmd := fmt.Sprintf("mkdir -p %s/output %s/.dummy", sandbox.SandboxWorkspace, sandbox.SandboxWorkspace) - _, _, _, err := sandboxExecFn(sandboxName, mkdirCmd, 10*time.Second) + _, _, _, err := r.execFn()(sandboxName, mkdirCmd, 10*time.Second) return err } @@ -81,8 +105,8 @@ func (r DummyRuntime) Run(ctx context.Context, params RunParams, printer *ui.Pri return 1, err } - results, scriptErr := executeBehaviourScript(ctx, params.SandboxName, params.RepoDir, script) - if writeErr := writeBehaviourResultsFn(params.SandboxName, results); writeErr != nil { + results, scriptErr := executeBehaviourScript(ctx, r, params.SandboxName, params.RepoDir, script) + if writeErr := r.writeResultsFn()(params.SandboxName, results); writeErr != nil { return 1, writeErr } @@ -104,7 +128,7 @@ func (r DummyRuntime) Run(ctx context.Context, params RunParams, printer *ui.Pri func (r DummyRuntime) ClearIterationArtifacts(sandboxName string) error { clearCmd := fmt.Sprintf("rm -rf %s/output/*", r.WorkspaceDir()) - _, _, _, err := sandboxExecFn(sandboxName, clearCmd, 10*time.Second) + _, _, _, err := r.execFn()(sandboxName, clearCmd, 10*time.Second) return err } @@ -138,7 +162,7 @@ func LoadBehaviourScript(path string) (*BehaviourScript, error) { return &script, nil } -func executeBehaviourScript(ctx context.Context, sandboxName, repoDir string, script *BehaviourScript) (BehaviourResults, error) { +func executeBehaviourScript(ctx context.Context, rt DummyRuntime, sandboxName, repoDir string, script *BehaviourScript) (BehaviourResults, error) { var results BehaviourResults var firstErr error for _, op := range script.Ops { @@ -146,7 +170,7 @@ func executeBehaviourScript(ctx context.Context, sandboxName, repoDir string, sc return results, fmt.Errorf("behaviour script cancelled: %w", err) } res := BehaviourOpResult{Description: op.Description} - if err := executeBehaviourOp(sandboxName, repoDir, op); err != nil { + if err := executeBehaviourOp(rt, sandboxName, repoDir, op); err != nil { res.Success = false res.Error = err.Error() if firstErr == nil { @@ -160,7 +184,7 @@ func executeBehaviourScript(ctx context.Context, sandboxName, repoDir string, sc return results, firstErr } -func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) error { +func executeBehaviourOp(rt DummyRuntime, sandboxName, repoDir string, op BehaviourOperation) error { switch op.Op { case "read_file": path := strings.TrimSpace(op.Args) @@ -172,7 +196,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro return err } cmd := fmt.Sprintf("test -r %s", shellQuote(remotePath)) - _, stderr, exitCode, err := sandboxExecFn(sandboxName, cmd, 30*time.Second) + _, stderr, exitCode, err := rt.execFn()(sandboxName, cmd, 30*time.Second) if err != nil { return fmt.Errorf("read_file exec: %w", err) } @@ -189,7 +213,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro return err } cmd := fmt.Sprintf("curl -sf -- %s -o /dev/null", shellQuote(rawURL)) - _, stderr, exitCode, err := sandboxExecFn(sandboxName, cmd, 60*time.Second) + _, stderr, exitCode, err := rt.execFn()(sandboxName, cmd, 60*time.Second) if err != nil { return fmt.Errorf("url_get exec: %w", err) } @@ -208,7 +232,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro } parentDir := filepath.Dir(remoteDest) mkdirCmd := fmt.Sprintf("mkdir -p %s", shellQuote(parentDir)) - if _, _, _, err := sandboxExecFn(sandboxName, mkdirCmd, 10*time.Second); err != nil { + if _, _, _, err := rt.execFn()(sandboxName, mkdirCmd, 10*time.Second); err != nil { return fmt.Errorf("write_fixture mkdir: %w", err) } tmp, err := os.CreateTemp("", "behaviour-fixture-*") @@ -221,7 +245,7 @@ func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) erro return fmt.Errorf("write_fixture write temp: %w", err) } tmp.Close() - if err := sandboxUploadFn(sandboxName, tmp.Name(), remoteDest); err != nil { + if err := rt.uploadFn()(sandboxName, tmp.Name(), remoteDest); err != nil { return fmt.Errorf("write_fixture upload: %w", err) } return nil @@ -278,7 +302,7 @@ func resolveSandboxPath(base, rel string) (string, error) { return resolved, nil } -func writeBehaviourResults(sandboxName string, results BehaviourResults) error { +func (r DummyRuntime) writeBehaviourResults(sandboxName string, results BehaviourResults) error { data, err := json.MarshalIndent(results, "", " ") if err != nil { return fmt.Errorf("marshaling behaviour results: %w", err) @@ -294,7 +318,7 @@ func writeBehaviourResults(sandboxName string, results BehaviourResults) error { } tmp.Close() remotePath := filepath.Join(sandbox.SandboxWorkspace, "output", behaviourResultsFile) - return sandboxUploadFn(sandboxName, tmp.Name(), remotePath) + return r.uploadFn()(sandboxName, tmp.Name(), remotePath) } func shellQuote(s string) string { diff --git a/internal/runtime/dummy_test.go b/internal/runtime/dummy_test.go index 478bbade51..9b3f86db35 100644 --- a/internal/runtime/dummy_test.go +++ b/internal/runtime/dummy_test.go @@ -94,7 +94,7 @@ func TestResolveSandboxPathRejectsAbsoluteOutsideWorkspace(t *testing.T) { func TestExecuteBehaviourOpUnknown(t *testing.T) { t.Parallel() - err := executeBehaviourOp("unused", t.TempDir(), BehaviourOperation{Op: "nope"}) + err := executeBehaviourOp(DummyRuntime{}, "unused", t.TempDir(), BehaviourOperation{Op: "nope"}) require.Error(t, err) assert.Contains(t, err.Error(), "unknown op") } @@ -192,7 +192,7 @@ func TestExecuteBehaviourScript_ValidationFailures(t *testing.T) { {Op: "read_file", Description: "missing path"}, {Op: "url_get", Description: "missing url"}, }} - results, err := executeBehaviourScript(context.Background(), "unused", t.TempDir(), script) + results, err := executeBehaviourScript(context.Background(), DummyRuntime{}, "unused", t.TempDir(), script) require.Error(t, err) require.Contains(t, err.Error(), "requires a path") require.Len(t, results.Operations, 2) @@ -205,7 +205,7 @@ func TestExecuteBehaviourScript_ValidationFailures(t *testing.T) { func TestExecuteBehaviourOp_ReadFileEmptyPath(t *testing.T) { t.Parallel() - err := executeBehaviourOp("unused", t.TempDir(), BehaviourOperation{Op: "read_file"}) + err := executeBehaviourOp(DummyRuntime{}, "unused", t.TempDir(), BehaviourOperation{Op: "read_file"}) require.Error(t, err) assert.Contains(t, err.Error(), "requires a path") } @@ -213,7 +213,7 @@ func TestExecuteBehaviourOp_ReadFileEmptyPath(t *testing.T) { func TestExecuteBehaviourOp_URLGetEmpty(t *testing.T) { t.Parallel() - err := executeBehaviourOp("unused", t.TempDir(), BehaviourOperation{Op: "url_get"}) + err := executeBehaviourOp(DummyRuntime{}, "unused", t.TempDir(), BehaviourOperation{Op: "url_get"}) require.Error(t, err) assert.Contains(t, err.Error(), "requires a URL") } @@ -273,7 +273,7 @@ func TestExecuteBehaviourOp_ReadFileExecFailure(t *testing.T) { base := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(base, "f.txt"), []byte("x"), 0o644)) - err := executeBehaviourOp("nonexistent-sandbox", base, BehaviourOperation{ + err := executeBehaviourOp(DummyRuntime{}, "nonexistent-sandbox", base, BehaviourOperation{ Op: "read_file", Args: "f.txt", }) @@ -283,7 +283,7 @@ func TestExecuteBehaviourOp_ReadFileExecFailure(t *testing.T) { func TestExecuteBehaviourOp_URLGetExecFailure(t *testing.T) { t.Parallel() - err := executeBehaviourOp("nonexistent-sandbox", t.TempDir(), BehaviourOperation{ + err := executeBehaviourOp(DummyRuntime{}, "nonexistent-sandbox", t.TempDir(), BehaviourOperation{ Op: "url_get", Args: "https://example.com", }) @@ -293,7 +293,7 @@ func TestExecuteBehaviourOp_URLGetExecFailure(t *testing.T) { func TestExecuteBehaviourOp_WriteFixtureInvalidArgs(t *testing.T) { t.Parallel() - err := executeBehaviourOp("nonexistent-sandbox", t.TempDir(), BehaviourOperation{ + err := executeBehaviourOp(DummyRuntime{}, "nonexistent-sandbox", t.TempDir(), BehaviourOperation{ Op: "write_fixture", Args: "only-one-part", }) @@ -303,16 +303,13 @@ func TestExecuteBehaviourOp_WriteFixtureInvalidArgs(t *testing.T) { func TestWriteBehaviourResults(t *testing.T) { t.Parallel() - err := writeBehaviourResults("nonexistent-sandbox", BehaviourResults{ + err := DummyRuntime{}.writeBehaviourResults("nonexistent-sandbox", BehaviourResults{ Operations: []BehaviourOpResult{{Success: true, Description: "ok"}}, }) require.Error(t, err) } func TestDummyRuntime_RunFailedOps(t *testing.T) { - orig := writeBehaviourResultsFn - writeBehaviourResultsFn = writeBehaviourResults - t.Cleanup(func() { writeBehaviourResultsFn = orig }) fullsendDir := t.TempDir() scriptDir := filepath.Join(fullsendDir, "behaviour") require.NoError(t, os.MkdirAll(scriptDir, 0o755)) @@ -334,10 +331,6 @@ func TestDummyRuntime_RunFailedOps(t *testing.T) { } func TestDummyRuntime_RunOpFailureReturnsNilGoError(t *testing.T) { - orig := writeBehaviourResultsFn - writeBehaviourResultsFn = func(string, BehaviourResults) error { return nil } - t.Cleanup(func() { writeBehaviourResultsFn = orig }) - fullsendDir := t.TempDir() scriptDir := filepath.Join(fullsendDir, "behaviour") require.NoError(t, os.MkdirAll(scriptDir, 0o755)) @@ -348,7 +341,7 @@ func TestDummyRuntime_RunOpFailureReturnsNilGoError(t *testing.T) { ` require.NoError(t, os.WriteFile(filepath.Join(scriptDir, "current-scenario.yaml"), []byte(script), 0o644)) - rt := DummyRuntime{} + rt := DummyRuntime{WriteResultsFn: func(string, BehaviourResults) error { return nil }} exit, err := rt.Run(context.Background(), RunParams{ SandboxName: "nonexistent-sandbox", RepoDir: t.TempDir(), @@ -359,16 +352,16 @@ func TestDummyRuntime_RunOpFailureReturnsNilGoError(t *testing.T) { } func TestExecuteBehaviourOp_URLGetNonZeroExit(t *testing.T) { - orig := sandboxExecFn - sandboxExecFn = func(_ string, cmd string, _ time.Duration) (string, string, int, error) { + t.Parallel() + + rt := DummyRuntime{ExecFn: func(_ string, cmd string, _ time.Duration) (string, string, int, error) { if strings.Contains(cmd, "curl") { return "", "blocked by sandbox policy", 22, nil } return "", "", 0, nil - } - t.Cleanup(func() { sandboxExecFn = orig }) + }} - err := executeBehaviourOp("sandbox", t.TempDir(), BehaviourOperation{ + err := executeBehaviourOp(rt, "sandbox", t.TempDir(), BehaviourOperation{ Op: "url_get", Args: "https://www.google.com/search?q=foo", }) @@ -378,13 +371,13 @@ func TestExecuteBehaviourOp_URLGetNonZeroExit(t *testing.T) { } func TestExecuteBehaviourOp_ReadFileNonZeroExit(t *testing.T) { - orig := sandboxExecFn - sandboxExecFn = func(_ string, _ string, _ time.Duration) (string, string, int, error) { + t.Parallel() + + rt := DummyRuntime{ExecFn: func(_ string, _ string, _ time.Duration) (string, string, int, error) { return "", "missing file", 1, nil - } - t.Cleanup(func() { sandboxExecFn = orig }) + }} - err := executeBehaviourOp("sandbox", t.TempDir(), BehaviourOperation{ + err := executeBehaviourOp(rt, "sandbox", t.TempDir(), BehaviourOperation{ Op: "read_file", Args: "output/missing.json", }) @@ -393,18 +386,16 @@ func TestExecuteBehaviourOp_ReadFileNonZeroExit(t *testing.T) { } func TestExecuteBehaviourOp_WriteFixtureSuccess(t *testing.T) { - origExec := sandboxExecFn - origUpload := sandboxUploadFn - sandboxExecFn = func(_ string, _ string, _ time.Duration) (string, string, int, error) { - return "", "", 0, nil + t.Parallel() + + rt := DummyRuntime{ + ExecFn: func(_ string, _ string, _ time.Duration) (string, string, int, error) { + return "", "", 0, nil + }, + UploadFn: func(_, _, _ string) error { return nil }, } - sandboxUploadFn = func(_, _, _ string) error { return nil } - t.Cleanup(func() { - sandboxExecFn = origExec - sandboxUploadFn = origUpload - }) - err := executeBehaviourOp("sandbox", t.TempDir(), BehaviourOperation{ + err := executeBehaviourOp(rt, "sandbox", t.TempDir(), BehaviourOperation{ Op: "write_fixture", Args: "output/agent-result.json, fixtures/triage/sufficient.json", Content: `{"action":"sufficient"}`, @@ -413,15 +404,15 @@ func TestExecuteBehaviourOp_WriteFixtureSuccess(t *testing.T) { } func TestWriteBehaviourResultsSuccess(t *testing.T) { - orig := sandboxUploadFn + t.Parallel() + var uploaded bool - sandboxUploadFn = func(_, _, _ string) error { + rt := DummyRuntime{UploadFn: func(_, _, _ string) error { uploaded = true return nil - } - t.Cleanup(func() { sandboxUploadFn = orig }) + }} - err := writeBehaviourResults("sandbox", BehaviourResults{ + err := rt.writeBehaviourResults("sandbox", BehaviourResults{ Operations: []BehaviourOpResult{{Description: "ok", Success: true}}, }) require.NoError(t, err) @@ -445,7 +436,7 @@ func TestExecuteBehaviourScript_CancelledContext(t *testing.T) { script := &BehaviourScript{Ops: []BehaviourOperation{ {Op: "read_file", Args: "x.txt", Description: "read"}, }} - _, err := executeBehaviourScript(ctx, "sandbox", t.TempDir(), script) + _, err := executeBehaviourScript(ctx, DummyRuntime{}, "sandbox", t.TempDir(), script) require.Error(t, err) assert.Contains(t, err.Error(), "cancelled") } From 699beb8d6b34189904cdaf9e8b99d5841064a964 Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Wed, 8 Jul 2026 21:59:57 +0300 Subject: [PATCH 21/22] ci(e2e): preserve CI behaviour artifacts for upload Skip per-scenario cleanup when ArtifactDir lives under BEHAVIOUR_ARTIFACT_DIR so failure uploads retain downloaded artifacts. Signed-off-by: Barak Korren Co-authored-by: Cursor --- e2e/behaviour/steps/cleanup.go | 29 +++++++++++++++++++++++------ e2e/behaviour/steps/cleanup_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 e2e/behaviour/steps/cleanup_test.go diff --git a/e2e/behaviour/steps/cleanup.go b/e2e/behaviour/steps/cleanup.go index a6dab36a45..47519a91dd 100644 --- a/e2e/behaviour/steps/cleanup.go +++ b/e2e/behaviour/steps/cleanup.go @@ -3,6 +3,7 @@ package steps import ( "context" "os" + "path/filepath" "strings" "github.com/fullsend-ai/fullsend/e2e/behaviour/world" @@ -15,12 +16,9 @@ func CleanupScenario(w *world.World) { worldLogf(w, "behaviour cleanup: close issue #%d: %v", w.IssueNumber, err) } } - if w.ArtifactDir != "" { - ciArtifactDir := strings.TrimSpace(os.Getenv("BEHAVIOUR_ARTIFACT_DIR")) - if ciArtifactDir == "" || w.ArtifactDir != ciArtifactDir { - if err := os.RemoveAll(w.ArtifactDir); err != nil { - worldLogf(w, "behaviour cleanup: remove artifact dir: %v", err) - } + if w.ArtifactDir != "" && shouldRemoveArtifactDir(w.ArtifactDir, os.Getenv("BEHAVIOUR_ARTIFACT_DIR")) { + if err := os.RemoveAll(w.ArtifactDir); err != nil { + worldLogf(w, "behaviour cleanup: remove artifact dir: %v", err) } } if len(w.DummyOps) > 0 { @@ -31,6 +29,25 @@ func CleanupScenario(w *world.World) { } } +// shouldRemoveArtifactDir reports whether cleanup may delete artifactDir. +// Dirs under BEHAVIOUR_ARTIFACT_DIR are preserved for CI upload-artifact. +func shouldRemoveArtifactDir(artifactDir, ciArtifactDir string) bool { + ciArtifactDir = strings.TrimSpace(ciArtifactDir) + if ciArtifactDir == "" { + return true + } + return !artifactDirUnderCIRoot(artifactDir, ciArtifactDir) +} + +func artifactDirUnderCIRoot(dir, ciRoot string) bool { + cleanDir := filepath.Clean(dir) + cleanRoot := filepath.Clean(ciRoot) + if cleanDir == cleanRoot { + return true + } + return strings.HasPrefix(cleanDir, cleanRoot+string(os.PathSeparator)) +} + func worldLogf(w *world.World, format string, args ...any) { if w.Logf != nil { w.Logf(format, args...) diff --git a/e2e/behaviour/steps/cleanup_test.go b/e2e/behaviour/steps/cleanup_test.go new file mode 100644 index 0000000000..35a332e6f8 --- /dev/null +++ b/e2e/behaviour/steps/cleanup_test.go @@ -0,0 +1,27 @@ +package steps + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShouldRemoveArtifactDir(t *testing.T) { + t.Parallel() + + ciRoot := "/tmp/behaviour-artifacts" + assert.False(t, shouldRemoveArtifactDir(ciRoot, ciRoot)) + assert.False(t, shouldRemoveArtifactDir(ciRoot+"/run-123", ciRoot)) + assert.True(t, shouldRemoveArtifactDir("/tmp/behaviour-artifacts-evil/run-123", ciRoot)) + assert.True(t, shouldRemoveArtifactDir("/var/tmp/local-run", ciRoot)) + assert.True(t, shouldRemoveArtifactDir("/tmp/local-run", "")) +} + +func TestArtifactDirUnderCIRoot(t *testing.T) { + t.Parallel() + + ciRoot := "/tmp/behaviour-artifacts" + assert.True(t, artifactDirUnderCIRoot(ciRoot, ciRoot)) + assert.True(t, artifactDirUnderCIRoot(ciRoot+"/run-456", ciRoot)) + assert.False(t, artifactDirUnderCIRoot("/tmp/behaviour-artifacts-evil/run", ciRoot)) +} From 0b5732587b5f3fefd25ceef9606adb41d095e42d Mon Sep 17 00:00:00 2001 From: Barak Korren Date: Wed, 8 Jul 2026 22:06:29 +0300 Subject: [PATCH 22/22] docs(adr): renumber behaviour ADR to 0066 Main already has ADR 0065 (provider-backed policy composition); renumber the behaviour-tests ADR and update cross-references. Signed-off-by: Barak Korren Co-authored-by: Cursor --- docs/ADRs/0003-org-config-repo-convention.md | 2 +- ...rs.md => 0066-behaviour-tests-with-gherkin-and-drivers.md} | 4 ++-- docs/architecture.md | 2 +- docs/guides/dev/behaviour-testing.md | 2 +- docs/problems/testing-agents.md | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename docs/ADRs/{0065-behaviour-tests-with-gherkin-and-drivers.md => 0066-behaviour-tests-with-gherkin-and-drivers.md} (96%) diff --git a/docs/ADRs/0003-org-config-repo-convention.md b/docs/ADRs/0003-org-config-repo-convention.md index e39e0b279e..a2ecec3b33 100644 --- a/docs/ADRs/0003-org-config-repo-convention.md +++ b/docs/ADRs/0003-org-config-repo-convention.md @@ -228,7 +228,7 @@ sandbox: filesystem: ephemeral ``` -Backend selection for behaviour tests (`defaults.runtime` / per-repo `runtime: dummy`) is documented in [ADR 0065](0065-behaviour-tests-with-gherkin-and-drivers.md), not in the org config example above. +Backend selection for behaviour tests (`defaults.runtime` / per-repo `runtime: dummy`) is documented in [ADR 0066](0066-behaviour-tests-with-gherkin-and-drivers.md), not in the org config example above. Per-repo overrides in `repos/.yaml` can override org defaults (within the bounds of org-wide guardrails that cannot be weakened). diff --git a/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md b/docs/ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md similarity index 96% rename from docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md rename to docs/ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md index 699591a11c..55f7f7b372 100644 --- a/docs/ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md +++ b/docs/ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md @@ -1,5 +1,5 @@ --- -title: "65. Behaviour tests with Gherkin and pluggable drivers" +title: "66. Behaviour tests with Gherkin and pluggable drivers" status: Accepted relates_to: - agent-infrastructure @@ -10,7 +10,7 @@ topics: - behaviour-tests --- -# 65. Behaviour tests with Gherkin and pluggable drivers +# 66. Behaviour tests with Gherkin and pluggable drivers Date: 2026-06-07 diff --git a/docs/architecture.md b/docs/architecture.md index 9c2cf85756..593ebcf8be 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -132,7 +132,7 @@ This is the thing that actually reasons and acts. Everything else in this docume ### Behaviour testing -End-to-end **behaviour tests** under `e2e/behaviour/` validate deterministic platform code — dispatch routing, harness loading, sandbox policy, SCM mutations — with the LLM layer removed via the dummy runtime. Tests exercise real GitHub and GitHub Actions through pluggable SCM and CI drivers; Gherkin scenarios stay install-mode agnostic while runner env vars select backends. This coverage is **orthogonal** to LLM and instruction testing in [testing-agents.md](problems/testing-agents.md). See [ADR 0065](ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md). +End-to-end **behaviour tests** under `e2e/behaviour/` validate deterministic platform code — dispatch routing, harness loading, sandbox policy, SCM mutations — with the LLM layer removed via the dummy runtime. Tests exercise real GitHub and GitHub Actions through pluggable SCM and CI drivers; Gherkin scenarios stay install-mode agnostic while runner env vars select backends. This coverage is **orthogonal** to LLM and instruction testing in [testing-agents.md](problems/testing-agents.md). See [ADR 0066](ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md). **Open questions:** diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index bd7ae6fafe..d27dca4437 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -82,4 +82,4 @@ E2E_GCP_WIF_PROVIDER=... # CI job GCP auth (not written to pool test-repo s Triage scenarios apply the `ready-for-triage` label (not `/fs-triage` comments) because the per-repo shim ignores `issue_comment` events from bot users and CI uses minted e2e installation tokens. -See [behaviour-drivers.md](behaviour-drivers.md) for driver configuration and [ADR 0065](../../ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md) for the decision record. +See [behaviour-drivers.md](behaviour-drivers.md) for driver configuration and [ADR 0066](../../ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md) for the decision record. diff --git a/docs/problems/testing-agents.md b/docs/problems/testing-agents.md index 73f50c4c89..8c7547b6d3 100644 --- a/docs/problems/testing-agents.md +++ b/docs/problems/testing-agents.md @@ -8,7 +8,7 @@ Testing application code is a solved problem with mature tooling: unit tests, in Today, if someone modifies a review agent's instructions, the only verification is human review of the prose change. There is no automated way to confirm the agent still behaves correctly after the modification. This is the equivalent of shipping code changes with no test suite — something we would never accept for application code. -**Behaviour tests are orthogonal.** [ADR 0065](../ADRs/0065-behaviour-tests-with-gherkin-and-drivers.md) describes Gherkin end-to-end tests under `e2e/behaviour/` that validate deterministic platform code (workflows, harness, sandbox policy, post-scripts) with inference explicitly removed via the dummy runtime. They do not evaluate instructions, prompts, or models, and they do not replace golden-set or statistical LLM evals described below. +**Behaviour tests are orthogonal.** [ADR 0066](../ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md) describes Gherkin end-to-end tests under `e2e/behaviour/` that validate deterministic platform code (workflows, harness, sandbox policy, post-scripts) with inference explicitly removed via the dummy runtime. They do not evaluate instructions, prompts, or models, and they do not replace golden-set or statistical LLM evals described below. ## What makes agent testing hard