diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f7ba8ea7de..fb876f6583 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -96,3 +96,34 @@ jobs: path: ${{ runner.temp }}/e2e-screenshots/ if-no-files-found: ignore retention-days: 5 + + behaviour: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Check for behaviour test token + id: behaviour-secrets + run: | + if [ -z "$E2E_BEHAVIOUR_GITHUB_TOKEN" ]; then + echo "::warning::E2E_BEHAVIOUR_GITHUB_TOKEN not set. Skipping behaviour tests." + echo "available=false" >> "$GITHUB_OUTPUT" + else + echo "available=true" >> "$GITHUB_OUTPUT" + fi + env: + E2E_BEHAVIOUR_GITHUB_TOKEN: ${{ secrets.E2E_BEHAVIOUR_GITHUB_TOKEN }} + + - name: Run behaviour tests + if: steps.behaviour-secrets.outputs.available == 'true' + run: make behaviour-test + env: + GITHUB_TOKEN: ${{ secrets.E2E_BEHAVIOUR_GITHUB_TOKEN }} + BEHAVIOUR_SCM: github + BEHAVIOUR_CI: githubactions + BEHAVIOUR_INSTALL_MODE: per-org diff --git a/Makefile b/Makefile index 9939839760..fb26a75dfb 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ .PHONY: help bootstrap lint lint-all check fmt \ mindmap go-build go-test go-lint go-fmt go-vet go-tidy \ lint-md-links script-test test \ - e2e-test e2e-playwright e2e-export-session e2e-upload-session + e2e-test e2e-playwright e2e-export-session e2e-upload-session behaviour-test # Let Go automatically download the toolchain version required by go.mod. # This ensures local builds use the right version without manual intervention. @@ -30,6 +30,7 @@ help: @echo " e2e-test - Run admin e2e tests (requires E2E_GITHUB_SESSION_FILE or E2E_GITHUB_USERNAME + E2E_GITHUB_PASSWORD)" @echo " e2e-export-session - Login to GitHub and export a Playwright session file" @echo " e2e-upload-session - Export session and upload it as a GitHub repo secret" + @echo " behaviour-test - Run Gherkin behaviour tests (requires GITHUB_TOKEN and behaviour org pool)" # Install all development tools needed for linting, formatting, and pre-commit hooks. # Prerequisites: uv (https://docs.astral.sh/uv/) and go (https://go.dev/) @@ -132,6 +133,13 @@ e2e-test: e2e-playwright fi; \ go test -tags e2e -v -count=1 -timeout 30m ./e2e/admin/ +behaviour-test: + @if [ -z "$$GITHUB_TOKEN" ] && [ -z "$$GH_TOKEN" ]; then \ + echo "GITHUB_TOKEN or GH_TOKEN is required for behaviour tests"; \ + exit 1; \ + fi + cd e2e/behaviour && go test -tags behaviour -v -count=1 -timeout 30m . + e2e-export-session: e2e-playwright @if [ -n "$$E2E_GITHUB_PASSWORD_FILE" ] && [ -z "$$E2E_GITHUB_PASSWORD" ]; then \ export E2E_GITHUB_PASSWORD="$$(cat "$$E2E_GITHUB_PASSWORD_FILE")"; \ diff --git a/docs/ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md b/docs/ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md new file mode 100644 index 0000000000..7fd1025595 --- /dev/null +++ b/docs/ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md @@ -0,0 +1,30 @@ +--- +status: Accepted +date: 2026-06-07 +relates_to: + - agent-infrastructure + - agent-architecture +--- + +# Behaviour tests with Gherkin and pluggable drivers + +## Context + +Fullsend needs end-to-end tests that validate **deterministic platform behaviour** — dispatch routing, harness loading, schema validation, post-scripts, token scoping, sandbox policy, and SCM mutations — without depending on LLM output. This is distinct from admin install e2e ([ADR 0040](0040-org-pool-for-parallel-e2e-tests.md)) and from LLM/instruction testing ([testing-agents.md](../problems/testing-agents.md)). + +Runtime selection is shared with production via `defaults.runtime` in org `config.yaml` ([runtimes.md](../runtimes.md)). Harness definitions remain as in [ADR 0024](0024-harness-definitions.md). Per-repo install mode ([ADR 0033](0033-per-repo-installation-mode.md)) is deferred for behaviour v1. + +## Decision + +- Add **behaviour tests** under `e2e/behaviour/` using **godog** and portable Gherkin feature files. +- Exercise **real SCM + real CI** through **driver interfaces** (`scm.Driver`, `ci.Driver`, `env.Setup`); v1 implementations target GitHub and GitHub Actions. +- Substitute inference with a **dummy runtime** (`defaults.runtime: dummy`) that executes scripted operations in the real OpenShell sandbox and emits `behaviour-results.json`. +- Select backends via **runner env** (`BEHAVIOUR_SCM`, `BEHAVIOUR_CI`, `BEHAVIOUR_INSTALL_MODE`); feature files stay install-mode agnostic. v1 runs **per-org only** against the halfsend org pool. +- Use **compatibility tags** (`@skip:*`, `@requires:*`) to filter scenarios for future backends; tags do not select configuration. + +## Consequences + +- Behaviour tests can pass while prompt quality regresses; LLM evals remain necessary for instruction coverage. +- Behaviour orgs must be installed with `--runtime dummy`; production orgs must not use dummy unintentionally. +- Adding GitLab or Tekton requires new drivers and runner env values, not feature file rewrites. +- Dummy runtime op vocabulary stays minimal; new ops require runtime + docs updates when scenarios need them. diff --git a/docs/architecture.md b/docs/architecture.md index 7127a40460..596617068a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -100,7 +100,11 @@ This is the thing that actually reasons and acts. Everything else in this docume **Decided (implementation):** -- The `fullsend run` runner delegates in-sandbox agent execution to a `runtime.Runtime` interface; the MVP registers Claude Code only. Bootstrap uses a portable `BootstrapInput` interface with optional extensions such as `ClaudeHooksBootstrap` for sandbox tool hooks. Transcript and debug artifact handling use a separate `TranscriptHandler` interface. See [runtimes.md](runtimes.md) for the per-runtime security feature matrix required when adding a new backend. +- The `fullsend run` runner delegates in-sandbox agent execution to a `runtime.Runtime` interface; production orgs default to Claude Code. Runtime selection is configured in `defaults.runtime` on the org `config.yaml` and resolved via `runtime.ResolveFromConfig()`. A **dummy** runtime executes scripted operations in the real OpenShell sandbox for behaviour tests (inference removed). Bootstrap uses a portable `BootstrapInput` interface with optional extensions such as `ClaudeHooksBootstrap` for sandbox tool hooks. Transcript and debug artifact handling use a separate `TranscriptHandler` interface. See [runtimes.md](runtimes.md) for the per-runtime security feature matrix required when adding a new backend. + +### Behaviour testing + +End-to-end **behaviour tests** under `e2e/behaviour/` validate deterministic platform code — dispatch routing, harness loading, sandbox policy, SCM mutations — with the LLM layer removed via the dummy runtime. Tests exercise real GitHub and GitHub Actions through pluggable SCM and CI drivers; Gherkin scenarios stay install-mode agnostic while runner env vars select backends. This coverage is **orthogonal** to LLM and instruction testing in [testing-agents.md](problems/testing-agents.md). See [ADR 0043](ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md). **Open questions:** diff --git a/docs/guides/README.md b/docs/guides/README.md index 5987135b98..3e72b1ab51 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -35,4 +35,6 @@ Guides for contributors developing and testing fullsend itself. - [Local development](dev/local-dev.md) — Run fullsend agents locally on macOS and Linux (amd64 + arm64) - [CLI internals](dev/cli-internals.md) — Command structure, installation pipeline, and sandbox runtime +- [Behaviour testing](dev/behaviour-testing.md) — Write Gherkin scenarios for end-to-end agent behaviour +- [Behaviour test drivers](dev/behaviour-drivers.md) — Implement SCM and CI drivers for behaviour tests - [Testing workflow changes](dev/testing-workflows.md) — Point a live GitHub org at a branch to test workflow, action, and agent changes before release diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md new file mode 100644 index 0000000000..fa0e950060 --- /dev/null +++ b/docs/guides/dev/behaviour-drivers.md @@ -0,0 +1,58 @@ +# Behaviour test drivers + +Behaviour tests isolate forge-specific code behind drivers so Gherkin scenarios stay portable. + +## Interfaces + +| Interface | Package | Responsibility | +|-----------|---------|----------------| +| `scm.Driver` | `e2e/behaviour/drivers/scm` | Issues, comments, labels (via GetIssue), file commits | +| `ci.Driver` | `e2e/behaviour/drivers/ci` | Workflow polling, logs, artifact download | +| `env.Setup` | `e2e/behaviour/drivers/env` | Validate org pool org has per-org install + enrolled test repo | + +v1 reference implementations: + +- `e2e/behaviour/drivers/scm/github/` +- `e2e/behaviour/drivers/ci/githubactions/` +- `e2e/behaviour/drivers/env/` (`PerOrg`) + +## Runner configuration + +Set when starting the suite (not in feature files): + +``` +BEHAVIOUR_SCM=github # future: gitlab, forgejo +BEHAVIOUR_CI=githubactions # future: tekton, gitlabci +BEHAVIOUR_INSTALL_MODE=per-org # v1 default and only supported value +``` + +The suite in `e2e/behaviour/suite_test.go` reads these env vars, validates them, and constructs concrete drivers. + +## Adding an SCM driver + +1. Implement `scm.Driver` in `e2e/behaviour/drivers/scm//`. +2. Register the driver in `suite_test.go` when `BEHAVIOUR_SCM=`. +3. Document the env var value here. +4. Add `@skip:` tags on scenarios that cannot run until the driver is complete. + +Use `forge.Client` for operations it already exposes; add REST helpers inside the driver package only when necessary (e.g. `GetIssue` with labels). + +## Adding a CI driver + +1. Implement `ci.Driver` — `WaitForWorkflow`, `AssertNoWorkflow`, `GetRunLogs`, `DownloadArtifacts`. +2. Map forge `WorkflowRun` types to portable polling logic; reuse patterns from `e2e/admin/admin_test.go`. +3. Register in suite init for the matching `BEHAVIOUR_CI` value. + +## Step definitions + +Steps must **not** import `internal/forge/github` directly — only drivers. This keeps scenarios vendor-agnostic. + +## Testing drivers + +Prefer unit tests with `httptest` for REST helpers. Optional smoke scenarios against live backends mirror admin e2e credentials (`GITHUB_TOKEN`, halfsend org pool). + +## Future backends checklist + +- [ ] GitLab SCM driver + `@skip:gitlab` tag removal +- [ ] Tekton or GitLab CI driver +- [ ] Per-repo install mode matrix + `@requires:per-repo` scenarios diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md new file mode 100644 index 0000000000..5fd6c1b697 --- /dev/null +++ b/docs/guides/dev/behaviour-testing.md @@ -0,0 +1,79 @@ +# Behaviour testing + +End-to-end Gherkin tests under `e2e/behaviour/` validate **deterministic platform code** with inference removed. They are **orthogonal** to LLM and instruction testing in [testing-agents.md](../../problems/testing-agents.md) and to admin install e2e in `e2e/admin/`. + +| | Behaviour tests | LLM evals | Admin e2e | Unit tests | +|---|-----------------|-----------|-----------|------------| +| **Target** | Platform workflows, sandbox, SCM | Prompts, models | Install/uninstall | Go functions | +| **Inference** | Dummy runtime | Real LLM | Real LLM | N/A | +| **Infrastructure** | Live GitHub + GHA | Varies | Live GitHub + GHA | None | + +## When to add a behaviour test + +Add one when a **user-visible workflow** must be verified end-to-end (dispatch → workflow → post-script → SCM state) and the assertion is **binary**. Prefer unit tests for pure Go logic and admin e2e for install provisioning. + +## Layout + +``` +e2e/behaviour/ + features/ # Portable Gherkin scenarios + fixtures/ # Static content for write_fixture ops + steps/ # Step definitions + world/ # Scenario state + drivers/ # SCM, CI, env interfaces + v1 impls + suite_test.go # godog entry (build tag: behaviour) +``` + +## Writing scenarios + +Describe **user-visible behaviour** only. Do not encode SCM vendor, CI platform, or install mode in feature files. + +### Dummy agent tables + +```gherkin +Given a dummy agent that would: + | description | op | args | + | Emit triage JSON | write_fixture | output/agent-result.json, fixtures/triage/sufficient.json | +``` + +| Column | Meaning | +|--------|---------| +| `description` | Human label matched by assertion steps | +| `op` | `read_file`, `url_get`, `run_command`, `write_fixture` | +| `args` | Op-specific; see below | + +**`write_fixture`:** `dest_path, fixtures/...` — content lives in `e2e/behaviour/fixtures/`, embedded in the committed scenario script at `.fullsend/behaviour/current-scenario.yaml`. + +### Assertion steps + +```gherkin +Then the agent will succeed to Emit triage JSON +And the agent will fail to Search for foo +And the agent will output issues.out with: + """ + expected content + """ +``` + +### Compatibility tags + +Use tags only for **exceptions** when a backend cannot run a scenario yet: `@skip:gitlab`, `@skip:per-org`, `@requires:per-repo`. Untagged scenarios run everywhere applicable. + +## Running locally + +```bash +export GITHUB_TOKEN=... # PAT with access to halfsend org pool +make behaviour-test +``` + +Test orgs (`halfsend-01` … `halfsend-06`) must have per-org fullsend installed with `--runtime dummy` and `test-repo` enrolled. + +Runner env (defaults shown): + +``` +BEHAVIOUR_SCM=github +BEHAVIOUR_CI=githubactions +BEHAVIOUR_INSTALL_MODE=per-org +``` + +See [behaviour-drivers.md](behaviour-drivers.md) for driver configuration and [ADR 0043](../../ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md) for the decision record. diff --git a/docs/problems/testing-agents.md b/docs/problems/testing-agents.md index de29e3e5c7..0fc66353fe 100644 --- a/docs/problems/testing-agents.md +++ b/docs/problems/testing-agents.md @@ -8,6 +8,8 @@ Testing application code is a solved problem with mature tooling: unit tests, in Today, if someone modifies a review agent's instructions, the only verification is human review of the prose change. There is no automated way to confirm the agent still behaves correctly after the modification. This is the equivalent of shipping code changes with no test suite — something we would never accept for application code. +**Behaviour tests are orthogonal.** [ADR 0043](../ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md) describes Gherkin end-to-end tests under `e2e/behaviour/` that validate deterministic platform code (workflows, harness, sandbox policy, post-scripts) with inference explicitly removed via the dummy runtime. They do not evaluate instructions, prompts, or models, and they do not replace golden-set or statistical LLM evals described below. + ## What makes agent testing hard ### Non-determinism diff --git a/docs/runtimes.md b/docs/runtimes.md index e33877bfed..fb1a2cc617 100644 --- a/docs/runtimes.md +++ b/docs/runtimes.md @@ -1,8 +1,15 @@ # Agent runtimes -Fullsend's `fullsend run` command delegates in-sandbox agent execution to a pluggable **runtime**. Today only **Claude Code** is registered; the `internal/runtime` package defines the contracts new runtimes must implement. +Fullsend's `fullsend run` command delegates in-sandbox agent execution to a pluggable **runtime**. Recognized values in org `config.yaml` `defaults.runtime` are **`claude`** (production default) and **`dummy`** (behaviour tests only). Install with `fullsend admin install --runtime dummy` on dedicated test orgs. The runner resolves the backend via `runtime.ResolveFromConfig()` after loading the org config. -When adding a runtime, fill in the security matrix below and wire the implementation through `runtime.Default()`. +When adding a runtime, fill in the security matrix below and register it in `runtime.Resolve()`. + +## Registered runtimes + +| Runtime | Purpose | Inference | +|---------|---------|-----------| +| `claude` | Production agent runs via Claude Code | Required | +| `dummy` | Behaviour tests — scripted ops in real sandbox | None | ## Security feature matrix diff --git a/e2e/admin/lock.go b/e2e/admin/lock.go index b203537729..4a4271284f 100644 --- a/e2e/admin/lock.go +++ b/e2e/admin/lock.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e || behaviour package admin @@ -196,6 +196,11 @@ func releaseLock(ctx context.Context, client forge.Client, org, runID string, t t.Logf("[e2e-lock] Lock released (run: %s)", truncateUUID(runID)) } +// ReleaseLock deletes the org lock repo when the run still holds it. +func ReleaseLock(ctx context.Context, client forge.Client, org, runID string, t *testing.T) { + releaseLock(ctx, client, org, runID, t) +} + // tryReclaimStaleLock checks whether the lock on org is stale (older than // staleLockTimeout) and force-acquires it if so. Returns true if the lock // was reclaimed. This runs during the first pass so stale locks from diff --git a/e2e/admin/testutil.go b/e2e/admin/testutil.go index d4e3bdbf8e..95c3aa443d 100644 --- a/e2e/admin/testutil.go +++ b/e2e/admin/testutil.go @@ -1,4 +1,4 @@ -//go:build e2e +//go:build e2e || behaviour package admin @@ -302,3 +302,18 @@ func retryOnNotFound(ctx context.Context, maxAttempts int, fn func() error) erro } return err } + +// AcquireOrg exports org pool acquisition for behaviour tests. +func AcquireOrg(ctx context.Context, client forge.Client, token, runID string, pool []string, timeout time.Duration, logf func(string, ...any)) (string, error) { + return acquireOrg(ctx, client, token, runID, pool, timeout, logf) +} + +// OrgPool returns the halfsend org names used for parallel e2e runs. +func OrgPool() []string { + return orgPool +} + +// NewLiveClient creates a GitHub API client from a token. +func NewLiveClient(token string) *gh.LiveClient { + return newLiveClient(token) +} diff --git a/e2e/behaviour/drivers/ci/driver.go b/e2e/behaviour/drivers/ci/driver.go new file mode 100644 index 0000000000..9b75917000 --- /dev/null +++ b/e2e/behaviour/drivers/ci/driver.go @@ -0,0 +1,16 @@ +package ci + +import ( + "context" + "time" + + "github.com/fullsend-ai/fullsend/internal/forge" +) + +// Driver abstracts CI workflow operations for behaviour tests. +type Driver interface { + WaitForWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) (*forge.WorkflowRun, error) + AssertNoWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) error + GetRunLogs(ctx context.Context, owner, repo string, runID int) (string, error) + DownloadArtifacts(ctx context.Context, owner, repo string, runID int, destDir string) error +} diff --git a/e2e/behaviour/drivers/ci/githubactions/githubactions.go b/e2e/behaviour/drivers/ci/githubactions/githubactions.go new file mode 100644 index 0000000000..9fe239af3e --- /dev/null +++ b/e2e/behaviour/drivers/ci/githubactions/githubactions.go @@ -0,0 +1,251 @@ +package githubactions + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/ci" + "github.com/fullsend-ai/fullsend/internal/forge" +) + +const ( + pollInterval = 15 * time.Second + dispatchWait = 12 * time.Minute + dispatchPoll = 5 * time.Second + dispatchMaxTry = 12 +) + +// Driver implements ci.Driver against GitHub Actions. +type Driver struct { + Client forge.Client + Token string +} + +func New(client forge.Client, token string) ci.Driver { + return &Driver{Client: client, Token: token} +} + +func (d *Driver) WaitForWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) (*forge.WorkflowRun, error) { + var triageRun *forge.WorkflowRun + for attempt := 0; attempt < dispatchMaxTry; attempt++ { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(dispatchPoll): + } + runs, err := d.Client.ListWorkflowRuns(ctx, owner, repo, workflowFile) + if err != nil { + continue + } + for _, run := range runs { + runTime, parseErr := time.Parse(time.RFC3339, run.CreatedAt) + if parseErr != nil || runTime.Before(after) { + continue + } + r := run + triageRun = &r + break + } + if triageRun != nil { + break + } + } + if triageRun == nil { + return nil, fmt.Errorf("workflow %s was not dispatched", workflowFile) + } + + deadline := time.Now().Add(dispatchWait) + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(pollInterval): + } + run, err := d.Client.GetWorkflowRun(ctx, owner, repo, triageRun.ID) + if err != nil { + continue + } + if run.Status == "completed" { + if run.Conclusion != "success" { + return run, fmt.Errorf("workflow %s run %d concluded with %q", workflowFile, run.ID, run.Conclusion) + } + return run, nil + } + } + return nil, fmt.Errorf("workflow %s run %d did not complete within deadline", workflowFile, triageRun.ID) +} + +func (d *Driver) AssertNoWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) error { + runs, err := d.Client.ListWorkflowRuns(ctx, owner, repo, workflowFile) + if err != nil { + return err + } + for _, run := range runs { + runTime, parseErr := time.Parse(time.RFC3339, run.CreatedAt) + if parseErr != nil { + continue + } + if !runTime.Before(after) { + return fmt.Errorf("unexpected workflow run %d for %s", run.ID, workflowFile) + } + } + return nil +} + +func (d *Driver) GetRunLogs(ctx context.Context, owner, repo string, runID int) (string, error) { + return d.Client.GetWorkflowRunLogs(ctx, owner, repo, runID) +} + +func (d *Driver) DownloadArtifacts(ctx context.Context, owner, repo string, runID int, destDir string) error { + listURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/actions/runs/%d/artifacts", owner, repo, runID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, listURL, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.Token) + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("list artifacts returned HTTP %d", resp.StatusCode) + } + + var result struct { + Artifacts []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"artifacts"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return err + } + + for _, art := range result.Artifacts { + if err := downloadArtifact(ctx, d.Token, owner, repo, art.ID, art.Name, destDir); err != nil { + return err + } + } + return nil +} + +func downloadArtifact(ctx context.Context, token, owner, repo string, artifactID int, name, destDir string) error { + dlURL := fmt.Sprintf("https://api.github.com/repos/%s/%s/actions/artifacts/%d/zip", owner, repo, artifactID) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download artifact %s returned HTTP %d", name, resp.StatusCode) + } + + zipData, err := io.ReadAll(io.LimitReader(resp.Body, 50<<20)) + if err != nil { + return err + } + + artDir := filepath.Join(destDir, name) + if err := os.MkdirAll(artDir, 0o755); err != nil { + return err + } + + zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData))) + if err != nil { + rawPath := filepath.Join(destDir, name+".bin") + return os.WriteFile(rawPath, zipData, 0o644) + } + + for _, f := range zr.File { + outPath := filepath.Join(artDir, f.Name) + if !strings.HasPrefix(filepath.Clean(outPath), filepath.Clean(artDir)+string(os.PathSeparator)) { + continue + } + if f.FileInfo().IsDir() { + _ = os.MkdirAll(outPath, 0o755) + continue + } + _ = os.MkdirAll(filepath.Dir(outPath), 0o755) + rc, err := f.Open() + if err != nil { + continue + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + continue + } + if err := os.WriteFile(outPath, data, 0o644); err != nil { + return err + } + } + return nil +} + +// FindBehaviourResults locates behaviour-results.json in downloaded artifacts. +func FindBehaviourResults(artifactRoot string) ([]byte, error) { + var found []byte + err := filepath.WalkDir(artifactRoot, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + if filepath.Base(path) == "behaviour-results.json" { + data, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + found = data + } + return nil + }) + if err != nil { + return nil, err + } + if found == nil { + return nil, fmt.Errorf("behaviour-results.json not found under %s", artifactRoot) + } + return found, nil +} + +// FindOutputFile searches artifact downloads for a sandbox output file by name. +func FindOutputFile(artifactRoot, fileName string) ([]byte, error) { + var found []byte + err := filepath.WalkDir(artifactRoot, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + if filepath.Base(path) == fileName { + data, readErr := os.ReadFile(path) + if readErr != nil { + return readErr + } + found = data + } + return nil + }) + if err != nil { + return nil, err + } + if found == nil { + return nil, fmt.Errorf("%s not found under %s", fileName, artifactRoot) + } + return found, nil +} diff --git a/e2e/behaviour/drivers/env/env.go b/e2e/behaviour/drivers/env/env.go new file mode 100644 index 0000000000..290070c870 --- /dev/null +++ b/e2e/behaviour/drivers/env/env.go @@ -0,0 +1,96 @@ +package env + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/fullsend-ai/fullsend/internal/config" + "github.com/fullsend-ai/fullsend/internal/forge" +) + +const defaultTestRepo = "test-repo" + +// Setup validates that a test org is ready for behaviour tests. +type Setup interface { + Validate(ctx context.Context, org string) error + TestRepo() string +} + +// PerOrg validates per-org fullsend installation with an enrolled test repo. +type PerOrg struct { + Client forge.Client + RepoName string +} + +func NewPerOrg(client forge.Client) *PerOrg { + return &PerOrg{Client: client, RepoName: defaultTestRepo} +} + +func (p *PerOrg) TestRepo() string { + if p.RepoName == "" { + return defaultTestRepo + } + return p.RepoName +} + +func (p *PerOrg) Validate(ctx context.Context, org string) error { + if _, err := p.Client.GetRepo(ctx, org, forge.ConfigRepoName); err != nil { + return fmt.Errorf("org %s missing %s repo: %w", org, forge.ConfigRepoName, err) + } + cfgData, err := p.Client.GetFileContent(ctx, org, forge.ConfigRepoName, "config.yaml") + if err != nil { + return fmt.Errorf("reading config.yaml: %w", err) + } + cfg, err := config.ParseOrgConfig(cfgData) + if err != nil { + return fmt.Errorf("parsing config.yaml: %w", err) + } + repoCfg, ok := cfg.Repos[p.TestRepo()] + if !ok || !repoCfg.Enabled { + return fmt.Errorf("org %s does not have enrolled repo %q", org, p.TestRepo()) + } + if cfg.Defaults.Runtime != "dummy" { + return fmt.Errorf("org %s config defaults.runtime is %q, want dummy for behaviour tests", org, cfg.Defaults.Runtime) + } + if _, err := p.Client.GetRepo(ctx, org, p.TestRepo()); err != nil { + return fmt.Errorf("test repo %s/%s not found: %w", org, p.TestRepo(), err) + } + return nil +} + +// RunnerConfig holds behaviour test runner configuration from environment. +type RunnerConfig struct { + SCM string + CI string + InstallMode string +} + +func LoadRunnerConfig() RunnerConfig { + return RunnerConfig{ + SCM: stringsTrimOrDefault(os.Getenv("BEHAVIOUR_SCM"), "github"), + CI: stringsTrimOrDefault(os.Getenv("BEHAVIOUR_CI"), "githubactions"), + InstallMode: stringsTrimOrDefault(os.Getenv("BEHAVIOUR_INSTALL_MODE"), "per-org"), + } +} + +func (c RunnerConfig) Validate() error { + if c.InstallMode != "per-org" { + return fmt.Errorf("behaviour tests v1 only support BEHAVIOUR_INSTALL_MODE=per-org, got %q", c.InstallMode) + } + if c.SCM != "github" { + return fmt.Errorf("unsupported BEHAVIOUR_SCM %q", c.SCM) + } + if c.CI != "githubactions" { + return fmt.Errorf("unsupported BEHAVIOUR_CI %q", c.CI) + } + return nil +} + +func stringsTrimOrDefault(value, fallback string) string { + if v := strings.TrimSpace(value); v != "" { + return v + } + return fallback +} diff --git a/e2e/behaviour/drivers/scm/driver.go b/e2e/behaviour/drivers/scm/driver.go new file mode 100644 index 0000000000..fbd2dd1d77 --- /dev/null +++ b/e2e/behaviour/drivers/scm/driver.go @@ -0,0 +1,16 @@ +package scm + +import ( + "context" + + "github.com/fullsend-ai/fullsend/internal/forge" +) + +// Driver abstracts SCM operations for behaviour tests. +type Driver interface { + CreateIssue(ctx context.Context, owner, repo, title, body string, labels ...string) (*forge.Issue, error) + AddComment(ctx context.Context, owner, repo string, number int, body string) (*forge.IssueComment, error) + GetIssue(ctx context.Context, owner, repo string, number int) (*forge.Issue, error) + CommitFile(ctx context.Context, owner, repo, path, message string, content []byte) error + CloseIssue(ctx context.Context, owner, repo string, number int) error +} diff --git a/e2e/behaviour/drivers/scm/github/github.go b/e2e/behaviour/drivers/scm/github/github.go new file mode 100644 index 0000000000..e00e92193f --- /dev/null +++ b/e2e/behaviour/drivers/scm/github/github.go @@ -0,0 +1,98 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm" + "github.com/fullsend-ai/fullsend/internal/forge" +) + +// Driver implements scm.Driver using forge.Client and GitHub REST where needed. +type Driver struct { + Client forge.Client + Token string +} + +func New(client forge.Client, token string) scm.Driver { + return &Driver{Client: client, Token: token} +} + +func (d *Driver) CreateIssue(ctx context.Context, owner, repo, title, body string, labels ...string) (*forge.Issue, error) { + return d.Client.CreateIssue(ctx, owner, repo, title, body, labels...) +} + +func (d *Driver) AddComment(ctx context.Context, owner, repo string, number int, body string) (*forge.IssueComment, error) { + return d.Client.CreateIssueComment(ctx, owner, repo, number, body) +} + +func (d *Driver) CloseIssue(ctx context.Context, owner, repo string, number int) error { + return d.Client.CloseIssue(ctx, owner, repo, number) +} + +func (d *Driver) CommitFile(ctx context.Context, owner, repo, path, message string, content []byte) error { + _, err := d.Client.CommitFiles(ctx, owner, repo, message, []forge.TreeFile{{ + Path: path, + Content: content, + Mode: "100644", + }}) + return err +} + +func (d *Driver) GetIssue(ctx context.Context, owner, repo string, number int) (*forge.Issue, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/%s/issues/%d", owner, repo, number) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+d.Token) + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, forge.ErrNotFound + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("get issue returned HTTP %d", resp.StatusCode) + } + + var payload struct { + Number int `json:"number"` + Title string `json:"title"` + Body string `json:"body"` + URL string `json:"html_url"` + Labels []struct { + Name string `json:"name"` + } `json:"labels"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return nil, err + } + labels := make([]string, len(payload.Labels)) + for i, l := range payload.Labels { + labels[i] = l.Name + } + return &forge.Issue{ + Number: payload.Number, + Title: payload.Title, + Body: payload.Body, + URL: payload.URL, + Labels: labels, + }, nil +} + +// ParseRepo splits "owner/repo" into owner and repo name. +func ParseRepo(fullName string) (owner, repo string, err error) { + parts := strings.Split(fullName, "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid repository %q: expected owner/repo", fullName) + } + return parts[0], parts[1], nil +} diff --git a/e2e/behaviour/features/triage/triage.feature b/e2e/behaviour/features/triage/triage.feature new file mode 100644 index 0000000000..087ebaa4bc --- /dev/null +++ b/e2e/behaviour/features/triage/triage.feature @@ -0,0 +1,22 @@ +Feature: Manual triage via slash command + + Scenario: Triage applies ready-to-code on sufficient issue + Given the enrolled test repository + And a dummy agent that would: + | description | op | args | + | Emit triage JSON | write_fixture | output/agent-result.json, fixtures/triage/sufficient.json | + And an issue with title "Login fails" and body containing "steps to reproduce" + When a member comments "/fs-triage" on the issue + Then the triage workflow completes successfully + And the agent will succeed to Emit triage JSON + And the issue has label "ready-to-code" + + Scenario: Sandbox blocks disallowed outbound URL + Given the enrolled test repository + And a dummy agent that would: + | description | op | args | + | Search for foo | url_get | https://www.google.com/search?q=foo | + And an issue + When a member comments "/fs-triage" on the issue + Then the triage workflow completes successfully + And the agent will fail to Search for foo diff --git a/e2e/behaviour/fixtures/triage/sufficient.json b/e2e/behaviour/fixtures/triage/sufficient.json new file mode 100644 index 0000000000..0a6c692c60 --- /dev/null +++ b/e2e/behaviour/fixtures/triage/sufficient.json @@ -0,0 +1,24 @@ +{ + "action": "sufficient", + "reasoning": "Issue includes clear reproduction steps and environment details.", + "clarity_scores": { + "symptom": 0.9, + "cause": 0.85, + "reproduction": 0.9, + "impact": 0.8, + "overall": 0.87 + }, + "triage_summary": { + "title": "Login fails", + "severity": "high", + "category": "bug", + "problem": "Application crashes on login", + "root_cause_hypothesis": "Session handling regression", + "reproduction_steps": ["Open app", "Enter credentials", "Click login"], + "environment": "Linux", + "impact": "Users cannot authenticate", + "recommended_fix": "Fix session token validation", + "proposed_test_case": "test_login_success" + }, + "comment": "## Triage Summary\n\nThis issue is ready for implementation." +} diff --git a/e2e/behaviour/steps/dummy_agent.go b/e2e/behaviour/steps/dummy_agent.go new file mode 100644 index 0000000000..3e3cb4df26 --- /dev/null +++ b/e2e/behaviour/steps/dummy_agent.go @@ -0,0 +1,138 @@ +package steps + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/cucumber/godog" + "gopkg.in/yaml.v3" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" + "github.com/fullsend-ai/fullsend/internal/runtime" +) + +const behaviourModuleRoot = "e2e/behaviour" + +func registerDummyAgentSteps(ctx *godog.ScenarioContext, w *world.World) { + ctx.Step(`^a dummy agent that would:$`, func(table *godog.Table) error { + return parseDummyAgentTable(w, table) + }) + ctx.Step(`^the agent will succeed to (.+)$`, func(description string) error { + return assertAgentSucceeds(w, description) + }) + ctx.Step(`^the agent will fail to (.+)$`, func(description string) error { + return assertAgentFails(w, description) + }) + ctx.Step(`^the agent will output ([^\s]+) with:$`, func(fileName, doc string) error { + return assertAgentOutput(w, fileName, doc) + }) +} + +func parseDummyAgentTable(w *world.World, table *godog.Table) error { + if len(table.Rows) < 2 { + return fmt.Errorf("dummy agent table requires a header and at least one row") + } + header := table.Rows[0] + col := map[string]int{} + for i, cell := range header.Cells { + col[strings.TrimSpace(cell.Value)] = i + } + for _, required := range []string{"description", "op", "args"} { + if _, ok := col[required]; !ok { + return fmt.Errorf("dummy agent table missing %q column", required) + } + } + + moduleRoot, err := findModuleSubdir(behaviourModuleRoot) + if err != nil { + return err + } + + var ops []runtime.BehaviourOperation + for _, row := range table.Rows[1:] { + op := runtime.BehaviourOperation{ + Description: strings.TrimSpace(row.Cells[col["description"]].Value), + Op: strings.TrimSpace(row.Cells[col["op"]].Value), + Args: strings.TrimSpace(row.Cells[col["args"]].Value), + } + if op.Op == "write_fixture" { + parts := strings.SplitN(op.Args, ",", 2) + if len(parts) != 2 { + return fmt.Errorf("write_fixture args must be dest_path, fixture_path") + } + fixtureRel := strings.TrimSpace(parts[1]) + fixturePath := filepath.Join(moduleRoot, fixtureRel) + content, err := os.ReadFile(fixturePath) + if err != nil { + return fmt.Errorf("reading fixture %s: %w", fixturePath, err) + } + op.Content = string(content) + } + ops = append(ops, op) + } + + script := runtime.BehaviourScript{Ops: ops} + data, err := yaml.Marshal(script) + if err != nil { + return fmt.Errorf("marshaling behaviour script: %w", err) + } + + message := fmt.Sprintf("behaviour: set dummy agent script (%s)", time.Now().UTC().Format(time.RFC3339)) + if err := w.SCM.CommitFile(context.Background(), w.Org, ".fullsend", world.BehaviourScriptRepoPath, message, data); err != nil { + return fmt.Errorf("committing behaviour script: %w", err) + } + + w.DummyOps = ops + return nil +} + +func assertAgentSucceeds(w *world.World, description string) error { + return assertAgentOutcome(w, description, true) +} + +func assertAgentFails(w *world.World, description string) error { + return assertAgentOutcome(w, description, false) +} + +func assertAgentOutcome(w *world.World, description string, expectSuccess bool) error { + w.DummyExpectations = append(w.DummyExpectations, world.DummyOpExpectation{ + Description: strings.TrimSpace(description), + ExpectSuccess: expectSuccess, + }) + return nil +} + +func assertAgentOutput(w *world.World, fileName, doc string) error { + w.OutputExpectations = append(w.OutputExpectations, world.OutputExpectation{ + FileName: strings.TrimSpace(fileName), + Content: strings.TrimSpace(doc), + Exact: true, + }) + return nil +} + +func findModuleSubdir(rel string) (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + candidate := filepath.Join(dir, rel) + if st, err := os.Stat(candidate); err == nil && st.IsDir() { + return candidate, nil + } + return "", fmt.Errorf("could not find %s under module root %s", rel, dir) + } + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + } + return "", fmt.Errorf("could not find go.mod while searching for %s", rel) +} diff --git a/e2e/behaviour/steps/registry.go b/e2e/behaviour/steps/registry.go new file mode 100644 index 0000000000..1f2e73ffda --- /dev/null +++ b/e2e/behaviour/steps/registry.go @@ -0,0 +1,12 @@ +package steps + +import ( + "github.com/cucumber/godog" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" +) + +func Register(ctx *godog.ScenarioContext, w *world.World) { + registerDummyAgentSteps(ctx, w) + registerTriageSteps(ctx, w) +} diff --git a/e2e/behaviour/steps/triage.go b/e2e/behaviour/steps/triage.go new file mode 100644 index 0000000000..b8b689de6f --- /dev/null +++ b/e2e/behaviour/steps/triage.go @@ -0,0 +1,191 @@ +package steps + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/cucumber/godog" + + gaci "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/ci/githubactions" + scmgh "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm/github" + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" + "github.com/fullsend-ai/fullsend/internal/forge" + "github.com/fullsend-ai/fullsend/internal/runtime" +) + +func registerTriageSteps(ctx *godog.ScenarioContext, w *world.World) { + ctx.Step(`^the enrolled test repository$`, func() error { return givenEnrolledTestRepository(w) }) + ctx.Step(`^an enrolled repository "([^"]+)"$`, func(fullName string) error { + return givenEnrolledRepository(w, fullName) + }) + ctx.Step(`^an issue with title "([^"]+)" and body containing "([^"]+)"$`, func(title, bodyContains string) error { + return givenIssueWithTitleAndBody(w, title, bodyContains) + }) + ctx.Step(`^an issue$`, func() error { return givenIssue(w) }) + ctx.Step(`^a member comments "([^"]+)" on the issue$`, func(comment string) error { + return whenMemberComments(w, comment) + }) + ctx.Step(`^the triage workflow completes successfully$`, func() error { + return thenTriageWorkflowCompletes(w) + }) + ctx.Step(`^the issue has label "([^"]+)"$`, func(label string) error { + return thenIssueHasLabel(w, label) + }) +} + +func givenEnrolledTestRepository(w *world.World) error { + w.RepoOwner = w.Org + w.RepoName = w.Env.TestRepo() + w.RepoFull = w.Org + "/" + w.RepoName + return nil +} + +func givenEnrolledRepository(w *world.World, fullName string) error { + owner, repo, err := scmgh.ParseRepo(fullName) + if err != nil { + return err + } + if owner != w.Org { + return fmt.Errorf("repository owner %q does not match test org %q", owner, w.Org) + } + if repo != w.Env.TestRepo() { + return fmt.Errorf("repository %q is not the enrolled test repo %q", repo, w.Env.TestRepo()) + } + w.RepoFull = fullName + w.RepoOwner = owner + w.RepoName = repo + return nil +} + +func givenIssueWithTitleAndBody(w *world.World, title, bodyContains string) error { + body := fmt.Sprintf("Behaviour test issue\n\n%s\n", bodyContains) + return createIssue(w, title, body) +} + +func givenIssue(w *world.World) error { + title := fmt.Sprintf("behaviour-issue-%d", time.Now().UnixNano()) + body := "Behaviour test issue body with steps to reproduce for triage." + return createIssue(w, title, body) +} + +func createIssue(w *world.World, title, body string) error { + if w.RepoOwner == "" || w.RepoName == "" { + w.RepoOwner = w.Org + w.RepoName = w.Env.TestRepo() + w.RepoFull = w.Org + "/" + w.RepoName + } + issue, err := w.SCM.CreateIssue(context.Background(), w.RepoOwner, w.RepoName, title, body) + if err != nil { + return err + } + w.IssueNumber = issue.Number + w.IssueTitle = title + w.ScenarioStart = time.Now() + return nil +} + +func whenMemberComments(w *world.World, comment string) error { + if w.IssueNumber == 0 { + return fmt.Errorf("no issue created") + } + _, err := w.SCM.AddComment(context.Background(), w.RepoOwner, w.RepoName, w.IssueNumber, comment) + return err +} + +func thenTriageWorkflowCompletes(w *world.World) error { + ctx := context.Background() + run, err := w.CI.WaitForWorkflow(ctx, w.Org, forge.ConfigRepoName, "triage.yml", w.ScenarioStart) + if err != nil { + return err + } + w.WorkflowRun = run + + artifactDir, err := os.MkdirTemp("", "behaviour-artifacts-*") + if err != nil { + return err + } + w.ArtifactDir = artifactDir + if err := w.CI.DownloadArtifacts(ctx, w.Org, forge.ConfigRepoName, run.ID, artifactDir); err != nil { + return err + } + + if err := verifyDummyExpectations(w, artifactDir); err != nil { + return err + } + return verifyOutputExpectations(w, artifactDir) +} + +func verifyDummyExpectations(w *world.World, artifactDir string) error { + data, err := gaci.FindBehaviourResults(artifactDir) + if err != nil { + return err + } + var results runtime.BehaviourResults + if err := json.Unmarshal(data, &results); err != nil { + return fmt.Errorf("parsing behaviour-results.json: %w", err) + } + byDescription := map[string]runtime.BehaviourOpResult{} + for _, res := range results.Operations { + byDescription[res.Description] = res + } + for _, exp := range w.DummyExpectations { + res, ok := byDescription[exp.Description] + if !ok { + return fmt.Errorf("operation %q not found in behaviour-results.json", exp.Description) + } + if res.Success != exp.ExpectSuccess { + return fmt.Errorf("operation %q: expected success=%v, got success=%v (error: %s)", exp.Description, exp.ExpectSuccess, res.Success, res.Error) + } + } + return nil +} + +func verifyOutputExpectations(w *world.World, artifactDir string) error { + for _, exp := range w.OutputExpectations { + data, err := gaci.FindOutputFile(artifactDir, exp.FileName) + if err != nil { + return err + } + actual := strings.TrimSpace(string(data)) + expected := strings.TrimSpace(exp.Content) + if exp.Exact { + if actual != expected { + return fmt.Errorf("output file %q: expected %q, got %q", exp.FileName, expected, actual) + } + continue + } + if !strings.Contains(actual, expected) { + return fmt.Errorf("output file %q: expected substring %q in %q", exp.FileName, expected, actual) + } + } + return nil +} + +func thenIssueHasLabel(w *world.World, label string) error { + issue, err := w.SCM.GetIssue(context.Background(), w.RepoOwner, w.RepoName, w.IssueNumber) + if err != nil { + return err + } + for _, name := range issue.Labels { + if name == label { + return nil + } + } + return fmt.Errorf("issue #%d labels %v do not include %q", w.IssueNumber, issue.Labels, label) +} + +func CleanupScenario(w *world.World) { + ctx := context.Background() + if w.IssueNumber > 0 { + _ = w.SCM.CloseIssue(ctx, w.RepoOwner, w.RepoName, w.IssueNumber) + } + if w.ArtifactDir != "" { + _ = os.RemoveAll(w.ArtifactDir) + } + empty := []byte("ops: []\n") + _ = w.SCM.CommitFile(ctx, w.Org, ".fullsend", world.BehaviourScriptRepoPath, "behaviour: clear dummy agent script", empty) +} diff --git a/e2e/behaviour/suite_test.go b/e2e/behaviour/suite_test.go new file mode 100644 index 0000000000..59d216bddd --- /dev/null +++ b/e2e/behaviour/suite_test.go @@ -0,0 +1,119 @@ +//go:build behaviour + +package behaviour_test + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/cucumber/godog" + "github.com/google/uuid" + + "github.com/fullsend-ai/fullsend/e2e/admin" + gaci "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/ci/githubactions" + scmgh "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm/github" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/env" + "github.com/fullsend-ai/fullsend/e2e/behaviour/steps" + "github.com/fullsend-ai/fullsend/e2e/behaviour/world" +) + +func TestBehaviourSuite(t *testing.T) { + if testing.Short() { + t.Skip("skipping behaviour tests in short mode") + } + + cfg := env.LoadRunnerConfig() + if err := cfg.Validate(); err != nil { + t.Fatalf("invalid behaviour runner config: %v", err) + } + + token := resolveToken(t) + client := admin.NewLiveClient(token) + ctx := context.Background() + + runID := uuid.New().String() + org, err := admin.AcquireOrg(ctx, client, token, runID, admin.OrgPool(), 10*time.Minute, t.Logf) + if err != nil { + t.Fatalf("acquiring org: %v", err) + } + t.Cleanup(func() { + admin.ReleaseLock(context.Background(), client, org, runID, t) + }) + + setup := env.NewPerOrg(client) + if err := setup.Validate(ctx, org); err != nil { + t.Skipf("org %s not ready for behaviour tests: %v", org, err) + } + + w := &world.World{ + Config: cfg, + SCM: scmgh.New(client, token), + CI: gaci.New(client, token), + Env: setup, + Org: org, + Token: token, + RepoOwner: org, + RepoName: setup.TestRepo(), + RepoFull: org + "/" + setup.TestRepo(), + } + + suite := godog.TestSuite{ + Name: "behaviour", + ScenarioInitializer: func(sc *godog.ScenarioContext) { initializeScenario(sc, w) }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{"features"}, + TestingT: t, + Tags: os.Getenv("GODOG_TAGS"), + }, + } + if st := suite.Run(); st != 0 { + t.Fatalf("behaviour suite failed with status %d", st) + } +} + +func initializeScenario(sc *godog.ScenarioContext, w *world.World) { + sc.Before(func(ctx context.Context, sc *godog.Scenario) (context.Context, error) { + for _, tag := range sc.Tags { + name := strings.TrimPrefix(tag.Name, "@") + switch { + case name == "skip:per-org" && w.Config.InstallMode == "per-org": + return ctx, godog.ErrSkip + case name == "skip:per-repo" && w.Config.InstallMode == "per-repo": + return ctx, godog.ErrSkip + case name == "requires:per-repo" && w.Config.InstallMode != "per-repo": + return ctx, godog.ErrSkip + case name == "skip:gitlab" && w.Config.SCM == "gitlab": + return ctx, godog.ErrSkip + } + } + w.ScenarioStart = time.Now() + w.DummyOps = nil + w.DummyExpectations = nil + w.OutputExpectations = nil + w.IssueNumber = 0 + w.IssueTitle = "" + w.WorkflowRun = nil + w.ArtifactDir = "" + return ctx, nil + }) + sc.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { + steps.CleanupScenario(w) + return ctx, err + }) + steps.Register(sc, w) +} + +func resolveToken(t *testing.T) string { + t.Helper() + for _, key := range []string{"GITHUB_TOKEN", "GH_TOKEN"} { + if token := strings.TrimSpace(os.Getenv(key)); token != "" { + return token + } + } + t.Skip("GITHUB_TOKEN or GH_TOKEN required for behaviour tests") + return "" +} diff --git a/e2e/behaviour/world/world.go b/e2e/behaviour/world/world.go new file mode 100644 index 0000000000..8002b097c8 --- /dev/null +++ b/e2e/behaviour/world/world.go @@ -0,0 +1,53 @@ +package world + +import ( + "time" + + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/ci" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/env" + "github.com/fullsend-ai/fullsend/e2e/behaviour/drivers/scm" + "github.com/fullsend-ai/fullsend/internal/forge" + "github.com/fullsend-ai/fullsend/internal/runtime" +) + +// DummyOpExpectation records expected success/failure for a dummy agent operation. +type DummyOpExpectation struct { + Description string + ExpectSuccess bool +} + +// OutputExpectation records expected sandbox output file content. +type OutputExpectation struct { + FileName string + Content string + Exact bool +} + +// World holds scenario state and injected drivers. +type World struct { + Config env.RunnerConfig + SCM scm.Driver + CI ci.Driver + Env env.Setup + + Org string + RepoFull string + RepoOwner string + RepoName string + Token string + + ScenarioStart time.Time + + DummyOps []runtime.BehaviourOperation + DummyExpectations []DummyOpExpectation + OutputExpectations []OutputExpectation + BehaviourScriptPath string + ArtifactDir string + + IssueNumber int + IssueTitle string + WorkflowRun *forge.WorkflowRun + TriageWorkflow string +} + +const BehaviourScriptRepoPath = "behaviour/current-scenario.yaml" diff --git a/go.mod b/go.mod index 85d0b8bec1..bdf33e7ed4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.0 require ( github.com/charmbracelet/lipgloss v1.1.0 + github.com/cucumber/godog v0.14.1 github.com/google/uuid v1.6.0 github.com/knights-analytics/hugot v0.7.1 github.com/playwright-community/playwright-go v0.5700.1 @@ -17,6 +18,15 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect + github.com/cucumber/messages/go/v21 v21.0.1 // indirect + github.com/gofrs/uuid v4.4.0+incompatible // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-memdb v1.3.4 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect +) + require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 6dcbb603c3..8a6a5fcd4b 100644 --- a/go.sum +++ b/go.sum @@ -30,10 +30,19 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= +github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= +github.com/cucumber/godog v0.14.1 h1:HGZhcOyyfaKclHjJ+r/q93iaTJZLKYW6Tv3HkmUE6+M= +github.com/cucumber/godog v0.14.1/go.mod h1:FX3rzIDybWABU4kuIXLZ/qtqEe1Ac5RdXmqvACJOces= +github.com/cucumber/messages/go/v21 v21.0.1 h1:wzA0LxwjlWQYZd32VTlAVDTkW6inOFmSM+RuOwHZiMI= +github.com/cucumber/messages/go/v21 v21.0.1/go.mod h1:zheH/2HS9JLVFukdrsPWoPdmUtmYQAQPLk7w5vWsk5s= +github.com/cucumber/messages/go/v22 v22.0.0/go.mod h1:aZipXTKc0JnjCsXrJnuZpWhtay93k7Rn3Dee7iyPJjs= github.com/daulet/tokenizers v1.27.0 h1:MmFYAEDFz69s/nNQfHg59DWqHz3v94m99kEZ/JbL+s4= github.com/daulet/tokenizers v1.27.0/go.mod h1:YjFY1o1HGMyWkQgbXJDghhvke/yFDp2vGdIO2hYs4MQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+ZlfuyaAdFlQ= @@ -50,6 +59,10 @@ github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/gomlx/exceptions v0.0.3 h1:HKnTgEjj4jlmhr8zVFkTP9qmV1ey7ypYYosQ8GzXWuM= @@ -67,6 +80,17 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= +github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/janpfeifer/go-benchmarks v0.1.1 h1:gLLy07/JrOKSnMWeUxSnjTdhkglgmrNR2IBDnR4kRqw= @@ -77,8 +101,11 @@ github.com/knights-analytics/hugot v0.7.1 h1:QljSB3o2qCg55zSidd9Aq6v5DFhDOFLHqhx github.com/knights-analytics/hugot v0.7.1/go.mod h1:D1VNAMjOPxBLnzXvwVKSnRxzAGEyD81uDzkfg8tat4k= github.com/knights-analytics/ortgenai v0.3.0 h1:ruHiWzxGnKVowlDQ4zKA3dafLLmFxus6R1vqA0rp6Ss= github.com/knights-analytics/ortgenai v0.3.0/go.mod h1:lSbQsRP5wY5NS+4W5CUGhdxjTzERQkR7WprAFxrBSt4= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= @@ -107,15 +134,23 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d h1:X4+kt6zM/OVO6gbJdAfJR60MGPsqCzbtXNnjoGqdfAs= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/viant/afs v1.30.0 h1:dbgVVSCPwGHUgpgkWJ5gdjKBqssT7OV7Z2M81CjwZEY= diff --git a/internal/cli/admin.go b/internal/cli/admin.go index f0396bea56..d54c6dd6b2 100644 --- a/internal/cli/admin.go +++ b/internal/cli/admin.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "regexp" + "slices" "sort" "strconv" "strings" @@ -235,6 +236,7 @@ func newInstallCmd() *cobra.Command { var skipMintCheck bool var publicApps bool var appSet string + var runtimeName string // Per-repo flags. var mintURL string @@ -327,6 +329,17 @@ Inference authentication: return err } + selectedRuntime := runtimeName + if !cmd.Flags().Changed("runtime") { + selectedRuntime = loadExistingRuntime(ctx, client, org) + } + if selectedRuntime == "" { + selectedRuntime = "claude" + } + if !slices.Contains(config.ValidRuntimes(), selectedRuntime) { + return fmt.Errorf("invalid --runtime %q: must be one of %s", selectedRuntime, strings.Join(config.ValidRuntimes(), ", ")) + } + if skipMintCheck { if err := validateSkipMintCheck(mintURL); err != nil { return err @@ -486,7 +499,7 @@ Inference authentication: printer.Blank() if dryRun { - return runDryRun(ctx, client, printer, org, repos, roles, inferenceProvider, inferenceProviderName, skipMintCheck, mintURL, allRepos) + return runDryRun(ctx, client, printer, org, repos, roles, selectedRuntime, inferenceProvider, inferenceProviderName, skipMintCheck, mintURL, allRepos) } if err := checkInstallScopes(ctx, client, printer); err != nil { @@ -529,7 +542,7 @@ Inference authentication: agentCreds = creds } - return runInstall(ctx, client, printer, org, repos, roles, agentCreds, inferenceProvider, inferenceProviderName, vendorBinary, mintProvider, mintProject, mintRegion, mintSourceDir, mintSkipDeploy, mintURL, skipMintCheck, allRepos) + return runInstall(ctx, client, printer, org, repos, roles, selectedRuntime, agentCreds, inferenceProvider, inferenceProviderName, vendorBinary, mintProvider, mintProject, mintRegion, mintSourceDir, mintSkipDeploy, mintURL, skipMintCheck, allRepos) }, } @@ -550,6 +563,7 @@ Inference authentication: cmd.Flags().BoolVar(&skipMintCheck, "skip-mint-check", false, "skip mint validation, GCP provisioning, and app setup; requires --mint-url") cmd.Flags().BoolVar(&publicApps, "public", false, "create public (unlisted) GitHub Apps installable by other orgs") cmd.Flags().StringVar(&appSet, "app-set", appsetup.DefaultAppSet, "app set name prefix for GitHub Apps (e.g., myorg creates myorg-fullsend, myorg-coder)") + cmd.Flags().StringVar(&runtimeName, "runtime", "claude", "agent runtime for fullsend run (claude or dummy; dummy is for behaviour test orgs only)") // Shared flags. cmd.Flags().StringVar(&mintURL, "mint-url", "", "token mint URL for OIDC token exchange") @@ -1179,7 +1193,7 @@ func newAnalyzeCmd() *cobra.Command { // runDryRun builds a layer stack with empty credentials and analyzes. // If discoveredRepos is non-nil, it will be used instead of calling ListOrgRepos. -func runDryRun(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, inferenceProvider inference.Provider, inferenceProviderName string, skipMintCheck bool, mintURL string, discoveredRepos []forge.Repository) error { +func runDryRun(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, runtimeName string, inferenceProvider inference.Provider, inferenceProviderName string, skipMintCheck bool, mintURL string, discoveredRepos []forge.Repository) error { printer.Header("Dry run - analyzing what install would do") printer.Blank() @@ -1218,6 +1232,7 @@ func runDryRun(ctx context.Context, client forge.Client, printer *ui.Printer, or // Build config with empty agents for analysis. cfg := config.NewOrgConfig(repoNames, enabledRepos, roles, nil, inferenceProviderName) + cfg.Defaults.Runtime = runtimeName cfg.Dispatch.Mode = "oidc-mint" user, err := client.GetAuthenticatedUser(ctx) @@ -1514,7 +1529,7 @@ func validateEnabledRepos(enabledRepos, discoveredNames []string) error { // runInstall performs the full installation. // If discoveredRepos is non-nil, it will be used instead of calling ListOrgRepos. -func runInstall(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, agentCreds []layers.AgentCredentials, inferenceProvider inference.Provider, inferenceProviderName string, vendorBinary bool, mintProvider, mintProject, mintRegion, mintSourceDir string, mintSkipDeploy bool, mintURL string, skipMintCheck bool, discoveredRepos []forge.Repository) error { +func runInstall(ctx context.Context, client forge.Client, printer *ui.Printer, org string, enabledRepos, roles []string, runtimeName string, agentCreds []layers.AgentCredentials, inferenceProvider inference.Provider, inferenceProviderName string, vendorBinary bool, mintProvider, mintProject, mintRegion, mintSourceDir string, mintSkipDeploy bool, mintURL string, skipMintCheck bool, discoveredRepos []forge.Repository) error { var allRepos []forge.Repository var err error @@ -1559,6 +1574,7 @@ func runInstall(ctx context.Context, client forge.Client, printer *ui.Printer, o } cfg := config.NewOrgConfig(repoNames, enabledRepos, roles, agents, inferenceProviderName) + cfg.Defaults.Runtime = runtimeName cfg.Dispatch.Mode = "oidc-mint" user, err := client.GetAuthenticatedUser(ctx) @@ -1966,6 +1982,21 @@ func printAnalysis(ctx context.Context, stack *layers.Stack, printer *ui.Printer return nil } +// loadExistingRuntime reads defaults.runtime from an existing config.yaml in +// .fullsend, if available. This prevents re-installs without --runtime from +// silently resetting the runtime selection. +func loadExistingRuntime(ctx context.Context, client forge.Client, org string) string { + data, err := client.GetFileContent(ctx, org, forge.ConfigRepoName, "config.yaml") + if err != nil { + return "" + } + cfg, err := config.ParseOrgConfig(data) + if err != nil { + return "" + } + return cfg.Defaults.Runtime +} + // loadExistingInferenceProvider reads the inference provider name from // an existing config.yaml in .fullsend, if available. This prevents // re-installs without --inference-project from silently erasing the inference section. diff --git a/internal/cli/run.go b/internal/cli/run.go index 80d08abc95..40a43b778f 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -392,7 +392,26 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep repoDir := fmt.Sprintf("%s/%s", sandbox.SandboxWorkspace, repoName) // 7. Bootstrap sandbox. - backend := agentruntime.Default() + var backend agentruntime.Backend + orgConfigPath := filepath.Join(absFullsendDir, "config.yaml") + if orgConfigData, readErr := os.ReadFile(orgConfigPath); readErr == nil { + orgCfg, parseErr := config.ParseOrgConfig(orgConfigData) + if parseErr != nil { + printer.StepFail("Failed to parse org config") + return fmt.Errorf("parsing org config for runtime selection: %w", parseErr) + } + var resolveErr error + backend, resolveErr = agentruntime.ResolveFromConfig(orgCfg) + if resolveErr != nil { + printer.StepFail("Failed to resolve runtime") + return fmt.Errorf("resolving runtime: %w", resolveErr) + } + } else if !os.IsNotExist(readErr) { + printer.StepFail("Failed to load org config") + return fmt.Errorf("reading org config for runtime selection: %w", readErr) + } else { + backend = agentruntime.Default() + } rt := backend.Runtime tx := backend.Transcripts bootstrapStart := time.Now() @@ -601,6 +620,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep AgentBaseName: agentBaseName, Model: h.Model, RepoDir: repoDir, + FullsendDir: absFullsendDir, PluginDirs: pluginDirs, Debug: debug, Timeout: timeout, diff --git a/internal/config/config.go b/internal/config/config.go index cdedd5f295..1dfc90783a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -31,6 +31,7 @@ type InferenceConfig struct { // RepoDefaults holds default settings applied to all repos. type RepoDefaults struct { Roles []string `yaml:"roles"` + Runtime string `yaml:"runtime,omitempty"` MaxImplementationRetries int `yaml:"max_implementation_retries"` AutoMerge bool `yaml:"auto_merge"` } @@ -63,6 +64,11 @@ func ValidProviders() []string { return []string{"vertex"} } +// ValidRuntimes returns the set of recognized agent runtimes. +func ValidRuntimes() []string { + return []string{"claude", "dummy"} +} + // DefaultAgentRoles returns the standard set of agent roles installed // when no custom roles are specified. The fix stage reuses the coder // app (role: coder) so it does not need a separate app or PEM. @@ -93,6 +99,7 @@ func NewOrgConfig(allRepos, enabledRepos, roles []string, agents []AgentEntry, i }, Defaults: RepoDefaults{ Roles: roles, + Runtime: "claude", MaxImplementationRetries: 2, AutoMerge: false, }, @@ -160,6 +167,12 @@ func (c *OrgConfig) Validate() error { return fmt.Errorf("invalid inference provider %q: must be one of %s", c.Inference.Provider, strings.Join(validProviders, ", ")) } } + if rt := c.Defaults.Runtime; rt != "" { + validRuntimes := ValidRuntimes() + if !slices.Contains(validRuntimes, rt) { + return fmt.Errorf("invalid runtime %q: must be one of %s", rt, strings.Join(validRuntimes, ", ")) + } + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a7d57e34f2..761d74194b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -378,6 +378,27 @@ func TestValidProviders(t *testing.T) { assert.Equal(t, []string{"vertex"}, providers) } +func TestValidRuntimes(t *testing.T) { + runtimes := ValidRuntimes() + assert.Contains(t, runtimes, "claude") + assert.Contains(t, runtimes, "dummy") +} + +func TestOrgConfigValidateRuntime(t *testing.T) { + cfg := &OrgConfig{ + Version: "1", + Dispatch: DispatchConfig{Platform: "github-actions"}, + Defaults: RepoDefaults{ + Roles: []string{"triage"}, + Runtime: "dummy", + }, + } + require.NoError(t, cfg.Validate()) + + cfg.Defaults.Runtime = "invalid" + require.Error(t, cfg.Validate()) +} + func TestParseOrgConfig_KillSwitch(t *testing.T) { yamlData := ` version: "1" diff --git a/internal/runtime/dummy.go b/internal/runtime/dummy.go new file mode 100644 index 0000000000..5d49c70ae6 --- /dev/null +++ b/internal/runtime/dummy.go @@ -0,0 +1,253 @@ +package runtime + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "gopkg.in/yaml.v3" + + "github.com/fullsend-ai/fullsend/internal/sandbox" + "github.com/fullsend-ai/fullsend/internal/ui" +) + +const behaviourScriptRelPath = "behaviour/current-scenario.yaml" +const behaviourResultsFile = "behaviour-results.json" + +// BehaviourOperation is a single scripted step for the dummy runtime. +type BehaviourOperation struct { + Description string `yaml:"description" json:"description"` + Op string `yaml:"op" json:"op"` + Args string `yaml:"args" json:"args"` + Content string `yaml:"content,omitempty" json:"content,omitempty"` +} + +// BehaviourScript is the YAML committed to .fullsend/behaviour/current-scenario.yaml. +type BehaviourScript struct { + Ops []BehaviourOperation `yaml:"ops"` +} + +// BehaviourOpResult records the outcome of one scripted operation. +type BehaviourOpResult struct { + Description string `json:"description"` + Success bool `json:"success"` + Error string `json:"error,omitempty"` +} + +// BehaviourResults is written to output/behaviour-results.json in the sandbox. +type BehaviourResults struct { + Operations []BehaviourOpResult `json:"operations"` +} + +// DummyRuntime executes scripted operations in the real OpenShell sandbox. +type DummyRuntime struct{} + +func (DummyRuntime) Name() string { return "dummy" } + +func (DummyRuntime) ConfigDir() string { return sandbox.SandboxWorkspace + "/.dummy" } + +func (DummyRuntime) WorkspaceDir() string { return sandbox.SandboxWorkspace } + +func (DummyRuntime) EnvExports() []string { return nil } + +func (DummyRuntime) Bootstrap(input BootstrapInput) error { + sandboxName := input.SandboxName() + mkdirCmd := fmt.Sprintf("mkdir -p %s/output %s/.dummy", sandbox.SandboxWorkspace, sandbox.SandboxWorkspace) + _, _, _, err := sandbox.Exec(sandboxName, mkdirCmd, 10*time.Second) + return err +} + +func (r DummyRuntime) Run(params RunParams, printer *ui.Printer, _ time.Time, _ *RunMetrics) (int, error) { + scriptPath := filepath.Join(params.FullsendDir, behaviourScriptRelPath) + script, err := LoadBehaviourScript(scriptPath) + if err != nil { + return 1, err + } + + results, execErr := executeBehaviourScript(params.SandboxName, params.RepoDir, script) + if writeErr := writeBehaviourResults(params.SandboxName, results); writeErr != nil && execErr == nil { + execErr = writeErr + } + + if execErr != nil { + printer.StepWarn("Dummy runtime: " + execErr.Error()) + } + + exitCode := 0 + for _, res := range results.Operations { + if !res.Success { + exitCode = 1 + break + } + } + return exitCode, execErr +} + +func (r DummyRuntime) ClearIterationArtifacts(sandboxName string) error { + clearCmd := fmt.Sprintf("rm -rf %s/output/*", r.WorkspaceDir()) + _, _, _, err := sandbox.Exec(sandboxName, clearCmd, 10*time.Second) + return err +} + +func (DummyRuntime) ExtractTranscripts(_ string, _ string, _ string) error { return nil } + +func (DummyRuntime) ExtractDebugLog(_ string, _ string, _ string) error { return nil } + +func (DummyRuntime) ParseTranscriptErrors(_ string) []TranscriptError { return nil } + +func (DummyRuntime) EmitTranscriptErrors(w io.Writer, summaries []TranscriptError) { + emitTranscriptErrors(w, summaries) +} + +// LoadBehaviourScript reads and parses a behaviour scenario script from disk. +func LoadBehaviourScript(path string) (*BehaviourScript, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading behaviour script %s: %w", path, err) + } + var script BehaviourScript + if err := yaml.Unmarshal(data, &script); err != nil { + return nil, fmt.Errorf("parsing behaviour script %s: %w", path, err) + } + if len(script.Ops) == 0 { + return nil, fmt.Errorf("behaviour script %s has no operations", path) + } + return &script, nil +} + +func executeBehaviourScript(sandboxName, repoDir string, script *BehaviourScript) (BehaviourResults, error) { + var results BehaviourResults + for _, op := range script.Ops { + res := BehaviourOpResult{Description: op.Description} + if err := executeBehaviourOp(sandboxName, repoDir, op); err != nil { + res.Success = false + res.Error = err.Error() + } else { + res.Success = true + } + results.Operations = append(results.Operations, res) + } + return results, nil +} + +func executeBehaviourOp(sandboxName, repoDir string, op BehaviourOperation) error { + switch op.Op { + case "read_file": + path := strings.TrimSpace(op.Args) + if path == "" { + return fmt.Errorf("read_file requires a path") + } + remotePath := resolveSandboxPath(repoDir, path) + cmd := fmt.Sprintf("test -r %s", shellQuote(remotePath)) + _, stderr, exitCode, err := sandbox.Exec(sandboxName, cmd, 30*time.Second) + if err != nil { + return fmt.Errorf("read_file exec: %w", err) + } + if exitCode != 0 { + return fmt.Errorf("read_file failed: %s", strings.TrimSpace(stderr)) + } + return nil + case "url_get": + url := strings.TrimSpace(op.Args) + if url == "" { + return fmt.Errorf("url_get requires a URL") + } + cmd := fmt.Sprintf("curl -sf %s -o /dev/null", shellQuote(url)) + _, stderr, exitCode, err := sandbox.Exec(sandboxName, cmd, 60*time.Second) + if err != nil { + return fmt.Errorf("url_get exec: %w", err) + } + if exitCode != 0 { + return fmt.Errorf("url_get failed: %s", strings.TrimSpace(stderr)) + } + return nil + case "run_command": + command := strings.TrimSpace(op.Args) + if command == "" { + return fmt.Errorf("run_command requires a shell command") + } + _, stderr, exitCode, err := sandbox.Exec(sandboxName, command, 5*time.Minute) + if err != nil { + return fmt.Errorf("run_command exec: %w", err) + } + if exitCode != 0 { + return fmt.Errorf("run_command exited %d: %s", exitCode, strings.TrimSpace(stderr)) + } + return nil + case "write_fixture": + dest, content, err := resolveWriteFixture(op) + if err != nil { + return err + } + remoteDest := resolveSandboxPath(sandbox.SandboxWorkspace, dest) + mkdirCmd := fmt.Sprintf("mkdir -p $(dirname %s)", shellQuote(remoteDest)) + if _, _, _, err := sandbox.Exec(sandboxName, mkdirCmd, 10*time.Second); err != nil { + return fmt.Errorf("write_fixture mkdir: %w", err) + } + tmp, err := os.CreateTemp("", "behaviour-fixture-*") + if err != nil { + return fmt.Errorf("write_fixture temp file: %w", err) + } + defer os.Remove(tmp.Name()) + if _, err := tmp.WriteString(content); err != nil { + tmp.Close() + return fmt.Errorf("write_fixture write temp: %w", err) + } + tmp.Close() + if err := sandbox.Upload(sandboxName, tmp.Name(), remoteDest); err != nil { + return fmt.Errorf("write_fixture upload: %w", err) + } + return nil + default: + return fmt.Errorf("unknown op %q", op.Op) + } +} + +func resolveWriteFixture(op BehaviourOperation) (dest string, content string, err error) { + parts := strings.SplitN(op.Args, ",", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("write_fixture args must be dest_path, fixture_path") + } + dest = strings.TrimSpace(parts[0]) + if dest == "" { + return "", "", fmt.Errorf("write_fixture requires dest_path") + } + if op.Content != "" { + return dest, op.Content, nil + } + return "", "", fmt.Errorf("write_fixture requires embedded content in script") +} + +func resolveSandboxPath(base, rel string) string { + if filepath.IsAbs(rel) || strings.HasPrefix(rel, sandbox.SandboxWorkspace) { + return rel + } + return filepath.Join(base, rel) +} + +func writeBehaviourResults(sandboxName string, results BehaviourResults) error { + data, err := json.MarshalIndent(results, "", " ") + if err != nil { + return fmt.Errorf("marshaling behaviour results: %w", err) + } + tmp, err := os.CreateTemp("", "behaviour-results-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + tmp.Close() + remotePath := filepath.Join(sandbox.SandboxWorkspace, "output", behaviourResultsFile) + return sandbox.Upload(sandboxName, tmp.Name(), remotePath) +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" +} diff --git a/internal/runtime/dummy_test.go b/internal/runtime/dummy_test.go new file mode 100644 index 0000000000..f858e7608b --- /dev/null +++ b/internal/runtime/dummy_test.go @@ -0,0 +1,54 @@ +package runtime + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadBehaviourScript(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "current-scenario.yaml") + content := `ops: + - description: Emit JSON + op: write_fixture + args: output/agent-result.json, fixtures/triage/sufficient.json + content: '{"action":"sufficient"}' +` + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + + script, err := LoadBehaviourScript(path) + require.NoError(t, err) + require.Len(t, script.Ops, 1) + assert.Equal(t, "Emit JSON", script.Ops[0].Description) + assert.Equal(t, "write_fixture", script.Ops[0].Op) + assert.Contains(t, script.Ops[0].Content, "sufficient") +} + +func TestResolveWriteFixtureEmbeddedContent(t *testing.T) { + t.Parallel() + + dest, content, err := resolveWriteFixture(BehaviourOperation{ + Op: "write_fixture", + Args: "output/agent-result.json, fixtures/triage/sufficient.json", + Content: "hello", + }) + require.NoError(t, err) + assert.Equal(t, "output/agent-result.json", dest) + assert.Equal(t, "hello", content) +} + +func TestResolveWriteFixtureMissingContent(t *testing.T) { + t.Parallel() + + _, _, err := resolveWriteFixture(BehaviourOperation{ + Op: "write_fixture", + Args: "output/agent-result.json, fixtures/triage/sufficient.json", + }) + require.Error(t, err) +} diff --git a/internal/runtime/registry.go b/internal/runtime/registry.go new file mode 100644 index 0000000000..2fbcd0ab95 --- /dev/null +++ b/internal/runtime/registry.go @@ -0,0 +1,31 @@ +package runtime + +import ( + "fmt" + "strings" + + "github.com/fullsend-ai/fullsend/internal/config" +) + +// Resolve returns the agent backend for the given runtime name. +func Resolve(name string) (Backend, error) { + switch name { + case "", "claude": + r := ClaudeRuntime{} + return Backend{Runtime: r, Transcripts: r}, nil + case "dummy": + r := DummyRuntime{} + return Backend{Runtime: r, Transcripts: r}, nil + default: + return Backend{}, fmt.Errorf("unknown runtime %q: must be one of %s", name, strings.Join(config.ValidRuntimes(), ", ")) + } +} + +// ResolveFromConfig selects the runtime backend from org config defaults. +func ResolveFromConfig(cfg *config.OrgConfig) (Backend, error) { + rt := "claude" + if cfg != nil && cfg.Defaults.Runtime != "" { + rt = cfg.Defaults.Runtime + } + return Resolve(rt) +} diff --git a/internal/runtime/registry_test.go b/internal/runtime/registry_test.go new file mode 100644 index 0000000000..26ebbf3ebe --- /dev/null +++ b/internal/runtime/registry_test.go @@ -0,0 +1,37 @@ +package runtime + +import ( + "testing" + + "github.com/fullsend-ai/fullsend/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolve(t *testing.T) { + t.Parallel() + + claude, err := Resolve("claude") + require.NoError(t, err) + assert.Equal(t, "claude", claude.Runtime.Name()) + + dummy, err := Resolve("dummy") + require.NoError(t, err) + assert.Equal(t, "dummy", dummy.Runtime.Name()) + + _, err = Resolve("unknown") + require.Error(t, err) +} + +func TestResolveFromConfig(t *testing.T) { + t.Parallel() + + defaultBackend, err := ResolveFromConfig(nil) + require.NoError(t, err) + assert.Equal(t, "claude", defaultBackend.Runtime.Name()) + + cfg := &config.OrgConfig{Defaults: config.RepoDefaults{Runtime: "dummy"}} + dummyBackend, err := ResolveFromConfig(cfg) + require.NoError(t, err) + assert.Equal(t, "dummy", dummyBackend.Runtime.Name()) +} diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 103d752ec5..d1c124ce0e 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -18,6 +18,7 @@ type RunParams struct { AgentBaseName string Model string RepoDir string + FullsendDir string PluginDirs []string Debug string Timeout time.Duration @@ -48,7 +49,7 @@ type Backend struct { Transcripts TranscriptHandler } -// Default returns the configured agent backend (Claude Code today). +// Default returns the Claude Code backend. Prefer ResolveFromConfig for org-aware selection. func Default() Backend { r := ClaudeRuntime{} return Backend{Runtime: r, Transcripts: r}