fullsend-ai · ifireball · Jun 7, 2026
@@ -96,3 +96,34 @@ jobs:
           path: ${{ runner.temp }}/e2e-screenshots/
           if-no-files-found: ignore
           retention-days: 5
+
+  behaviour:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Check for behaviour test token
+        id: behaviour-secrets
+        run: |
+          if [ -z "$E2E_BEHAVIOUR_GITHUB_TOKEN" ]; then
+            echo "::warning::E2E_BEHAVIOUR_GITHUB_TOKEN not set. Skipping behaviour tests."
+            echo "available=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          fi
+        env:
+          E2E_BEHAVIOUR_GITHUB_TOKEN: ${{ secrets.E2E_BEHAVIOUR_GITHUB_TOKEN }}
+
+      - name: Run behaviour tests
+        if: steps.behaviour-secrets.outputs.available == 'true'
+        run: make behaviour-test
+        env:
+          GITHUB_TOKEN: ${{ secrets.E2E_BEHAVIOUR_GITHUB_TOKEN }}
+          BEHAVIOUR_SCM: github
+          BEHAVIOUR_CI: githubactions
+          BEHAVIOUR_INSTALL_MODE: per-org
@@ -2,7 +2,7 @@
 .PHONY: help bootstrap lint lint-all check fmt \
        mindmap go-build go-test go-lint go-fmt go-vet go-tidy \
        lint-md-links script-test test \
-       e2e-test e2e-playwright e2e-export-session e2e-upload-session
+       e2e-test e2e-playwright e2e-export-session e2e-upload-session behaviour-test
 
 # Let Go automatically download the toolchain version required by go.mod.
 # This ensures local builds use the right version without manual intervention.
@@ -30,6 +30,7 @@ help:
 	@echo "  e2e-test             - Run admin e2e tests (requires E2E_GITHUB_SESSION_FILE or E2E_GITHUB_USERNAME + E2E_GITHUB_PASSWORD)"
 	@echo "  e2e-export-session   - Login to GitHub and export a Playwright session file"
 	@echo "  e2e-upload-session   - Export session and upload it as a GitHub repo secret"
+	@echo "  behaviour-test       - Run Gherkin behaviour tests (requires GITHUB_TOKEN and behaviour org pool)"
 
 # Install all development tools needed for linting, formatting, and pre-commit hooks.
 # Prerequisites: uv (https://docs.astral.sh/uv/) and go (https://go.dev/)
@@ -132,6 +133,13 @@ e2e-test: e2e-playwright
 	fi; \
 	go test -tags e2e -v -count=1 -timeout 30m ./e2e/admin/
 
+behaviour-test:
+	@if [ -z "$$GITHUB_TOKEN" ] && [ -z "$$GH_TOKEN" ]; then \
+		echo "GITHUB_TOKEN or GH_TOKEN is required for behaviour tests"; \
+		exit 1; \
+	fi
+	cd e2e/behaviour && go test -tags behaviour -v -count=1 -timeout 30m .
+
 e2e-export-session: e2e-playwright
 	@if [ -n "$$E2E_GITHUB_PASSWORD_FILE" ] && [ -z "$$E2E_GITHUB_PASSWORD" ]; then \
 		export E2E_GITHUB_PASSWORD="$$(cat "$$E2E_GITHUB_PASSWORD_FILE")"; \

@@ -0,0 +1,30 @@
+---
+status: Accepted
+date: 2026-06-07
+relates_to:
+  - agent-infrastructure
+  - agent-architecture
+---
+
+# Behaviour tests with Gherkin and pluggable drivers
+
+## Context
+
+Fullsend needs end-to-end tests that validate **deterministic platform behaviour** — dispatch routing, harness loading, schema validation, post-scripts, token scoping, sandbox policy, and SCM mutations — without depending on LLM output. This is distinct from admin install e2e ([ADR 0040](0040-org-pool-for-parallel-e2e-tests.md)) and from LLM/instruction testing ([testing-agents.md](../problems/testing-agents.md)).
+
+Runtime selection is shared with production via `defaults.runtime` in org `config.yaml` ([runtimes.md](../runtimes.md)). Harness definitions remain as in [ADR 0024](0024-harness-definitions.md). Per-repo install mode ([ADR 0033](0033-per-repo-installation-mode.md)) is deferred for behaviour v1.
+
+## Decision
+
+- Add **behaviour tests** under `e2e/behaviour/` using **godog** and portable Gherkin feature files.
+- Exercise **real SCM + real CI** through **driver interfaces** (`scm.Driver`, `ci.Driver`, `env.Setup`); v1 implementations target GitHub and GitHub Actions.
+- Substitute inference with a **dummy runtime** (`defaults.runtime: dummy`) that executes scripted operations in the real OpenShell sandbox and emits `behaviour-results.json`.
+- Select backends via **runner env** (`BEHAVIOUR_SCM`, `BEHAVIOUR_CI`, `BEHAVIOUR_INSTALL_MODE`); feature files stay install-mode agnostic. v1 runs **per-org only** against the halfsend org pool.
+- Use **compatibility tags** (`@skip:*`, `@requires:*`) to filter scenarios for future backends; tags do not select configuration.
+
+## Consequences
+
+- Behaviour tests can pass while prompt quality regresses; LLM evals remain necessary for instruction coverage.
+- Behaviour orgs must be installed with `--runtime dummy`; production orgs must not use dummy unintentionally.
+- Adding GitLab or Tekton requires new drivers and runner env values, not feature file rewrites.
+- Dummy runtime op vocabulary stays minimal; new ops require runtime + docs updates when scenarios need them.
@@ -100,7 +100,11 @@ This is the thing that actually reasons and acts. Everything else in this docume
 
 **Decided (implementation):**
 
-- The `fullsend run` runner delegates in-sandbox agent execution to a `runtime.Runtime` interface; the MVP registers Claude Code only. Bootstrap uses a portable `BootstrapInput` interface with optional extensions such as `ClaudeHooksBootstrap` for sandbox tool hooks. Transcript and debug artifact handling use a separate `TranscriptHandler` interface. See [runtimes.md](runtimes.md) for the per-runtime security feature matrix required when adding a new backend.
+- The `fullsend run` runner delegates in-sandbox agent execution to a `runtime.Runtime` interface; production orgs default to Claude Code. Runtime selection is configured in `defaults.runtime` on the org `config.yaml` and resolved via `runtime.ResolveFromConfig()`. A **dummy** runtime executes scripted operations in the real OpenShell sandbox for behaviour tests (inference removed). Bootstrap uses a portable `BootstrapInput` interface with optional extensions such as `ClaudeHooksBootstrap` for sandbox tool hooks. Transcript and debug artifact handling use a separate `TranscriptHandler` interface. See [runtimes.md](runtimes.md) for the per-runtime security feature matrix required when adding a new backend.
+
+### Behaviour testing
+
+End-to-end **behaviour tests** under `e2e/behaviour/` validate deterministic platform code — dispatch routing, harness loading, sandbox policy, SCM mutations — with the LLM layer removed via the dummy runtime. Tests exercise real GitHub and GitHub Actions through pluggable SCM and CI drivers; Gherkin scenarios stay install-mode agnostic while runner env vars select backends. This coverage is **orthogonal** to LLM and instruction testing in [testing-agents.md](problems/testing-agents.md). See [ADR 0043](ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md).
 
 **Open questions:**
 

@@ -35,4 +35,6 @@ Guides for contributors developing and testing fullsend itself.
 
 - [Local development](dev/local-dev.md) — Run fullsend agents locally on macOS and Linux (amd64 + arm64)
 - [CLI internals](dev/cli-internals.md) — Command structure, installation pipeline, and sandbox runtime
+- [Behaviour testing](dev/behaviour-testing.md) — Write Gherkin scenarios for end-to-end agent behaviour
+- [Behaviour test drivers](dev/behaviour-drivers.md) — Implement SCM and CI drivers for behaviour tests
 - [Testing workflow changes](dev/testing-workflows.md) — Point a live GitHub org at a branch to test workflow, action, and agent changes before release
@@ -0,0 +1,58 @@
+# Behaviour test drivers
+
+Behaviour tests isolate forge-specific code behind drivers so Gherkin scenarios stay portable.
+
+## Interfaces
+
+| Interface | Package | Responsibility |
+|-----------|---------|----------------|
+| `scm.Driver` | `e2e/behaviour/drivers/scm` | Issues, comments, labels (via GetIssue), file commits |
+| `ci.Driver` | `e2e/behaviour/drivers/ci` | Workflow polling, logs, artifact download |
+| `env.Setup` | `e2e/behaviour/drivers/env` | Validate org pool org has per-org install + enrolled test repo |
+
+v1 reference implementations:
+
+- `e2e/behaviour/drivers/scm/github/`
+- `e2e/behaviour/drivers/ci/githubactions/`
+- `e2e/behaviour/drivers/env/` (`PerOrg`)
+
+## Runner configuration
+
+Set when starting the suite (not in feature files):
+
+```
+BEHAVIOUR_SCM=github              # future: gitlab, forgejo
+BEHAVIOUR_CI=githubactions        # future: tekton, gitlabci
+BEHAVIOUR_INSTALL_MODE=per-org    # v1 default and only supported value
+```
+
+The suite in `e2e/behaviour/suite_test.go` reads these env vars, validates them, and constructs concrete drivers.
+
+## Adding an SCM driver
+
+1. Implement `scm.Driver` in `e2e/behaviour/drivers/scm/<vendor>/`.
+2. Register the driver in `suite_test.go` when `BEHAVIOUR_SCM=<vendor>`.
+3. Document the env var value here.
+4. Add `@skip:<vendor>` tags on scenarios that cannot run until the driver is complete.
+
+Use `forge.Client` for operations it already exposes; add REST helpers inside the driver package only when necessary (e.g. `GetIssue` with labels).
+
+## Adding a CI driver
+
+1. Implement `ci.Driver` — `WaitForWorkflow`, `AssertNoWorkflow`, `GetRunLogs`, `DownloadArtifacts`.
+2. Map forge `WorkflowRun` types to portable polling logic; reuse patterns from `e2e/admin/admin_test.go`.
+3. Register in suite init for the matching `BEHAVIOUR_CI` value.
+
+## Step definitions
+
+Steps must **not** import `internal/forge/github` directly — only drivers. This keeps scenarios vendor-agnostic.
+
+## Testing drivers
+
+Prefer unit tests with `httptest` for REST helpers. Optional smoke scenarios against live backends mirror admin e2e credentials (`GITHUB_TOKEN`, halfsend org pool).
+
+## Future backends checklist
+
+- [ ] GitLab SCM driver + `@skip:gitlab` tag removal
+- [ ] Tekton or GitLab CI driver
+- [ ] Per-repo install mode matrix + `@requires:per-repo` scenarios
@@ -0,0 +1,79 @@
+# Behaviour testing
+
+End-to-end Gherkin tests under `e2e/behaviour/` validate **deterministic platform code** with inference removed. They are **orthogonal** to LLM and instruction testing in [testing-agents.md](../../problems/testing-agents.md) and to admin install e2e in `e2e/admin/`.
+
+| | Behaviour tests | LLM evals | Admin e2e | Unit tests |
+|---|-----------------|-----------|-----------|------------|
+| **Target** | Platform workflows, sandbox, SCM | Prompts, models | Install/uninstall | Go functions |
+| **Inference** | Dummy runtime | Real LLM | Real LLM | N/A |
+| **Infrastructure** | Live GitHub + GHA | Varies | Live GitHub + GHA | None |
+
+## When to add a behaviour test
+
+Add one when a **user-visible workflow** must be verified end-to-end (dispatch → workflow → post-script → SCM state) and the assertion is **binary**. Prefer unit tests for pure Go logic and admin e2e for install provisioning.
+
+## Layout
+
+```
+e2e/behaviour/
+  features/          # Portable Gherkin scenarios
+  fixtures/          # Static content for write_fixture ops
+  steps/             # Step definitions
+  world/             # Scenario state
+  drivers/           # SCM, CI, env interfaces + v1 impls
+  suite_test.go      # godog entry (build tag: behaviour)
+```
+
+## Writing scenarios
+
+Describe **user-visible behaviour** only. Do not encode SCM vendor, CI platform, or install mode in feature files.
+
+### Dummy agent tables
+
+```gherkin
+Given a dummy agent that would:
+  | description      | op            | args                                                      |
+  | Emit triage JSON | write_fixture | output/agent-result.json, fixtures/triage/sufficient.json |
+```
+
+| Column | Meaning |
+|--------|---------|
+| `description` | Human label matched by assertion steps |
+| `op` | `read_file`, `url_get`, `run_command`, `write_fixture` |
+| `args` | Op-specific; see below |
+
+**`write_fixture`:** `dest_path, fixtures/...` — content lives in `e2e/behaviour/fixtures/`, embedded in the committed scenario script at `.fullsend/behaviour/current-scenario.yaml`.
+
+### Assertion steps
+
+```gherkin
+Then the agent will succeed to Emit triage JSON
+And the agent will fail to Search for foo
+And the agent will output issues.out with:
+  """
+  expected content
+  """
+```
+
+### Compatibility tags
+
+Use tags only for **exceptions** when a backend cannot run a scenario yet: `@skip:gitlab`, `@skip:per-org`, `@requires:per-repo`. Untagged scenarios run everywhere applicable.
+
+## Running locally
+
+```bash
+export GITHUB_TOKEN=...   # PAT with access to halfsend org pool
+make behaviour-test
+```
+
+Test orgs (`halfsend-01` … `halfsend-06`) must have per-org fullsend installed with `--runtime dummy` and `test-repo` enrolled.
+
+Runner env (defaults shown):
+
+```
+BEHAVIOUR_SCM=github
+BEHAVIOUR_CI=githubactions
+BEHAVIOUR_INSTALL_MODE=per-org
+```
+
+See [behaviour-drivers.md](behaviour-drivers.md) for driver configuration and [ADR 0043](../../ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md) for the decision record.
@@ -8,6 +8,8 @@ Testing application code is a solved problem with mature tooling: unit tests, in
 
 Today, if someone modifies a review agent's instructions, the only verification is human review of the prose change. There is no automated way to confirm the agent still behaves correctly after the modification. This is the equivalent of shipping code changes with no test suite — something we would never accept for application code.
 
+**Behaviour tests are orthogonal.** [ADR 0043](../ADRs/0043-behaviour-tests-with-gherkin-and-drivers.md) describes Gherkin end-to-end tests under `e2e/behaviour/` that validate deterministic platform code (workflows, harness, sandbox policy, post-scripts) with inference explicitly removed via the dummy runtime. They do not evaluate instructions, prompts, or models, and they do not replace golden-set or statistical LLM evals described below.
+
 ## What makes agent testing hard
 
 ### Non-determinism

@@ -1,8 +1,15 @@
 # Agent runtimes
 
-Fullsend's `fullsend run` command delegates in-sandbox agent execution to a pluggable **runtime**. Today only **Claude Code** is registered; the `internal/runtime` package defines the contracts new runtimes must implement.
+Fullsend's `fullsend run` command delegates in-sandbox agent execution to a pluggable **runtime**. Recognized values in org `config.yaml` `defaults.runtime` are **`claude`** (production default) and **`dummy`** (behaviour tests only). Install with `fullsend admin install --runtime dummy` on dedicated test orgs. The runner resolves the backend via `runtime.ResolveFromConfig()` after loading the org config.
 
-When adding a runtime, fill in the security matrix below and wire the implementation through `runtime.Default()`.
+When adding a runtime, fill in the security matrix below and register it in `runtime.Resolve()`.
+
+## Registered runtimes
+
+| Runtime | Purpose | Inference |
+|---------|---------|-----------|
+| `claude` | Production agent runs via Claude Code | Required |
+| `dummy` | Behaviour tests — scripted ops in real sandbox | None |
 
 ## Security feature matrix
 

@@ -1,4 +1,4 @@
-//go:build e2e
+//go:build e2e || behaviour
 
 package admin
 
@@ -196,6 +196,11 @@ func releaseLock(ctx context.Context, client forge.Client, org, runID string, t
 	t.Logf("[e2e-lock] Lock released (run: %s)", truncateUUID(runID))
 }
 
+// ReleaseLock deletes the org lock repo when the run still holds it.
+func ReleaseLock(ctx context.Context, client forge.Client, org, runID string, t *testing.T) {
+	releaseLock(ctx, client, org, runID, t)
+}
+
 // tryReclaimStaleLock checks whether the lock on org is stale (older than
 // staleLockTimeout) and force-acquires it if so. Returns true if the lock
 // was reclaimed. This runs during the first pass so stale locks from

@@ -1,4 +1,4 @@
-//go:build e2e
+//go:build e2e || behaviour
 
 package admin
 
@@ -302,3 +302,18 @@ func retryOnNotFound(ctx context.Context, maxAttempts int, fn func() error) erro
 	}
 	return err
 }
+
+// AcquireOrg exports org pool acquisition for behaviour tests.
+func AcquireOrg(ctx context.Context, client forge.Client, token, runID string, pool []string, timeout time.Duration, logf func(string, ...any)) (string, error) {
+	return acquireOrg(ctx, client, token, runID, pool, timeout, logf)
+}
+
+// OrgPool returns the halfsend org names used for parallel e2e runs.
+func OrgPool() []string {
+	return orgPool
+}
+
+// NewLiveClient creates a GitHub API client from a token.
+func NewLiveClient(token string) *gh.LiveClient {
+	return newLiveClient(token)
+}
@@ -0,0 +1,16 @@
+package ci
+
+import (
+	"context"
+	"time"
+
+	"github.com/fullsend-ai/fullsend/internal/forge"
+)
+
+// Driver abstracts CI workflow operations for behaviour tests.
+type Driver interface {
+	WaitForWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) (*forge.WorkflowRun, error)
+	AssertNoWorkflow(ctx context.Context, owner, repo, workflowFile string, after time.Time) error
+	GetRunLogs(ctx context.Context, owner, repo string, runID int) (string, error)
+	DownloadArtifacts(ctx context.Context, owner, repo string, runID int, destDir string) error
+}