diff --git a/Makefile b/Makefile index 600c8d0395..11b34f45c9 100644 --- a/Makefile +++ b/Makefile @@ -197,7 +197,7 @@ e2e-test: go test -tags e2e -v -count=1 -timeout 30m ./e2e/admin/ behaviour-test: - go test -tags behaviour -v -count=1 -timeout 45m ./e2e/behaviour/ + go test -tags behaviour -race -v -count=1 -timeout 45m ./e2e/behaviour/ # Functional agent evals — run agents against ephemeral GitHub repos and judge results. # Required env: EVAL_ORG (GitHub org for ephemeral repos), plus GCP creds for Vertex AI. diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md index 32ef22af7e..a3d19cd43e 100644 --- a/docs/guides/dev/behaviour-drivers.md +++ b/docs/guides/dev/behaviour-drivers.md @@ -32,7 +32,7 @@ The suite in `e2e/behaviour/suite_test.go` (or an external runner) acquires a po ### Install driver (v1 per-repo) -Uses `fullsend inference provision /test-repo` then `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with the repo-scoped WIF provider from provision (`E2E_GCP_PROJECT_ID`). Pool orgs must already have shared GitHub Apps, org-level mint enrollment, and per-repo mint enrollment for `test-repo` (one-time GCP admin step on the hosted mint project). Numbered `test-repo-01` … `test-repo-12` names are also enrolled for planned parallelization; the driver does not select them yet. The driver does not run `fullsend admin install` or `fullsend mint enroll`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). +Uses `fullsend inference provision /test-repo` then `fullsend github setup /test-repo --vendor --direct --skip-app-setup --runtime dummy` with the repo-scoped WIF provider from provision (`E2E_GCP_PROJECT_ID`). Pool orgs must already have shared GitHub Apps, org-level mint enrollment, and per-repo mint enrollment for `test-repo` (one-time GCP admin step on the hosted mint project). Numbered `test-repo-01` … `test-repo-12` are enrolled and actively leased from `world.RepoPool` during parallel scenario execution (see `GODOG_CONCURRENCY` in [behaviour-testing.md](behaviour-testing.md#parallel-execution)). The driver does not run `fullsend admin install` or `fullsend mint enroll`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). Teardown removes shim workflows, stale branches, and open fullsend PRs on `test-repo` via `pkg/e2etest.TeardownPerRepoInstall`. diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index 84fb66c000..feed4e0b8d 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -79,6 +79,30 @@ Use tags only for **exceptions** when a backend cannot run a scenario yet: `@ski make behaviour-test ``` +### Parallel execution + +The suite runs scenarios in parallel by default (`GODOG_CONCURRENCY=12`, +matching the repo pool size). Each scenario gets its own `World` clone and +leases a unique `test-repo-NN` from the pool, so no cross-scenario state +is shared. The `behaviour-test` Make target includes `-race` to catch +data races under concurrent execution. + +To adjust concurrency: + +```bash +# Run at default concurrency (12) +make behaviour-test + +# Run with explicit concurrency +GODOG_CONCURRENCY=4 make behaviour-test + +# Serial mode for debugging +GODOG_CONCURRENCY=1 make behaviour-test +``` + +Serial mode (`GODOG_CONCURRENCY=1`) is useful when debugging a single +scenario or when `-v` output from multiple scenarios would interleave. + In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`) on the hosted mint project. Pool `test-repo` repos are enrolled once by a GCP admin — not during CI install. The install driver provisions repo-scoped inference WIF via `fullsend inference provision` before `github setup`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). ### Lazy create+install (`RepoEnsurer`) diff --git a/e2e/behaviour/suite_test.go b/e2e/behaviour/suite_test.go index e9aeeb4d12..034dcc4b6f 100644 --- a/e2e/behaviour/suite_test.go +++ b/e2e/behaviour/suite_test.go @@ -5,6 +5,7 @@ package behaviour_test import ( "context" "os" + "strconv" "testing" "github.com/cucumber/godog" @@ -19,12 +20,27 @@ import ( "github.com/fullsend-ai/fullsend/pkg/e2etest" ) +// poolSize is the number of enrolled test-repo-NN repos in the pool org. +// GODOG_CONCURRENCY must not exceed this — extra workers would block in +// pool.Acquire with no warning because the pool org only has test-repo-01 +// through test-repo-12 with per-repo mint enrollment. +const poolSize = 12 + func TestBehaviourSuite(t *testing.T) { if testing.Short() { t.Skip("skipping behaviour tests in short mode") } - if c := os.Getenv("GODOG_CONCURRENCY"); c != "" && c != "1" { - t.Fatalf("behaviour suite does not support GODOG_CONCURRENCY=%q: per-scenario World isolation is in place but drivers (SCM, CI, Install) are shared by reference and have not been validated under -race with concurrent scenarios yet; see #5441 for parallel support", c) + + concurrency := poolSize + if c := os.Getenv("GODOG_CONCURRENCY"); c != "" { + n, err := strconv.Atoi(c) + if err != nil || n < 1 { + t.Fatalf("GODOG_CONCURRENCY must be a positive integer, got %q", c) + } + concurrency = n + } + if concurrency > poolSize { + t.Fatalf("GODOG_CONCURRENCY=%d exceeds repo pool size %d", concurrency, poolSize) } cfg := env.LoadRunnerConfig() @@ -65,7 +81,7 @@ func TestBehaviourSuite(t *testing.T) { } }) - pool, err := world.NewRepoPool(12) + pool, err := world.NewRepoPool(poolSize) if err != nil { t.Fatalf("creating repo pool: %v", err) } @@ -96,7 +112,7 @@ func TestBehaviourSuite(t *testing.T) { Paths: []string{"features"}, TestingT: t, Tags: os.Getenv("GODOG_TAGS"), - Concurrency: 1, + Concurrency: concurrency, }, } if st := suiteRunner.Run(); st != 0 {