Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 104 additions & 1 deletion .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ permissions: {}
on:
push:
branches: [main]
# SYNC-WITH: grep regex in "Check for e2e-relevant changes" step in the e2e job
# SYNC-WITH: grep in "Check for e2e-relevant changes" (e2e job) and
Comment thread
ifireball marked this conversation as resolved.
# "Check for behaviour-relevant changes" (behaviour job). push.paths is the
# union of both filters (plus **/*.go); each job grep may be narrower.
paths:
- '**/*.go'
- 'go.mod'
Expand All @@ -19,6 +21,17 @@ on:
- 'internal/security/hooks/**'
- 'internal/dispatch/gcf/mintsrc/**'
- 'internal/sentencetoken/english.json'
- 'internal/runtime/**'
- 'internal/sandbox/**'
- 'internal/config/**'
- 'internal/cli/github.go'
- 'internal/cli/run.go'
- 'internal/layers/**'
- 'internal/forge/**'
- 'internal/harness/**'
- 'internal/dispatch/**'
- 'internal/mintclient/**'
- 'cmd/fullsend/**'
- 'Makefile'
- '.github/scripts/**'
- '.github/workflows/e2e.yml'
Expand Down Expand Up @@ -158,3 +171,93 @@ jobs:
path: ${{ runner.temp }}/e2e-screenshots/
if-no-files-found: ignore
retention-days: 5

behaviour:
# Same gate authorization as e2e — checks out untrusted PR head with secrets.
needs: gate
if: >-
!cancelled() &&
(github.event_name != 'pull_request_target' || needs.gate.outputs.authorized == 'true')
runs-on: ubuntu-24.04
timeout-minutes: 30
permissions:
contents: read
id-token: write
steps:
- name: Check for behaviour-relevant changes
id: changes
if: github.event_name == 'pull_request_target' || github.event_name == 'merge_group'
env:
GH_TOKEN: ${{ github.token }}
EVENT_NAME: ${{ github.event_name }}
PR_NUMBER: ${{ github.event.pull_request.number }}
REPO: ${{ github.repository }}
MERGE_GROUP_BASE: ${{ github.event.merge_group.base_sha }}
MERGE_GROUP_HEAD: ${{ github.event.merge_group.head_sha }}
# SYNC-WITH: push.paths behaviour entries above
run: |
if [ "$EVENT_NAME" = "merge_group" ]; then
FILES=$(gh api "repos/${REPO}/compare/${MERGE_GROUP_BASE}...${MERGE_GROUP_HEAD}" --jq '.files[].filename') || {
echo "::warning::Failed to fetch merge group files — running behaviour tests as a precaution"
echo "relevant=true" >> "$GITHUB_OUTPUT"
exit 0
}
FILE_COUNT=$(echo "$FILES" | wc -l)
if [ "$FILE_COUNT" -ge 300 ]; then
echo "::warning::Compare API returned $FILE_COUNT files (possible truncation at 300) — running behaviour tests as a precaution"
echo "relevant=true" >> "$GITHUB_OUTPUT"
exit 0
fi
else
FILES=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename') || {
echo "::warning::Failed to fetch PR files — running behaviour tests as a precaution"
echo "relevant=true" >> "$GITHUB_OUTPUT"
exit 0
}
fi
# e2e/admin: behaviour suite reuses exported admin test helpers
if echo "$FILES" | grep -qE '^e2e/behaviour/|^e2e/admin/|^internal/runtime/|^internal/sandbox/|^internal/config/|^internal/cli/|^internal/layers/|^internal/scaffold/fullsend-repo/|^internal/forge/|^internal/harness/|^internal/dispatch/|^internal/security/hooks/|^internal/mintclient/|^cmd/fullsend/|^go\.(mod|sum)$|^Makefile$|^\.github/workflows/e2e\.yml$|^\.github/actions/check-e2e-authorization/|^scripts/check-e2e-authorization\.sh$'; then
echo "relevant=true" >> "$GITHUB_OUTPUT"
else
echo "::notice::No behaviour-relevant files changed — skipping behaviour tests"
echo "relevant=false" >> "$GITHUB_OUTPUT"
fi
Comment thread
ifireball marked this conversation as resolved.
Comment thread
ifireball marked this conversation as resolved.

- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
if: steps.changes.outputs.relevant != 'false'
with:
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
persist-credentials: false
allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }}

- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
if: steps.changes.outputs.relevant != 'false'
with:
go-version-file: go.mod

- name: Authenticate to GCP
if: steps.changes.outputs.relevant != 'false'
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
with:
workload_identity_provider: ${{ secrets.E2E_GCP_WIF_PROVIDER }}
service_account: ${{ secrets.E2E_GCP_SERVICE_ACCOUNT }}

- name: Run behaviour tests
if: steps.changes.outputs.relevant != 'false'
run: make behaviour-test
env:
BEHAVIOUR_SCM: github
BEHAVIOUR_CI: githubactions
BEHAVIOUR_INSTALL_MODE: per-repo
BEHAVIOUR_ARTIFACT_DIR: ${{ runner.temp }}/behaviour-artifacts
E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID }}
E2E_GCP_WIF_PROVIDER: ${{ secrets.E2E_GCP_WIF_PROVIDER }}

- name: Upload behaviour debug artifacts
if: failure() && steps.changes.outputs.relevant != 'false'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: behaviour-artifacts-${{ github.event_name == 'pull_request_target' && github.event.pull_request.number || github.run_id }}
path: ${{ runner.temp }}/behaviour-artifacts/
if-no-files-found: ignore
retention-days: 5
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
.PHONY: help bootstrap lint lint-all check fmt \
mindmap go-build go-test go-lint go-fmt go-vet go-tidy \
lint-md-links script-test test \
e2e-test lint-eval-cases functional-tests
e2e-test behaviour-test lint-eval-cases functional-tests

# Let Go automatically download the toolchain version required by go.mod.
# This ensures local builds use the right version without manual intervention.
Expand All @@ -28,6 +28,7 @@ help:
@echo " script-test - Run shell script tests (post-triage, post-code, post-review, pre-fetch-prior-review, reconcile-repos, validate-output-schema)"
@echo " test - Run all checks: lint-all, go-test, script-test, lint-eval-cases"
@echo " e2e-test - Run admin e2e tests (CI: OIDC mint; local: gh auth login or GH_TOKEN)"
@echo " behaviour-test - Run Gherkin behaviour tests (CI: OIDC mint; local: gh auth login or GH_TOKEN)"
@echo " lint-eval-cases - Lint eval case definitions (annotations.yaml completeness)"
@echo " functional-tests - Run functional agent tests (requires EVAL_ORG, FULLSEND_DIR, GH_TOKEN, GCP creds)"

Expand Down Expand Up @@ -135,6 +136,9 @@ test: lint-all go-test script-test lint-eval-cases
e2e-test:
go test -tags e2e -v -count=1 -timeout 30m ./e2e/admin/

behaviour-test:
Comment thread
ifireball marked this conversation as resolved.
go test -tags behaviour -v -count=1 -timeout 30m ./e2e/behaviour/

# Functional agent evals — run agents against ephemeral GitHub repos and judge results.
# Required env: EVAL_ORG (GitHub org for ephemeral repos), plus GCP creds for Vertex AI.
# GH_TOKEN defaults to `gh auth token` if not set.
Expand Down
9 changes: 9 additions & 0 deletions e2e/behaviour/suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//go:build behaviour

package behaviour_test

import "testing"

// TestBehaviourCIWiring is a placeholder until the full Gherkin suite lands.
// It verifies the behaviour build tag, Makefile target, and e2e workflow job compile and run.
func TestBehaviourCIWiring(t *testing.T) {}
Loading