feat: add functional test framework for agent pipelines

.github/workflows/functional-tests.yml

@@ -0,0 +1,177 @@
+name: Functional Tests
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'eval/**'
+      - 'internal/scaffold/**'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'eval/**'
+      - 'internal/scaffold/**'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: functional-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  functional-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v6.0.2
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - uses: actions/setup-python@v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7.6.0
+
+      - name: Install agent-eval-harness
+        # Installs from the git submodule checked out above (submodules: true)
+        run: uv pip install --system -e 'eval/.agent-eval-harness[anthropic]'
+
+      - name: Install yq
+        run: |
+          curl -sSfL "https://github.com/mikefarah/yq/releases/download/v4.47.1/yq_linux_amd64" -o /usr/local/bin/yq
+          chmod +x /usr/local/bin/yq
+
+      - name: Configure git identity
+        run: |
+          git config --global user.name "fullsend-eval[bot]"
+          git config --global user.email "fullsend-eval[bot]@users.noreply.github.com"
+
+      - name: Build fullsend
+        run: make go-build
+
+      - name: Add bin to PATH
+        run: echo "${{ github.workspace }}/bin" >> "$GITHUB_PATH"
+
+      # TODO: The openshell setup below (version, CLI, gateway, Podman,
+      # gateway start) is duplicated from action.yml. Extract into a
+      # shared script (e.g. .github/scripts/setup-openshell.sh) so the
+      # version and config stay in sync across both places.
+      - name: Set OpenShell version
+        run: echo "OPENSHELL_VERSION=0.0.38" >> "${GITHUB_ENV}"
+
+      - name: Install OpenShell CLI
+        run: |
+          uv tool install "openshell==${OPENSHELL_VERSION}"
+          openshell --version
+
+      - name: Download openshell-gateway
+        run: |
+          set -euo pipefail
+          arch="$(uname -m)"
+          case "${arch}" in
+            x86_64) ;;
+            aarch64|arm64) arch=aarch64 ;;
+            *) echo "::error::Unsupported architecture: ${arch}"; exit 1 ;;
+          esac
+          GATEWAY_ASSET="openshell-gateway-${arch}-unknown-linux-gnu.tar.gz"
+          GATEWAY_URL="https://github.com/NVIDIA/OpenShell/releases/download/v${OPENSHELL_VERSION}/${GATEWAY_ASSET}"
+          curl -fsSL "${GATEWAY_URL}" -o "/tmp/${GATEWAY_ASSET}"
+          tar xzf "/tmp/${GATEWAY_ASSET}" -C "${{ runner.temp }}"
+          rm -f "/tmp/${GATEWAY_ASSET}"
+
+      - name: Install Podman
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y podman
+
+      - name: Configure rootless Podman
+        run: |
+          whoami_user="$(whoami)"
+          grep -q "^${whoami_user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${whoami_user}"
+          podman system migrate
+
+      - name: Start Podman API service
+        run: |
+          SOCKET_PATH="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/podman/podman.sock"
+          if [ ! -S "${SOCKET_PATH}" ]; then
+            mkdir -p "$(dirname "${SOCKET_PATH}")"
+            podman system service --time=0 "unix://${SOCKET_PATH}" &
+            for _i in $(seq 1 30); do
+              [ -S "${SOCKET_PATH}" ] && podman --url "unix://${SOCKET_PATH}" info >/dev/null 2>&1 && break
+              sleep 1
+            done
+            [ -S "${SOCKET_PATH}" ] || { echo "::error::Podman socket not ready"; exit 1; }
+          fi
+
+      - name: Start openshell-gateway
+        run: |
+          set -euo pipefail
+          OPENSHELL_SSH_HANDSHAKE_SECRET="ci-$(openssl rand -hex 16)"
+          export OPENSHELL_SSH_HANDSHAKE_SECRET
+          echo "::add-mask::${OPENSHELL_SSH_HANDSHAKE_SECRET}"
+          export OPENSHELL_SUPERVISOR_IMAGE="ghcr.io/nvidia/openshell/supervisor:dfd47683e7da4f1a4a8fa5d77f92d3696e6a41f9"
+          "${{ runner.temp }}/openshell-gateway" \
+            --bind-address 0.0.0.0 \
+            --health-port 8081 \
+            --drivers podman \
+            --disable-tls \
+            --db-url "sqlite:/tmp/gateway.db?mode=rwc" \
+            >/tmp/gateway.log 2>&1 &
+          for _i in $(seq 1 30); do
+            curl -sf http://127.0.0.1:8081/healthz >/dev/null 2>&1 && break
+            sleep 2
+          done
+          curl -sf http://127.0.0.1:8081/healthz >/dev/null 2>&1 || {
+            echo "::error::Gateway health check failed"
+            cat /tmp/gateway.log 2>/dev/null || true
+            exit 1
+          }
+          openshell gateway add http://127.0.0.1:8080 --local --name local
+          openshell gateway select local
+
+      - name: Install validation dependencies
+        run: pip install --quiet "jsonschema>=4.18.0"
+
+      - name: Authenticate to GCP
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ secrets.E2E_GCP_WIF_PROVIDER }}
+          service_account: ${{ secrets.E2E_GCP_SERVICE_ACCOUNT }}
+
+      - name: Prepare sandbox credentials
+        run: |
+          echo "HOST_GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" >> "$GITHUB_ENV"
+          bash internal/scaffold/fullsend-repo/scripts/prepare-sandbox-credentials.sh
+
+      - name: Run functional tests
+        env:
+          EVAL_ORG: ${{ vars.EVAL_ORG }}
+          GH_TOKEN: ${{ secrets.EVAL_GH_TOKEN }}
+          ANTHROPIC_VERTEX_PROJECT_ID: ${{ vars.EVALS_VERTEX_PROJECT_ID }}
+          GOOGLE_CLOUD_PROJECT: ${{ secrets.E2E_GCP_PROJECT_ID }}
+          CLOUD_ML_REGION: ${{ vars.EVALS_GCP_REGION }}
+          EVALS_HOST_CREDENTIALS: ${{ env.HOST_GOOGLE_APPLICATION_CREDENTIALS }}
+        run: make functional-tests
+
+      - name: Scrub secrets from eval results
+        if: always()
+        run: find eval/runs/ -name '.eval-env' -delete 2>/dev/null || true; find /tmp/agent-eval/ -name '.eval-env' -delete 2>/dev/null || true
+
+      - name: Upload eval results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-results
+          path: |
+            eval/runs/
+            !eval/runs/**/.eval-env
+          retention-days: 30

.gitignore

@@ -21,3 +21,4 @@ bin/
 .env.*
 !.env.example
 .transcripts/
+eval/runs/

.gitmodules

@@ -2,3 +2,7 @@
 	path = experiments
 	url = git@github.com:fullsend-ai/experiments.git
 	branch = main
+[submodule "eval/.agent-eval-harness"]
+	path = eval/.agent-eval-harness
+	url = https://github.com/ralphbean/agent-eval-harness.git
+	branch = worktree-execution-hooks

Makefile

@@ -2,7 +2,8 @@
 .PHONY: help bootstrap lint lint-all check fmt \
        mindmap go-build go-test go-lint go-fmt go-vet go-tidy \
        lint-md-links script-test test \
-       e2e-test e2e-playwright e2e-export-session e2e-upload-session
+       e2e-test e2e-playwright e2e-export-session e2e-upload-session \
+       lint-eval-cases functional-tests
 
 # Let Go automatically download the toolchain version required by go.mod.
 # This ensures local builds use the right version without manual intervention.
@@ -30,6 +31,8 @@ help:
 	@echo "  e2e-test             - Run admin e2e tests (requires E2E_GITHUB_SESSION_FILE or E2E_GITHUB_USERNAME + E2E_GITHUB_PASSWORD)"
 	@echo "  e2e-export-session   - Login to GitHub and export a Playwright session file"
 	@echo "  e2e-upload-session   - Export session and upload it as a GitHub repo secret"
+	@echo "  lint-eval-cases      - Lint eval case definitions (annotations.yaml completeness)"
+	@echo "  functional-tests     - Run functional agent tests (requires EVAL_ORG, FULLSEND_DIR, GH_TOKEN, GCP creds)"
 
 # Install all development tools needed for linting, formatting, and pre-commit hooks.
 # Prerequisites: uv (https://docs.astral.sh/uv/) and go (https://go.dev/)
@@ -118,7 +121,7 @@ script-test:
 	python3 internal/scaffold/fullsend-repo/scripts/process-fix-result-test.py
 	python3 skills/topissues/scripts/topissues_test.py
 
-test: lint-all go-test script-test
+test: lint-all go-test script-test lint-eval-cases
 
 E2E_SESSION_FILE ?= $(CURDIR)/.playwright/session.json
 
@@ -149,3 +152,19 @@ e2e-playwright:
 		echo "==> Installing Playwright Chromium..."; \
 		go run github.com/playwright-community/playwright-go/cmd/playwright install chromium; \
 	fi
+
+# Functional agent evals — run agents against ephemeral GitHub repos and judge results.
+# Required env: EVAL_ORG (GitHub org for ephemeral repos), plus GCP creds for Vertex AI.
+# GH_TOKEN defaults to `gh auth token` if not set.
+FULLSEND_DIR ?= $(CURDIR)/internal/scaffold/fullsend-repo
+EVAL_AGENTS  ?= triage
+
+lint-eval-cases:
+	@for agent in $(EVAL_AGENTS); do \
+		./eval/lint-cases.sh "$$agent"; \
+	done
+
+functional-tests: lint-eval-cases
+	@for agent in $(EVAL_AGENTS); do \
+		FULLSEND_DIR="$(FULLSEND_DIR)" ./eval/run-functional.sh "$$agent"; \
+	done

docs/ADRs/0051-agent-eval-harness-for-test-infrastructure.md

@@ -0,0 +1,72 @@
+---
+title: "47. agent-eval-harness for test infrastructure"
+status: Accepted
+relates_to:
+  - testing-agents
+topics:
+  - testing
+---
+
+# 47. agent-eval-harness for test infrastructure
+
+Date: 2026-05-29
+
+## Status
+
+Accepted
+
+<!-- Once this ADR is Accepted, its content is frozen. Do not edit the Context,
+     Decision, or Consequences sections. If circumstances change, write a new
+     ADR that supersedes this one. Only status changes and links to superseding
+     ADRs should be added after acceptance. -->
+
+## Context
+
+[ADR 0052](0052-functional-tests-for-agent-pipelines.md) establishes
+functional tests as a test category for agent pipelines. That decision is
+silent on which framework orchestrates them — it could be custom scripts,
+Inspect AI, or something else.
+
+Building test infrastructure from scratch is expensive and tangential to our
+core problem. We need test case management, judge orchestration, scoring,
+threshold gating, and regression detection. We do not need to build any of
+these ourselves.
+
+[agent-eval-harness](https://github.com/opendatahub-io/agent-eval-harness)
+is a generic evaluation framework for agents and skills. It provides dataset
+management, LLM-graded and deterministic judges, scoring pipelines, MLflow
+integration, and an
+[opaque CLI runner contract](https://github.com/opendatahub-io/agent-eval-harness/blob/main/docs/opaque-cli-runner-contract.md)
+that delegates execution to an external command. The CLI runner was added in
+[issue #59](https://github.com/opendatahub-io/agent-eval-harness/issues/59),
+which we filed specifically to make `fullsend run` testable without forking
+or extending the harness with fullsend-specific code.
+
+## Decision
+
+We adopt agent-eval-harness as the framework for fullsend functional tests.
+Fullsend's `eval/scripts/run-fullsend.sh` implements the opaque CLI runner
+contract — it accepts a workspace and output directory, runs `fullsend run`
+inside a sandbox, and writes captured fixture state to the output directory.
+Everything upstream (case iteration, judge invocation, scoring, thresholds)
+is handled by agent-eval-harness.
+
+When adding new test capabilities, prefer extending or contributing to
+agent-eval-harness over building fullsend-specific tooling.
+
+## Consequences
+
+- Fullsend functional tests inherit agent-eval-harness capabilities (MLflow
+  logging, pairwise comparison, dataset generation) without building them.
+- The opaque CLI runner contract is the integration boundary. Fullsend owns
+  execution; the harness owns everything else.
+- agent-eval-harness becomes a runtime dependency for test execution, adding
+  a Python dependency alongside the Go codebase.
+- Bugs or gaps in agent-eval-harness may require upstream contributions. We
+  have already done this once (issue #59).
+- Future prompt evals (layer 2 in the test pyramid) can reuse the same
+  harness with a different runner, keeping test infrastructure unified.
+- This decision, like any ADR, can be reversed or superseded. If we find a
+  better framework or discover that agent-eval-harness limits us in practice,
+  we can switch. The purpose of this ADR is to drive consistency for the
+  foreseeable future, not to lock us in permanently.