Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 73 additions & 32 deletions .github/workflows/functional-tests.yml
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,29 @@ on:
types: [opened, synchronize, reopened, labeled]
merge_group:
workflow_dispatch:
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
workflow_call:
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
inputs:
fullsend_ref:
description: 'fullsend ref to build from (main or release tag v*)'
required: false
type: string
default: main
secrets:
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
E2E_GCP_WIF_PROVIDER:
required: false
E2E_GCP_SERVICE_ACCOUNT:
required: false
E2E_GCP_PROJECT_ID:
required: false
EVAL_GH_TOKEN:
required: false

concurrency:
group: >-
${{ github.event_name == 'pull_request_target'
&& format('functional-{0}', github.event.pull_request.number)
|| github.repository != 'fullsend-ai/agents'
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
&& format('functional-call-{0}', github.run_id)
|| format('{0}-{1}', github.workflow, github.ref) }}
cancel-in-progress: >-
${{ github.event_name == 'pull_request_target'
Expand Down Expand Up @@ -77,6 +95,7 @@ jobs:
needs: gate
if: >-
!cancelled() &&
github.repository_owner == 'fullsend-ai' &&
(github.event_name != 'pull_request_target' || needs.gate.outputs.authorized == 'true')
runs-on: ubuntu-24.04
timeout-minutes: 5
Expand All @@ -87,7 +106,11 @@ jobs:
steps:
- name: Get changed files
id: changes
if: github.event_name != 'workflow_dispatch'
if: >-
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
github.repository == 'fullsend-ai/agents' &&
(github.event_name == 'pull_request_target' ||
github.event_name == 'push' ||
github.event_name == 'merge_group')
env:
GH_TOKEN: ${{ github.token }}
EVENT_NAME: ${{ github.event_name }}
Expand Down Expand Up @@ -138,7 +161,13 @@ jobs:

- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
repository: ${{ github.repository != 'fullsend-ai/agents' && 'fullsend-ai/agents' || github.repository }}
ref: >-
${{ github.repository != 'fullsend-ai/agents'
&& github.workflow_sha
|| (github.event_name == 'pull_request_target'
&& github.event.pull_request.head.sha
|| github.sha) }}
persist-credentials: false
allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }}

Expand Down Expand Up @@ -171,6 +200,7 @@ jobs:
needs: [gate, detect]
if: >-
!cancelled() &&
github.repository_owner == 'fullsend-ai' &&
needs.detect.outputs.agents != '[]' &&
(github.event_name != 'pull_request_target' || needs.gate.outputs.authorized == 'true')
runs-on: ubuntu-24.04
Expand All @@ -185,7 +215,13 @@ jobs:
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
repository: ${{ github.repository != 'fullsend-ai/agents' && 'fullsend-ai/agents' || github.repository }}
ref: >-
${{ github.repository != 'fullsend-ai/agents'
&& github.workflow_sha
|| (github.event_name == 'pull_request_target'
&& github.event.pull_request.head.sha
|| github.sha) }}
persist-credentials: false
allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' }}
submodules: true
Expand All @@ -211,37 +247,43 @@ jobs:
git config --global user.name "fullsend-eval[bot]"
git config --global user.email "fullsend-eval[bot]@users.noreply.github.com"

- name: Install fullsend
- name: Validate fullsend ref
env:
GH_TOKEN: ${{ github.token }}
FULLSEND_REF: ${{ inputs.fullsend_ref || 'main' }}
run: |
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
FULLSEND_VERSION="${FULLSEND_VERSION:-latest}"
if [ "$FULLSEND_VERSION" = "latest" ]; then
FULLSEND_VERSION=$(gh release view --repo fullsend-ai/fullsend --json tagName -q .tagName)
fi
if [ -z "$FULLSEND_VERSION" ]; then
echo "::error::Failed to resolve fullsend version"
if [[ "${FULLSEND_REF}" != "main" && ! "${FULLSEND_REF}" =~ ^v[0-9] ]]; then
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
echo "::error::fullsend_ref must be 'main' or a release tag (v*), got '${FULLSEND_REF}'"
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
exit 1
fi
{ echo 'FULLSEND_VERSION<<GHEOF'; echo "${FULLSEND_VERSION}"; echo 'GHEOF'; } >> "$GITHUB_ENV"
VERSION_NUM="${FULLSEND_VERSION#v}"
TARBALL="fullsend_${VERSION_NUM}_linux_amd64.tar.gz"
gh release download "$FULLSEND_VERSION" \
--repo fullsend-ai/fullsend \
--pattern "$TARBALL" \
--dir /tmp
tar xzf "/tmp/${TARBALL}" -C /usr/local/bin fullsend
chmod +x /usr/local/bin/fullsend
fullsend --version

- name: Configure OpenShell gateway
- name: Clone fullsend
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
env:
GH_TOKEN: ${{ github.token }}
FULLSEND_REF: ${{ inputs.fullsend_ref || 'main' }}
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
run: |
FULLSEND_CHECKOUT="${RUNNER_TEMP}/fullsend"
git clone --depth 1 --branch "${FULLSEND_REF}" \
"https://x-access-token:${GH_TOKEN}@github.com/fullsend-ai/fullsend.git" \
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
"${FULLSEND_CHECKOUT}"
echo "FULLSEND_CHECKOUT=${FULLSEND_CHECKOUT}" >> "$GITHUB_ENV"

- name: Setup Go
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
with:
go-version-file: ${{ env.FULLSEND_CHECKOUT }}/go.mod
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
cache-dependency-path: ${{ env.FULLSEND_CHECKOUT }}/go.sum

Comment thread
maruiz93 marked this conversation as resolved.
Resolved
- name: Build fullsend
Comment thread
maruiz93 marked this conversation as resolved.
Resolved
run: |
eval "$(gh api "repos/fullsend-ai/fullsend/contents/.github/scripts/openshell-version.sh?ref=${FULLSEND_VERSION}" --jq .content | base64 -d | grep -E '^OPENSHELL_(VERSION|SHA)=')"
make -C "${FULLSEND_CHECKOUT}" go-build
echo "${FULLSEND_CHECKOUT}/bin" >> "$GITHUB_PATH"
"${FULLSEND_CHECKOUT}/bin/fullsend" --version

- name: Configure OpenShell gateway
run: |
eval "$(grep -E '^OPENSHELL_(VERSION|SHA)=' "${FULLSEND_CHECKOUT}/.github/scripts/openshell-version.sh")"
if [ -z "$OPENSHELL_VERSION" ] || [ -z "$OPENSHELL_SHA" ]; then
SAFE_VERSION=$(echo "$FULLSEND_VERSION" | tr -d '\n\r' | sed -e 's/%0[aAdD]//g' -e 's/::/: :/g')
echo "::error::Failed to resolve OpenShell version from fullsend tag ${SAFE_VERSION}"
echo "::error::Failed to resolve OpenShell version from fullsend checkout"
exit 1
fi
echo "OPENSHELL_VERSION=${OPENSHELL_VERSION}" >> "$GITHUB_ENV"
Expand All @@ -257,14 +299,8 @@ jobs:
EOF

- name: Install OpenShell CLI
env:
GH_TOKEN: ${{ github.token }}
run: |
SCRIPT_DIR="$(mktemp -d)"
gh api "repos/fullsend-ai/fullsend/contents/.github/scripts/install-openshell.sh?ref=${FULLSEND_VERSION}" --jq .content | base64 -d > "${SCRIPT_DIR}/install-openshell.sh"
gh api "repos/fullsend-ai/fullsend/contents/.github/scripts/openshell-version.sh?ref=${FULLSEND_VERSION}" --jq .content | base64 -d > "${SCRIPT_DIR}/openshell-version.sh"
chmod +x "${SCRIPT_DIR}/install-openshell.sh"
"${SCRIPT_DIR}/install-openshell.sh"
"${FULLSEND_CHECKOUT}/.github/scripts/install-openshell.sh"

- name: Install Podman
run: |
Expand Down Expand Up @@ -370,6 +406,7 @@ jobs:
GATE_RESULT: ${{ needs.gate.result }}
DETECT_RESULT: ${{ needs.detect.result }}
TESTS_RESULT: ${{ needs.functional-tests.result }}
CROSS_REPO: ${{ github.repository != 'fullsend-ai/agents' }}
run: |
if [ "$GATE_RESULT" = "failure" ] || [ "$GATE_RESULT" = "cancelled" ]; then
echo "::error::Gate job ${GATE_RESULT}"
Expand All @@ -383,3 +420,7 @@ jobs:
echo "::error::One or more functional tests ${TESTS_RESULT}"
exit 1
fi
if [ "$CROSS_REPO" = "true" ] && [ "$TESTS_RESULT" = "skipped" ]; then
echo "::error::Cross-repo call completed with zero functional tests — agents list was empty or detect was skipped"
exit 1
fi
Loading