Skip to content
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
ba4bb37
test(e2e): add skill agent Vitest coverage
jyaunches Jun 11, 2026
3d90c7b
test(e2e): address skill agent review feedback
jyaunches Jun 11, 2026
5a3f792
ci(e2e): wire skill agent Vitest job
jyaunches Jun 11, 2026
b282f16
test(e2e): keep skill probe command single-line
jyaunches Jun 11, 2026
7d27cd7
test(e2e): fix skill probe shell syntax
jyaunches Jun 11, 2026
5750bba
test(e2e): skip skill agent on provider outage
jyaunches Jun 11, 2026
50ceb08
ci(e2e): allow selective Vitest jobs
jyaunches Jun 11, 2026
dc492b8
test(e2e): add skill agent Vitest coverage
jyaunches Jun 11, 2026
422abb0
test(e2e): address skill agent review feedback
jyaunches Jun 11, 2026
7488799
ci(e2e): wire skill agent Vitest job
jyaunches Jun 11, 2026
ef80221
test(e2e): keep skill probe command single-line
jyaunches Jun 11, 2026
449c8d6
test(e2e): fix skill probe shell syntax
jyaunches Jun 11, 2026
8ebd2c5
test(e2e): skip skill agent on provider outage
jyaunches Jun 11, 2026
d4550b7
ci(e2e): allow selective Vitest jobs
jyaunches Jun 11, 2026
379ee2b
Merge branch 'e2e-migrate/test-skill-agent-e2e'
jyaunches Jun 11, 2026
2095029
ci(e2e): allow selective Vitest job dispatch
jyaunches Jun 11, 2026
e7152aa
test(e2e): preserve skill agent fail-closed checks
jyaunches Jun 11, 2026
cf2a3f7
test(e2e): harden skill agent workflow guards
jyaunches Jun 11, 2026
8e12853
test(e2e): narrow skill agent selectors
jyaunches Jun 11, 2026
5ae97ec
ci(e2e): validate Vitest job selectors
jyaunches Jun 11, 2026
fbafb01
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
ec346fe
test(e2e): address recovery guard feedback
jyaunches Jun 11, 2026
c415b1e
Revert "test(e2e): address recovery guard feedback"
jyaunches Jun 11, 2026
1976fc0
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
384ab71
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
c48831d
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
e8f1691
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
a8065bf
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
13c1c2b
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
464bdcf
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 11, 2026
bdb6887
test(e2e): install openshell for skill-agent vitest
jyaunches Jun 11, 2026
e51c162
Merge remote-tracking branch 'origin/main' into e2e-migrate/test-skil…
jyaunches Jun 12, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 128 additions & 2 deletions .github/workflows/e2e-vitest-scenarios.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,17 @@ on:
required: false
type: string
default: ""
jobs:
description: "Optional comma-separated free-standing live Vitest job ids. Empty runs all enabled jobs."
required: false
default: ""
type: string

permissions:
contents: read

concurrency:
group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}
group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}-${{ inputs.jobs || 'all-jobs' }}
cancel-in-progress: false

jobs:
Expand All @@ -47,6 +52,7 @@ jobs:
name: Generate Vitest scenario matrix
env:
SCENARIOS: ${{ inputs.scenarios }}
JOBS: ${{ inputs.jobs }}
run: |
set -euo pipefail
args=(--emit-live-matrix)
Expand All @@ -57,6 +63,31 @@ jobs:
fi
args+=(--scenarios "${SCENARIOS}")
fi
if [ -n "${JOBS}" ]; then
if [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
echo "::error::Invalid jobs input: ${JOBS}" >&2
exit 1
fi
allowed_jobs=(
openshell-version-pin-vitest
onboard-negative-paths-vitest
skill-agent-vitest
)
IFS=',' read -r -a requested_jobs <<< "${JOBS}"
for requested_job in "${requested_jobs[@]}"; do
known_job=0
for allowed_job in "${allowed_jobs[@]}"; do
if [[ "${requested_job}" == "${allowed_job}" ]]; then
known_job=1
break
fi
done
if [[ "${known_job}" -ne 1 ]]; then
echo "::error::Unknown jobs input: ${requested_job}" >&2
exit 1
fi
done
fi
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")"
echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
Expand All @@ -74,6 +105,7 @@ jobs:

live-scenarios:
needs: generate-matrix
if: ${{ inputs.jobs == '' }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
strategy:
Expand Down Expand Up @@ -174,6 +206,8 @@ jobs:
# because the matrix above only runs registry-scenarios.test.ts. Modeled on
# #5049's free-standing pattern.
openshell-version-pin-vitest:
needs: generate-matrix
if: ${{ needs.generate-matrix.result == 'success' && (inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',openshell-version-pin-vitest,')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
Expand Down Expand Up @@ -213,6 +247,8 @@ jobs:
retention-days: 14

onboard-negative-paths-vitest:
needs: generate-matrix
if: ${{ needs.generate-matrix.result == 'success' && (inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-vitest,')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
Expand Down Expand Up @@ -255,11 +291,94 @@ jobs:
if-no-files-found: ignore
retention-days: 14

# Focused coverage for test/e2e/test-skill-agent-e2e.sh's live skill
# injection + real OpenClaw agent-turn contract. The retained legacy bash
# lane remains in nightly-e2e.yaml until #5098 Phase 11 shell retirement.
skill-agent-vitest:
needs: generate-matrix
if: ${{ needs.generate-matrix.result == 'success' && (inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',skill-agent-vitest,')) }}
runs-on: ubuntu-latest
timeout-minutes: 30
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/skill-agent
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- name: Authenticate to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
shell: bash
run: |
set -euo pipefail
if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then
echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls."
exit 0
fi
login_succeeded=0
for attempt in 1 2 3; do
if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then
login_succeeded=1
break
fi
if [[ "$attempt" -lt 3 ]]; then
echo "::warning::Docker Hub login attempt ${attempt} failed; retrying."
sleep 5
fi
done
if [[ "$login_succeeded" -ne 1 ]]; then
echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls."
fi

- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
with:
node-version: 22
cache: npm

- name: Install root dependencies
run: npm ci --ignore-scripts

- name: Build CLI
run: npm run build:cli

- name: Run skill-agent live test
env:
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
run: |
set -euo pipefail
npx vitest run --project e2e-scenarios-live \
test/e2e-scenario/live/skill-agent.test.ts \
--silent=false --reporter=default

- name: Upload skill-agent artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-vitest-scenarios-skill-agent
path: |
e2e-artifacts/vitest/skill-agent/*/artifact-summary.json
e2e-artifacts/vitest/skill-agent/*/cleanup.json
e2e-artifacts/vitest/skill-agent/*/cleanup-skill-agent-summary.json
e2e-artifacts/vitest/skill-agent/*/scenario.json
e2e-artifacts/vitest/skill-agent/*/scenario-result.json
e2e-artifacts/vitest/skill-agent/*/shell/*.result.json
e2e-artifacts/vitest/skill-agent/*/shell/*.stdout.txt
e2e-artifacts/vitest/skill-agent/*/shell/*.stderr.txt
include-hidden-files: false
if-no-files-found: ignore
retention-days: 14

# Focused coverage slice for the #2603/#3145 OpenClaw websocket
# protocol/history contract. The retained legacy bash lane remains the
# source for full closeout until a later PR proves replacement and deletes it.
openclaw-tui-chat-correlation-vitest:
if: ${{ inputs.scenarios == '' }}
needs: generate-matrix
if: ${{ needs.generate-matrix.result == 'success' && inputs.jobs == '' }}
runs-on: ubuntu-latest
timeout-minutes: 75
env:
Expand Down Expand Up @@ -338,6 +457,8 @@ jobs:
# restore the /tmp guard chain after pod recreate). Will fail on `main`
# until the #2701 fix lands; flips green afterwards.
gateway-guard-recovery:
needs: generate-matrix
if: ${{ needs.generate-matrix.result == 'success' && inputs.jobs == '' }}
runs-on: ubuntu-latest
timeout-minutes: 45
env:
Expand Down Expand Up @@ -434,6 +555,7 @@ jobs:
live-scenarios,
openshell-version-pin-vitest,
onboard-negative-paths-vitest,
skill-agent-vitest,
openclaw-tui-chat-correlation-vitest,
gateway-guard-recovery,
]
Expand All @@ -451,6 +573,7 @@ jobs:
const workflowBranch = context.ref.replace('refs/heads/', '');
const prNumberInput = ${{ toJSON(inputs.pr_number) }} || '';
const requestedScenarios = ${{ toJSON(inputs.scenarios) }} || '';
const requestedJobs = ${{ toJSON(inputs.jobs) }} || '';

let prNumber = prNumberInput ? Number.parseInt(prNumberInput, 10) : undefined;
if (!prNumber) {
Expand Down Expand Up @@ -491,6 +614,9 @@ jobs:
requestedScenarios
? `**Requested scenarios:** \`${requestedScenarios}\``
: '**Requested scenarios:** _(default — all supported)_',
requestedJobs
? `**Requested jobs:** \`${requestedJobs}\``
: '**Requested jobs:** _(default — all enabled)_',
`**Summary:** ${passed.length} passed, ${failed.length} failed, ${skipped.length} skipped`,
'',
'| Job | Result |',
Expand Down
Loading
Loading