diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 6f9c145efec..174b24ae263 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -40,7 +40,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest" if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then echo "::error::Use either scenarios or jobs, not both." >&2 exit 1 @@ -93,12 +93,12 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,rebuild-openclaw-vitest,token-rotation-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest" args=(--emit-live-matrix) matrix="" hermes_selected=false registry_scenarios=() - free_standing_scenarios=(openshell-version-pin onboard-negative-paths inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference) + free_standing_scenarios=(openshell-version-pin onboard-negative-paths inference-routing runtime-overrides hermes-e2e hermes-root-entrypoint-smoke network-policy rebuild-openclaw token-rotation openclaw-tui-chat-correlation double-onboard issue-4434-tui-unreachable-inference model-router-provider-routed-inference) is_free_standing_scenario() { local id="$1" local known @@ -1117,6 +1117,96 @@ jobs: if-no-files-found: ignore retention-days: 14 + # Focused coverage slice for the provider-routed Model Router inference + # contract. The retained legacy bash lane remains the source for full + # closeout until a later PR proves replacement and deletes it. + model-router-provider-routed-inference-vitest: + needs: [validate-jobs, generate-matrix] + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',model-router-provider-routed-inference-vitest,') || contains(format(',{0},', inputs.scenarios), ',model-router-provider-routed-inference,') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + DOCKER_CONFIG: ${{ runner.temp }}/docker-config-model-router-provider-routed-inference + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/model-router-provider-routed-inference + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run Model Router provider-routed inference live test + # Direct Vitest coverage for + # test/e2e/test-model-router-provider-routed-inference.sh. It preserves + # the real provider-routed onboard, host model-router health, and + # sandbox inference.local completion boundaries without adding registry + # or migration-ledger wiring. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/model-router-provider-routed-inference.test.ts \ + --silent=false --reporter=default + + - name: Upload Model Router provider-routed inference artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-model-router-provider-routed-inference + path: e2e-artifacts/vitest/model-router-provider-routed-inference/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -1310,6 +1400,7 @@ jobs: token-rotation-vitest, launchable-smoke-vitest, double-onboard-vitest, + model-router-provider-routed-inference-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, issue-4434-tui-unreachable-inference-vitest, diff --git a/test/e2e-scenario/live/model-router-provider-routed-inference.test.ts b/test/e2e-scenario/live/model-router-provider-routed-inference.test.ts new file mode 100644 index 00000000000..3cbd7d187ef --- /dev/null +++ b/test/e2e-scenario/live/model-router-provider-routed-inference.test.ts @@ -0,0 +1,235 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; + +// Focused Vitest live replacement for +// test/e2e/test-model-router-provider-routed-inference.sh. Keep this as a +// direct CLI/sandbox test: the legacy contract is the real provider-routed +// onboard boundary plus host model-router health and sandbox inference.local +// completion semantics, not a new scenario registry entry. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-model-router"; +const ONBOARD_TIMEOUT_MS = 25 * 60_000; +const HEALTH_ATTEMPTS = 20; +const COMPLETION_ATTEMPTS = 3; + +interface ModelRouterHealth { + healthy_count?: unknown; +} + +interface ChatCompletionResponse { + model?: unknown; + choices?: Array<{ + message?: { content?: unknown }; + text?: unknown; + }>; +} + +function resultText(result: { stdout: string; stderr: string }): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseJson(raw: string): T | undefined { + try { + return JSON.parse(raw) as T; + } catch { + return undefined; + } +} + +function hasHealthyEndpoint(raw: string): boolean { + const health = parseJson(raw); + return typeof health?.healthy_count === "number" && health.healthy_count > 0; +} + +function routedPongReason(raw: string): "ok" | string { + const response = parseJson(raw); + if (!response) return "response was not JSON"; + const model = String(response.model ?? ""); + if (model !== "nvidia-routed" && !model.startsWith("nvidia-routed")) { + return "response model was not provider-routed"; + } + const content = (response.choices ?? []) + .map((choice) => { + if (typeof choice.message?.content === "string") return choice.message.content; + if (typeof choice.text === "string") return choice.text; + return ""; + }) + .join("\n"); + if (!/\bPONG\b/i.test(content)) return "response missing PONG content"; + return "ok"; +} + +function withProviderRoutedEnv(apiKey: string): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NVIDIA_API_KEY: apiKey, + NEMOCLAW_PROVIDER_KEY: apiKey, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_POLICY_TIER: "open", + NEMOCLAW_PROVIDER: "routed", + }; +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "model-router provider-routed onboard returns routed inference.local PONG", + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + expect( + fs.existsSync(CLI_ENTRYPOINT), + "run `npm run build:cli` before live repo CLI scenarios", + ).toBe(true); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-model-router-provider-routed", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required for provider-routed Model Router onboarding: ${resultText(docker)}`, + ); + } + skip("Docker is required for provider-routed Model Router onboarding"); + } + + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + await artifacts.writeJson("scenario.json", { + id: "model-router-provider-routed-inference", + runner: "vitest", + boundary: "direct-cli-onboard-and-sandbox-exec", + legacySource: "test/e2e/test-model-router-provider-routed-inference.sh", + contract: [ + "Docker is available before onboarding", + "NVIDIA_API_KEY is present and nvapi-prefixed", + "nemoclaw onboard --fresh completes with NEMOCLAW_PROVIDER=routed", + "host model-router health reports at least one healthy endpoint", + "sandbox inference.local returns model nvidia-routed with PONG content", + ], + }); + + const cleanEnv = buildAvailabilityProbeEnv(); + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-model-router-provider-routed", + env: cleanEnv, + timeoutMs: 120_000, + }); + + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-model-router-provider-routed", + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }); + }); + + const onboard = await host.command( + "node", + [ + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + ], + { + artifactName: "onboard-model-router-provider-routed", + env: withProviderRoutedEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + let lastHealth = ""; + for (let attempt = 1; attempt <= HEALTH_ATTEMPTS; attempt += 1) { + const health = await host.command( + "curl", + ["-s", "--max-time", "10", "http://127.0.0.1:4000/health"], + { + artifactName: `model-router-health-${attempt}`, + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 15_000, + }, + ); + lastHealth = health.stdout || health.stderr; + if (health.exitCode === 0 && hasHealthyEndpoint(lastHealth)) break; + if (attempt < HEALTH_ATTEMPTS) await sleep(3_000); + } + expect( + hasHealthyEndpoint(lastHealth), + `model-router has no healthy endpoints; expected #3255 main-equivalent failure: ${lastHealth.slice(0, 500)}`, + ).toBe(true); + + const payload = JSON.stringify({ + model: "nvidia-routed", + messages: [ + { + role: "user", + content: "Return only the exact word PONG. Do not include reasoning or any other text.", + }, + ], + max_tokens: 128, + }); + let lastCompletion = ""; + let completionReason = "not attempted"; + for (let attempt = 1; attempt <= COMPLETION_ATTEMPTS; attempt += 1) { + const completion = await sandbox.exec( + SANDBOX_NAME, + [ + "curl", + "-sk", + "--max-time", + "90", + "https://inference.local/v1/chat/completions", + "-H", + "Content-Type: application/json", + "--data-raw", + payload, + ], + { + artifactName: `sandbox-inference-local-routed-completion-${attempt}`, + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }, + ); + lastCompletion = completion.stdout || completion.stderr; + completionReason = routedPongReason(lastCompletion); + if (completion.exitCode === 0 && completionReason === "ok") break; + if (/inference service unavailable|HTTP 503|healthy_count.*0/i.test(lastCompletion)) break; + if (attempt < COMPLETION_ATTEMPTS) await sleep(5_000); + } + expect( + completionReason, + `Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure: ${lastCompletion.slice(0, 500)}`, + ).toBe("ok"); + + await artifacts.writeJson("scenario-result.json", { + id: "model-router-provider-routed-inference", + assertions: { + dockerRunning: docker.exitCode === 0, + onboardCompleted: onboard.exitCode === 0, + modelRouterHealthy: hasHealthyEndpoint(lastHealth), + routedPongCompletion: completionReason === "ok", + }, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index de38091ac63..c4cfe853e5c 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -184,6 +184,26 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["rebuild-openclaw-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ + scenarios: "model-router-provider-routed-inference", + }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["model-router-provider-routed-inference-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ + jobs: "model-router-provider-routed-inference-vitest", + }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["model-router-provider-routed-inference-vitest"], + registryScenarios: [], + }); }); it("keeps jobs-only dispatches from selecting the Hermes secret-bearing job", () => { @@ -249,6 +269,24 @@ describe("e2e-vitest-scenarios workflow boundary", () => { hermes_selected: "false", matrix: "[]", }); + expect( + generateMatrixForDispatch({ + JOBS: "model-router-provider-routed-inference-vitest", + SCENARIOS: "", + }), + ).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); + expect( + generateMatrixForDispatch({ + JOBS: "", + SCENARIOS: "model-router-provider-routed-inference", + }), + ).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); }); it("flags direct dispatch-input interpolation and unsafe artifact upload", () => { diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 8249918bd0c..0f259f6d11b 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -30,6 +30,7 @@ const FREE_STANDING_SCENARIO_JOBS = new Map([ ["token-rotation", "token-rotation-vitest"], ["openclaw-tui-chat-correlation", "openclaw-tui-chat-correlation-vitest"], ["issue-4434-tui-unreachable-inference", "issue-4434-tui-unreachable-inference-vitest"], + ["model-router-provider-routed-inference", "model-router-provider-routed-inference-vitest"], ]); const ALLOWED_FREE_STANDING_JOBS = new Set([ ...FREE_STANDING_SCENARIO_JOBS.values(), @@ -292,6 +293,7 @@ function validateJobsSelector(errors: string[], jobs: WorkflowRecord): void { requireRunContains(errors, validate, "token-rotation-vitest"); requireRunContains(errors, validate, "openclaw-tui-chat-correlation-vitest"); requireRunContains(errors, validate, "gateway-guard-recovery"); + requireRunContains(errors, validate, "model-router-provider-routed-inference-vitest"); requireRunContains(errors, validate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$"); requireRunContains(errors, validate, "Invalid jobs input; use comma-separated job ids"); requireRunDoesNotContain(errors, validate, "Invalid jobs input: ${JOBS}"); @@ -1277,6 +1279,188 @@ function validateHermesRootEntrypointSmokeVitestJob( } } +function validateModelRouterProviderRoutedInferenceVitestJob( + errors: string[], + jobs: WorkflowRecord, +): void { + const jobName = "model-router-provider-routed-inference-vitest"; + const scenarioName = "model-router-provider-routed-inference"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing model-router-provider-routed-inference-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("model-router-provider-routed-inference-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + + const jobEnv = asRecord(job.env); + if ( + jobEnv.DOCKER_CONFIG !== + "${{ runner.temp }}/docker-config-model-router-provider-routed-inference" + ) { + errors.push( + "model-router-provider-routed-inference-vitest job must isolate Docker auth with DOCKER_CONFIG under runner.temp", + ); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/model-router-provider-routed-inference" + ) { + errors.push( + "model-router-provider-routed-inference-vitest job must write artifacts under e2e-artifacts/vitest/model-router-provider-routed-inference", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push( + "model-router-provider-routed-inference-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI", + ); + } + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push( + "model-router-provider-routed-inference-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1", + ); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push( + "model-router-provider-routed-inference-vitest job must force OPENSHELL_GATEWAY=nemoclaw", + ); + } + for (const secret of ["NVIDIA_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + requireEnvDoesNotExposeSecret( + errors, + "model-router-provider-routed-inference-vitest job", + jobEnv, + secret, + ); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `model-router-provider-routed-inference-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run Model Router provider-routed inference live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) { + errors.push("model-router-provider-routed-inference-vitest job missing checkout step"); + } + requireFullShaAction(errors, checkout, "model-router-provider-routed-inference-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push( + "model-router-provider-routed-inference-vitest checkout step must set persist-credentials=false", + ); + } + + const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerLoginEnv = asRecord(dockerLogin?.env); + if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "model-router-provider-routed-inference-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "model-router-provider-routed-inference-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerLogin, 'mkdir -p "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, 'chmod 700 "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, "docker login docker.io"); + requireRunContains(errors, dockerLogin, "--password-stdin"); + requireRunContains(errors, dockerLogin, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) { + errors.push("model-router-provider-routed-inference-vitest job missing step: Set up Node"); + } + requireFullShaAction(errors, setupNode, "model-router-provider-routed-inference-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const runVitest = requireJobStep( + errors, + jobName, + steps, + "Run Model Router provider-routed inference live test", + ); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push( + "model-router-provider-routed-inference-vitest Vitest step must receive NVIDIA_API_KEY from secrets", + ); + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains( + errors, + runVitest, + "test/e2e-scenario/live/model-router-provider-routed-inference.test.ts", + ); + + const upload = requireJobStep( + errors, + jobName, + steps, + "Upload Model Router provider-routed inference artifacts", + ); + requireFullShaAction( + errors, + upload, + "model-router-provider-routed-inference-vitest upload-artifact", + ); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-model-router-provider-routed-inference") { + errors.push("model-router-provider-routed-inference-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains( + errors, + uploadPath, + "e2e-artifacts/vitest/model-router-provider-routed-inference/", + ); + if (uploadWith["include-hidden-files"] !== false) { + errors.push( + "model-router-provider-routed-inference-vitest artifact upload must set include-hidden-files: false", + ); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push( + "model-router-provider-routed-inference-vitest artifact upload must ignore missing fixture artifacts", + ); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("model-router-provider-routed-inference-vitest artifact upload retention-days must be 14"); + } + + const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); + if (cleanup?.if !== "always()") { + errors.push("model-router-provider-routed-inference-vitest Docker auth cleanup must always run"); + } + requireRunContains(errors, cleanup, "docker logout docker.io"); + requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); +} + export function validateE2eVitestScenariosWorkflowBoundary( workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, ): string[] { @@ -1345,6 +1529,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( requireRunContains(errors, generate, "network-policy-vitest"); requireRunContains(errors, generate, "rebuild-openclaw-vitest"); requireRunContains(errors, generate, "token-rotation-vitest"); + requireRunContains(errors, generate, "model-router-provider-routed-inference-vitest"); + requireRunContains(errors, generate, "model-router-provider-routed-inference"); requireRunContains(errors, generate, 'matrix="[]"'); requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts"); requireRunContains(errors, generate, "--emit-live-matrix"); @@ -1519,6 +1705,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "issue-4434-tui-unreachable-inference-vitest", "issue-4434-tui-unreachable-inference", ); + validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]); if (Object.keys(reportToPr).length === 0) { @@ -1543,6 +1730,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "openclaw-tui-chat-correlation-vitest", "gateway-guard-recovery", "issue-4434-tui-unreachable-inference-vitest", + "model-router-provider-routed-inference-vitest", ]) { if (!needs.includes(required)) errors.push(`report-to-pr job must wait for ${required}`); }