Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions .github/workflows/e2e-vitest-scenarios.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,81 @@ jobs:
if-no-files-found: ignore
retention-days: 14

model-router-provider-routed-inference-vitest:
if: ${{ inputs.scenarios == '' }}
runs-on: ubuntu-latest
timeout-minutes: 45
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/model-router-provider-routed-inference
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false

- name: Authenticate to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
shell: bash
run: |
set -euo pipefail
if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then
echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls."
exit 0
fi
login_succeeded=0
for attempt in 1 2 3; do
if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then
login_succeeded=1
break
fi
if [[ "$attempt" -lt 3 ]]; then
echo "::warning::Docker Hub login attempt ${attempt} failed; retrying."
sleep 5
fi
done
if [[ "$login_succeeded" -ne 1 ]]; then
echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls."
fi

- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
with:
node-version: 22
cache: npm

- name: Install root dependencies
run: npm ci --ignore-scripts

- name: Build CLI
run: npm run build:cli

- name: Run Model Router provider-routed inference live test
# Direct Vitest coverage for
# test/e2e/test-model-router-provider-routed-inference.sh. It preserves
# the real provider-routed onboard, host model-router health, and
# sandbox inference.local completion boundaries without adding registry
# or migration-ledger wiring.
env:
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
run: |
set -euo pipefail
npx vitest run --project e2e-scenarios-live \
test/e2e-scenario/live/model-router-provider-routed-inference.test.ts \
--silent=false --reporter=default

- name: Upload Model Router provider-routed inference artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-vitest-scenarios-model-router-provider-routed-inference
path: e2e-artifacts/vitest/model-router-provider-routed-inference/
include-hidden-files: false
if-no-files-found: ignore
retention-days: 14

# Focused coverage slice for the #2603/#3145 OpenClaw websocket
# protocol/history contract. The retained legacy bash lane remains the
# source for full closeout until a later PR proves replacement and deletes it.
Expand Down
235 changes: 235 additions & 0 deletions test/e2e-scenario/live/model-router-provider-routed-inference.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import path from "node:path";

import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";

// Focused Vitest live replacement for
// test/e2e/test-model-router-provider-routed-inference.sh. Keep this as a
// direct CLI/sandbox test: the legacy contract is the real provider-routed
// onboard boundary plus host model-router health and sandbox inference.local
// completion semantics, not a new scenario registry entry.

const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js");
const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-model-router";
const ONBOARD_TIMEOUT_MS = 25 * 60_000;
const HEALTH_ATTEMPTS = 20;
const COMPLETION_ATTEMPTS = 3;

interface ModelRouterHealth {
healthy_count?: unknown;
}

interface ChatCompletionResponse {
model?: unknown;
choices?: Array<{
message?: { content?: unknown };
text?: unknown;
}>;
}

function resultText(result: { stdout: string; stderr: string }): string {
return [result.stdout, result.stderr].filter(Boolean).join("\n");
}

function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

function parseJson<T>(raw: string): T | undefined {
try {
return JSON.parse(raw) as T;
} catch {
return undefined;
}
}

function hasHealthyEndpoint(raw: string): boolean {
const health = parseJson<ModelRouterHealth>(raw);
return typeof health?.healthy_count === "number" && health.healthy_count > 0;
}

function routedPongReason(raw: string): "ok" | string {
const response = parseJson<ChatCompletionResponse>(raw);
if (!response) return "response was not JSON";
const model = String(response.model ?? "");
if (model !== "nvidia-routed" && !model.startsWith("nvidia-routed")) {
return "response model was not provider-routed";
}
const content = (response.choices ?? [])
.map((choice) => {
if (typeof choice.message?.content === "string") return choice.message.content;
if (typeof choice.text === "string") return choice.text;
return "";
})
.join("\n");
if (!/\bPONG\b/i.test(content)) return "response missing PONG content";
return "ok";
}

function withProviderRoutedEnv(apiKey: string): NodeJS.ProcessEnv {
return {
...buildAvailabilityProbeEnv(),
NVIDIA_API_KEY: apiKey,
NEMOCLAW_PROVIDER_KEY: apiKey,
NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
NEMOCLAW_NON_INTERACTIVE: "1",
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
NEMOCLAW_POLICY_TIER: "open",
NEMOCLAW_PROVIDER: "routed",
};
}

test.skipIf(!shouldRunLiveE2EScenarios())(
"model-router provider-routed onboard returns routed inference.local PONG",
async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => {
expect(
fs.existsSync(CLI_ENTRYPOINT),
"run `npm run build:cli` before live repo CLI scenarios",
).toBe(true);

const docker = await host.command("docker", ["info"], {
artifactName: "prereq-docker-info-model-router-provider-routed",
env: buildAvailabilityProbeEnv(),
timeoutMs: 30_000,
});
if (docker.exitCode !== 0) {
if (process.env.GITHUB_ACTIONS === "true") {
throw new Error(
`Docker is required for provider-routed Model Router onboarding: ${resultText(docker)}`,
);
}
skip("Docker is required for provider-routed Model Router onboarding");
}

const apiKey = secrets.required("NVIDIA_API_KEY");
expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true);

await artifacts.writeJson("scenario.json", {
id: "model-router-provider-routed-inference",
runner: "vitest",
boundary: "direct-cli-onboard-and-sandbox-exec",
legacySource: "test/e2e/test-model-router-provider-routed-inference.sh",
contract: [
"Docker is available before onboarding",
"NVIDIA_API_KEY is present and nvapi-prefixed",
"nemoclaw onboard --fresh completes with NEMOCLAW_PROVIDER=routed",
"host model-router health reports at least one healthy endpoint",
"sandbox inference.local returns model nvidia-routed with PONG content",
],
});

const cleanEnv = buildAvailabilityProbeEnv();
await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], {
artifactName: "pre-cleanup-nemoclaw-destroy-model-router-provider-routed",
env: cleanEnv,
timeoutMs: 120_000,
});

cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => {
await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], {
artifactName: "cleanup-nemoclaw-destroy-model-router-provider-routed",
env: buildAvailabilityProbeEnv(),
timeoutMs: 120_000,
});
});

const onboard = await host.command(
"node",
[
CLI_ENTRYPOINT,
"onboard",
"--fresh",
"--non-interactive",
"--yes-i-accept-third-party-software",
],
{
artifactName: "onboard-model-router-provider-routed",
env: withProviderRoutedEnv(apiKey),
redactionValues: [apiKey],
timeoutMs: ONBOARD_TIMEOUT_MS,
},
);
expect(onboard.exitCode, resultText(onboard)).toBe(0);

let lastHealth = "";
for (let attempt = 1; attempt <= HEALTH_ATTEMPTS; attempt += 1) {
const health = await host.command(
"curl",
["-s", "--max-time", "10", "http://127.0.0.1:4000/health"],
{
artifactName: `model-router-health-${attempt}`,
env: buildAvailabilityProbeEnv(),
redactionValues: [apiKey],
timeoutMs: 15_000,
},
);
lastHealth = health.stdout || health.stderr;
if (health.exitCode === 0 && hasHealthyEndpoint(lastHealth)) break;
if (attempt < HEALTH_ATTEMPTS) await sleep(3_000);
}
expect(
hasHealthyEndpoint(lastHealth),
`model-router has no healthy endpoints; expected #3255 main-equivalent failure: ${lastHealth.slice(0, 500)}`,
).toBe(true);

const payload = JSON.stringify({
model: "nvidia-routed",
messages: [
{
role: "user",
content: "Return only the exact word PONG. Do not include reasoning or any other text.",
},
],
max_tokens: 128,
});
let lastCompletion = "";
let completionReason = "not attempted";
for (let attempt = 1; attempt <= COMPLETION_ATTEMPTS; attempt += 1) {
const completion = await sandbox.exec(
SANDBOX_NAME,
[
"curl",
"-sk",
"--max-time",
"90",
"https://inference.local/v1/chat/completions",
"-H",
"Content-Type: application/json",
"--data-raw",
payload,
],
{
artifactName: `sandbox-inference-local-routed-completion-${attempt}`,
env: buildAvailabilityProbeEnv(),
redactionValues: [apiKey],
timeoutMs: 120_000,
},
);
lastCompletion = completion.stdout || completion.stderr;
completionReason = routedPongReason(lastCompletion);
if (completion.exitCode === 0 && completionReason === "ok") break;
if (/inference service unavailable|HTTP 503|healthy_count.*0/i.test(lastCompletion)) break;
if (attempt < COMPLETION_ATTEMPTS) await sleep(5_000);
}
expect(
completionReason,
`Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure: ${lastCompletion.slice(0, 500)}`,
).toBe("ok");

await artifacts.writeJson("scenario-result.json", {
id: "model-router-provider-routed-inference",
assertions: {
dockerRunning: docker.exitCode === 0,
onboardCompleted: onboard.exitCode === 0,
modelRouterHealthy: hasHealthyEndpoint(lastHealth),
routedPongCompletion: completionReason === "ok",
},
});
},
);
Loading