diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index dec04499970..458981e1570 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -6,7 +6,8 @@ run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.p on: schedule: - - cron: "0 0 * * *" + - cron: "0 0 * * 1-6" + - cron: "0 0 * * 0" workflow_dispatch: inputs: targets: @@ -15,7 +16,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, and staging-brev-launchable are skipped unless selected." + description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, issue-2478-crash-loop-recovery-soak, and staging-brev-launchable are skipped unless selected." required: false default: "" type: string @@ -5581,7 +5582,7 @@ jobs: issue-2478-crash-loop-recovery: needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery,') }} + if: ${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 1-6') || (github.event_name == 'workflow_dispatch' && ((inputs.jobs == '' && inputs.targets == '') || contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery,'))) }} runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -5593,6 +5594,71 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_SANDBOX_NAME: "e2e-2478" + NEMOCLAW_E2E_RECOVERY_PROFILE: "functional" + NEMOCLAW_E2E_CRASH_CYCLES: "1" + NEMOCLAW_E2E_SOAK_SECONDS: "15" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: "Run issue #2478 crash-loop recovery live Vitest test" + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + echo "Using OPENSHELL_BIN=$OPENSHELL_BIN" + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/issue-2478-crash-loop-recovery.test.ts + + - name: "Upload issue #2478 crash-loop recovery artifacts" + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + issue-2478-crash-loop-recovery-soak: + needs: generate-matrix + if: ${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 0') || (github.event_name == 'workflow_dispatch' && (contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery-soak,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery-soak,'))) }} + runs-on: ubuntu-latest + timeout-minutes: 35 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "issue-2478-crash-loop-recovery-soak" + E2E_DEFAULT_ENABLED: "0" + E2E_CHANGE_FOCUSED: "0" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/issue-2478-crash-loop-recovery-soak + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-2478-soak" + NEMOCLAW_E2E_RECOVERY_PROFILE: "soak" + NEMOCLAW_E2E_CRASH_CYCLES: "5" + NEMOCLAW_E2E_SOAK_SECONDS: "300" OPENSHELL_GATEWAY: "nemoclaw" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -5907,6 +5973,7 @@ jobs: openclaw-inference-switch, bedrock-runtime-compatible-anthropic, issue-2478-crash-loop-recovery, + issue-2478-crash-loop-recovery-soak, device-auth-health, channels-add-remove, tunnel-lifecycle, diff --git a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts index 3a7ac973fd2..58f1cefc291 100644 --- a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts +++ b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts @@ -20,11 +20,12 @@ import type { HostCliClient } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import type { NemoClawInstance } from "../fixtures/phases/onboarding.ts"; import { ubuntuRepoDocker } from "../registry/matrix.ts"; +import { resolveIssue2478RecoverySettings } from "./issue-2478-recovery-profile.ts"; const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw"); const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-2478"; -const CRASH_CYCLES = positiveInteger(process.env.NEMOCLAW_E2E_CRASH_CYCLES, 5); -const SOAK_SECONDS = positiveInteger(process.env.NEMOCLAW_E2E_SOAK_SECONDS, 300); +const RECOVERY_SETTINGS = resolveIssue2478RecoverySettings(process.env); +const TARGET_ID = process.env.E2E_TARGET_ID ?? "issue-2478-crash-loop-recovery"; const COMPATIBLE_MODEL = process.env.NEMOCLAW_COMPAT_MODEL ?? "test-model"; const COMPATIBLE_AUTH_VALUE = ["nemoclaw", "e2e", "compatible", "mock"].join("-"); const PROXY_ENV_PATH = "/tmp/nemoclaw-proxy-env.sh"; @@ -36,11 +37,6 @@ const ONBOARD_ARGS = [ "--yes-i-accept-third-party-software", ]; -function positiveInteger(raw: string | undefined, fallback: number): number { - const parsed = raw ? Number(raw) : fallback; - return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; -} - function probeEnv(): NodeJS.ProcessEnv { return { ...buildAvailabilityProbeEnv(), @@ -440,10 +436,11 @@ test("gateway recovery preserves guard chain and avoids crash loop (#2478)", { }, }, async ({ artifacts, cleanup, environment, gateway, host, progress, runtime, sandbox }) => { await artifacts.target.declare({ - id: "issue-2478-crash-loop-recovery", + id: TARGET_ID, issues: ["#2478", "#2701"], - crashCycles: CRASH_CYCLES, - soakSeconds: SOAK_SECONDS, + recoveryProfile: RECOVERY_SETTINGS.profile, + crashCycles: RECOVERY_SETTINGS.crashCycles, + soakSeconds: RECOVERY_SETTINGS.soakSeconds, compatibleEndpointModel: COMPATIBLE_MODEL, }); @@ -479,7 +476,7 @@ test("gateway recovery preserves guard chain and avoids crash loop (#2478)", { progress.phase("exercise repeated gateway crash recovery"); let previousPid = initialPid!; - for (let cycle = 1; cycle <= CRASH_CYCLES; cycle += 1) { + for (let cycle = 1; cycle <= RECOVERY_SETTINGS.crashCycles; cycle += 1) { await killGatewayPid(sandbox, instance.sandboxName, previousPid, `cycle-${cycle}-kill-gateway`); await runProbeOnly(host, instance.sandboxName, `cycle-${cycle}-connect-probe-only`); const nextPid = await waitForGatewayPid(gateway, instance, 45_000); @@ -523,7 +520,12 @@ test("gateway recovery preserves guard chain and avoids crash loop (#2478)", { }); progress.phase("measure gateway and inference stability"); - const soak = await sampleGatewayStability(gateway, runtime, instance, SOAK_SECONDS); + const soak = await sampleGatewayStability( + gateway, + runtime, + instance, + RECOVERY_SETTINGS.soakSeconds, + ); await artifacts.writeJson("soak-summary.json", soak); const distinctPids = new Set(soak.samples.filter((pid): pid is number => pid !== null)); const emptySamples = soak.samples.filter((pid) => pid === null).length; diff --git a/test/e2e/live/issue-2478-recovery-profile.ts b/test/e2e/live/issue-2478-recovery-profile.ts new file mode 100644 index 00000000000..14e39c2df8e --- /dev/null +++ b/test/e2e/live/issue-2478-recovery-profile.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type Issue2478RecoveryProfile = "functional" | "soak"; + +export interface Issue2478RecoverySettings { + profile: Issue2478RecoveryProfile; + crashCycles: number; + soakSeconds: number; +} + +const PROFILE_DEFAULTS: Record< + Issue2478RecoveryProfile, + Pick +> = { + functional: { + crashCycles: 1, + soakSeconds: 15, + }, + soak: { + crashCycles: 5, + soakSeconds: 300, + }, +}; + +function positiveInteger(name: string, raw: string | undefined, fallback: number): number { + if (raw === undefined) return fallback; + + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer, got '${raw}'`); + } + return parsed; +} + +export function resolveIssue2478RecoverySettings( + env: NodeJS.ProcessEnv, +): Issue2478RecoverySettings { + const rawProfile = env.NEMOCLAW_E2E_RECOVERY_PROFILE ?? "soak"; + if (rawProfile !== "functional" && rawProfile !== "soak") { + throw new Error( + `NEMOCLAW_E2E_RECOVERY_PROFILE must be 'functional' or 'soak', got '${rawProfile}'`, + ); + } + + const defaults = PROFILE_DEFAULTS[rawProfile]; + return { + profile: rawProfile, + crashCycles: positiveInteger( + "NEMOCLAW_E2E_CRASH_CYCLES", + env.NEMOCLAW_E2E_CRASH_CYCLES, + defaults.crashCycles, + ), + soakSeconds: positiveInteger( + "NEMOCLAW_E2E_SOAK_SECONDS", + env.NEMOCLAW_E2E_SOAK_SECONDS, + defaults.soakSeconds, + ), + }; +} diff --git a/test/e2e/support/issue-2478-recovery-profile.test.ts b/test/e2e/support/issue-2478-recovery-profile.test.ts new file mode 100644 index 00000000000..eb62a4b1cb4 --- /dev/null +++ b/test/e2e/support/issue-2478-recovery-profile.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { resolveIssue2478RecoverySettings } from "../live/issue-2478-recovery-profile.ts"; + +describe("gateway crash-loop recovery profiles", () => { + it("keeps the functional profile to one crash cycle and one stability sample (#7919)", () => { + expect( + resolveIssue2478RecoverySettings({ + NEMOCLAW_E2E_RECOVERY_PROFILE: "functional", + }), + ).toEqual({ + profile: "functional", + crashCycles: 1, + soakSeconds: 15, + }); + }); + + it("retains the full repeated-cycle soak contract by default (#7919)", () => { + expect(resolveIssue2478RecoverySettings({})).toEqual({ + profile: "soak", + crashCycles: 5, + soakSeconds: 300, + }); + }); + + it("keeps the Sunday soak as a strict superset of the functional profile (#7919)", () => { + const functional = resolveIssue2478RecoverySettings({ + NEMOCLAW_E2E_RECOVERY_PROFILE: "functional", + }); + const soak = resolveIssue2478RecoverySettings({ + NEMOCLAW_E2E_RECOVERY_PROFILE: "soak", + }); + + expect(soak.crashCycles).toBeGreaterThan(functional.crashCycles); + expect(soak.soakSeconds).toBeGreaterThan(functional.soakSeconds); + }); + + it("allows positive explicit cycle and soak overrides for diagnostics (#7919)", () => { + expect( + resolveIssue2478RecoverySettings({ + NEMOCLAW_E2E_RECOVERY_PROFILE: "soak", + NEMOCLAW_E2E_CRASH_CYCLES: "2", + NEMOCLAW_E2E_SOAK_SECONDS: "45", + }), + ).toEqual({ + profile: "soak", + crashCycles: 2, + soakSeconds: 45, + }); + }); + + it("rejects unknown profiles instead of silently weakening the soak (#7919)", () => { + expect(() => + resolveIssue2478RecoverySettings({ + NEMOCLAW_E2E_RECOVERY_PROFILE: "quick", + }), + ).toThrow(/must be 'functional' or 'soak'/); + }); + + it.each([ + ["NEMOCLAW_E2E_CRASH_CYCLES", "0"], + ["NEMOCLAW_E2E_SOAK_SECONDS", "abc"], + ] as const)("rejects malformed %s overrides instead of silently using defaults (#7919)", (name, value) => { + expect(() => + resolveIssue2478RecoverySettings({ + [name]: value, + }), + ).toThrow(`${name} must be a positive integer`); + }); +}); diff --git a/test/e2e/support/issue-2478-recovery-workflow-boundary.test.ts b/test/e2e/support/issue-2478-recovery-workflow-boundary.test.ts new file mode 100644 index 00000000000..61adc990bcd --- /dev/null +++ b/test/e2e/support/issue-2478-recovery-workflow-boundary.test.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +describe("recovery workflow scheduling (#7919)", () => { + it("rejects recovery schedule, profile, and change-focused soak drift (#7919)", () => { + const workflow = readWorkflow() as { + on: { schedule: Array<{ cron: string }> }; + jobs: Record; if: string }>; + }; + workflow.on.schedule = [{ cron: "0 0 * * *" }]; + workflow.jobs["issue-2478-crash-loop-recovery"]!.env.NEMOCLAW_E2E_CRASH_CYCLES = "5"; + workflow.jobs["issue-2478-crash-loop-recovery-soak"]!.env.E2E_CHANGE_FOCUSED = "1"; + workflow.jobs["issue-2478-crash-loop-recovery-soak"]!.if = + "${{ github.event_name != 'workflow_dispatch' }}"; + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "workflow schedule must include 0 0 * * 1-6", + "workflow schedule must include 0 0 * * 0", + "workflow schedule must separate Monday through Saturday functional runs from the Sunday soak", + "issue-2478-crash-loop-recovery job env NEMOCLAW_E2E_CRASH_CYCLES must be 1", + 'issue-2478-crash-loop-recovery-soak job E2E_CHANGE_FOCUSED must be "0" when set', + "issue-2478-crash-loop-recovery-soak job must keep its soak schedule and selector", + ]), + ); + }); +}); diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts index 6900363b5bd..1d8dcd0e69e 100644 --- a/test/release-e2e-evidence.test.ts +++ b/test/release-e2e-evidence.test.ts @@ -94,12 +94,13 @@ describe("release E2E evidence", () => { targets: "", }); const parallelExplicitJobs = plan.dispatches.parallelExplicit.jobs.split(","); - expect(parallelExplicitJobs).toHaveLength(4); + expect(parallelExplicitJobs).toHaveLength(5); expect(new Set(parallelExplicitJobs)).toEqual( new Set([ "openshell-gateway-auth-contract", "mcp-bridge-dev", "hermes-gpu-startup", + "issue-2478-crash-loop-recovery-soak", "sandbox-rlimits-connect", ]), ); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index e2035913b52..2cd010b1bff 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -135,6 +135,7 @@ const LIVE_TEST_FILE_PATTERN = /test\/e2e\/live\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0- const FREE_STANDING_JOB_MARKER = "E2E_JOB"; const FREE_STANDING_TARGET_MARKER = "E2E_TARGET_ID"; const FREE_STANDING_DEFAULT_ENABLED_MARKER = "E2E_DEFAULT_ENABLED"; +const FREE_STANDING_CHANGE_FOCUSED_MARKER = "E2E_CHANGE_FOCUSED"; const EXPLICIT_ONLY_JOBS_WITHOUT_ENV_MARKER = new Set(["hermes-gpu-startup"]); const COMMON_SECRET_ENV_NAMES = [ "NVIDIA_API_KEY", @@ -146,6 +147,8 @@ const COMMON_SECRET_ENV_NAMES = [ const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set([ "hermes-e2e", "hermes-gpu-startup", + "issue-2478-crash-loop-recovery", + "issue-2478-crash-loop-recovery-soak", "staging-brev-launchable", ]); const ADAPTER_MANAGED_INFERENCE_JOBS = new Set(["hermes-e2e"]); @@ -444,7 +447,13 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { allowedJobs.push(jobId); workflowJobs.push(jobId); - for (const file of collectLiveTestFiles(rawJob)) addMapValue(liveTestToJobs, file, jobId); + if (Object.hasOwn(env, FREE_STANDING_CHANGE_FOCUSED_MARKER)) { + if (env[FREE_STANDING_CHANGE_FOCUSED_MARKER] !== "0") { + errors.push(`${jobId} job ${FREE_STANDING_CHANGE_FOCUSED_MARKER} must be "0" when set`); + } + } else { + for (const file of collectLiveTestFiles(rawJob)) addMapValue(liveTestToJobs, file, jobId); + } if (Object.hasOwn(env, FREE_STANDING_DEFAULT_ENABLED_MARKER)) { if (env[FREE_STANDING_DEFAULT_ENABLED_MARKER] !== "0") { errors.push(`${jobId} job ${FREE_STANDING_DEFAULT_ENABLED_MARKER} must be "0" when set`); @@ -981,8 +990,15 @@ function requireScheduledRun(errors: string[], triggers: WorkflowRecord): void { const cronEntries = schedule .map((entry) => asRecord(entry).cron) .filter((cron): cron is string => typeof cron === "string"); - if (!cronEntries.includes("0 0 * * *")) { - errors.push("workflow schedule must run daily at 00:00 UTC"); + for (const cron of ["0 0 * * 1-6", "0 0 * * 0"]) { + if (!cronEntries.includes(cron)) { + errors.push(`workflow schedule must include ${cron}`); + } + } + if (cronEntries.includes("0 0 * * *")) { + errors.push( + "workflow schedule must separate Monday through Saturday functional runs from the Sunday soak", + ); } } @@ -3272,88 +3288,132 @@ function validateTunnelLifecycleJob(errors: string[], jobs: WorkflowRecord): voi } function validateIssue2478CrashLoopRecoveryJob(errors: string[], jobs: WorkflowRecord): void { - const jobName = "issue-2478-crash-loop-recovery"; - const targetName = "issue-2478-crash-loop-recovery"; - const job = asRecord(jobs[jobName]); - if (Object.keys(job).length === 0) { - errors.push("workflow missing issue-2478-crash-loop-recovery job"); - return; - } + const functionalSelector = + "${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 1-6') || (github.event_name == 'workflow_dispatch' && ((inputs.jobs == '' && inputs.targets == '') || contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery,'))) }}"; + const soakSelector = + "${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 0') || (github.event_name == 'workflow_dispatch' && (contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery-soak,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery-soak,'))) }}"; + const variants = [ + { + jobName: "issue-2478-crash-loop-recovery", + targetName: "issue-2478-crash-loop-recovery", + selector: functionalSelector, + timeoutMinutes: 30, + artifactName: "issue-2478-crash-loop-recovery", + sandboxName: "e2e-2478", + profile: "functional", + crashCycles: "1", + soakSeconds: "15", + explicitOnly: false, + }, + { + jobName: "issue-2478-crash-loop-recovery-soak", + targetName: "issue-2478-crash-loop-recovery-soak", + selector: soakSelector, + timeoutMinutes: 35, + artifactName: "issue-2478-crash-loop-recovery-soak", + sandboxName: "e2e-2478-soak", + profile: "soak", + crashCycles: "5", + soakSeconds: "300", + explicitOnly: true, + }, + ] as const; - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("issue-2478-crash-loop-recovery job must run on ubuntu-latest"); - } - if (job["timeout-minutes"] !== 30) { - errors.push("issue-2478-crash-loop-recovery job must keep the 30 minute timeout"); - } - validateFreeStandingJobSelector(errors, jobs, jobName, targetName); + for (const variant of variants) { + const job = asRecord(jobs[variant.jobName]); + if (Object.keys(job).length === 0) { + errors.push(`workflow missing ${variant.jobName} job`); + continue; + } - const jobEnv = asRecord(job.env); - if ("DOCKER_CONFIG" in jobEnv) { - errors.push("issue-2478-crash-loop-recovery job must not set DOCKER_CONFIG at job level"); - } - const expectedEnv: Record = { - E2E_JOB: "1", - E2E_TARGET_ID: targetName, - E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/issue-2478-crash-loop-recovery", - NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", - NEMOCLAW_RUN_LIVE_E2E: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_SANDBOX_NAME: "e2e-2478", - OPENSHELL_GATEWAY: "nemoclaw", - }; - for (const [key, value] of Object.entries(expectedEnv)) { - if (jobEnv[key] !== value) { - errors.push(`issue-2478-crash-loop-recovery job env ${key} must be ${value}`); + if (job["runs-on"] !== "ubuntu-latest") { + errors.push(`${variant.jobName} job must run on ubuntu-latest`); + } + if (job["timeout-minutes"] !== variant.timeoutMinutes) { + errors.push(`${variant.jobName} job must keep the ${variant.timeoutMinutes} minute timeout`); + } + if (job.if !== variant.selector) { + errors.push(`${variant.jobName} job must keep its ${variant.profile} schedule and selector`); } - } - for (const secret of [...COMMON_SECRET_ENV_NAMES]) { - requireEnvDoesNotExposeSecret(errors, "issue-2478-crash-loop-recovery job", jobEnv, secret); - } - const steps = asSteps(job.steps); - requireNoDispatchInputInterpolation(errors, steps); - for (const step of steps) { - const stepName = `issue-2478-crash-loop-recovery step '${step.name ?? step.uses ?? ""}'`; - const stepEnv = asRecord(step.env); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); - if (step.name !== "Authenticate to Docker Hub") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); - requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + const jobEnv = asRecord(job.env); + if ("DOCKER_CONFIG" in jobEnv) { + errors.push(`${variant.jobName} job must not set DOCKER_CONFIG at job level`); + } + const expectedEnv: Record = { + E2E_JOB: "1", + E2E_TARGET_ID: variant.targetName, + E2E_ARTIFACT_DIR: `\${{ github.workspace }}/e2e-artifacts/live/${variant.artifactName}`, + NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SANDBOX_NAME: variant.sandboxName, + NEMOCLAW_E2E_RECOVERY_PROFILE: variant.profile, + NEMOCLAW_E2E_CRASH_CYCLES: variant.crashCycles, + NEMOCLAW_E2E_SOAK_SECONDS: variant.soakSeconds, + OPENSHELL_GATEWAY: "nemoclaw", + }; + if (variant.explicitOnly) { + expectedEnv.E2E_DEFAULT_ENABLED = "0"; + expectedEnv.E2E_CHANGE_FOCUSED = "0"; + } + for (const [key, value] of Object.entries(expectedEnv)) { + if (jobEnv[key] !== value) { + errors.push(`${variant.jobName} job env ${key} must be ${value}`); + } + } + for (const secret of COMMON_SECRET_ENV_NAMES) { + requireEnvDoesNotExposeSecret(errors, `${variant.jobName} job`, jobEnv, secret); } - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); - } - const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); - if (!checkout) { - errors.push("issue-2478-crash-loop-recovery job missing checkout step"); - } - requireFullShaAction(errors, checkout, "issue-2478-crash-loop-recovery checkout"); - if (asRecord(checkout?.with)["persist-credentials"] !== false) { - errors.push("issue-2478-crash-loop-recovery checkout step must set persist-credentials=false"); - } + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `${variant.jobName} step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } - const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell CLI"); - requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) { + errors.push(`${variant.jobName} job missing checkout step`); + } + requireFullShaAction(errors, checkout, `${variant.jobName} checkout`); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push(`${variant.jobName} checkout step must set persist-credentials=false`); + } - const runVitest = requireJobStep( - errors, - jobName, - steps, - "Run issue #2478 crash-loop recovery live Vitest test", - ); - const runVitestEnv = asRecord(runVitest?.env); - requireEnvDoesNotExposeSecret( - errors, - "issue-2478-crash-loop-recovery live E2E step", - runVitestEnv, - "NVIDIA_INFERENCE_API_KEY", - ); - requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); - requireRunContains(errors, runVitest, "test/e2e/live/issue-2478-crash-loop-recovery.test.ts"); + const installOpenShell = requireJobStep( + errors, + variant.jobName, + steps, + "Install OpenShell CLI", + ); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + + const runVitest = requireJobStep( + errors, + variant.jobName, + steps, + "Run issue #2478 crash-loop recovery live Vitest test", + ); + const runVitestEnv = asRecord(runVitest?.env); + requireEnvDoesNotExposeSecret( + errors, + `${variant.jobName} live E2E step`, + runVitestEnv, + "NVIDIA_INFERENCE_API_KEY", + ); + requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); + requireRunContains(errors, runVitest, "test/e2e/live/issue-2478-crash-loop-recovery.test.ts"); + } } function validateChannelsAddRemoveJob(errors: string[], jobs: WorkflowRecord): void {