ci(e2e): split recovery validation from soak

.github/workflows/e2e.yaml

@@ -6,7 +6,8 @@ run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.p
 
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "0 0 * * 1-6"
+    - cron: "0 0 * * 0"
   workflow_dispatch:
     inputs:
       targets:
@@ -15,7 +16,7 @@ on:
         default: ""
         type: string
       jobs:
-        description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, and staging-brev-launchable are skipped unless selected."
+        description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, issue-2478-crash-loop-recovery-soak, and staging-brev-launchable are skipped unless selected."
         required: false
         default: ""
         type: string
@@ -5581,7 +5582,7 @@ jobs:
 
   issue-2478-crash-loop-recovery:
     needs: generate-matrix
-    if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery,') }}
+    if: ${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 1-6') || (github.event_name == 'workflow_dispatch' && ((inputs.jobs == '' && inputs.targets == '') || contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery,'))) }}
     runs-on: ubuntu-latest
     timeout-minutes: 30
     env:
@@ -5593,6 +5594,71 @@ jobs:
       NEMOCLAW_NON_INTERACTIVE: "1"
       NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
       NEMOCLAW_SANDBOX_NAME: "e2e-2478"
+      NEMOCLAW_E2E_RECOVERY_PROFILE: "functional"
+      NEMOCLAW_E2E_CRASH_CYCLES: "1"
+      NEMOCLAW_E2E_SOAK_SECONDS: "15"
+      OPENSHELL_GATEWAY: "nemoclaw"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ inputs.checkout_repository || github.repository }}
+          ref: ${{ inputs.checkout_sha || github.sha }}
+          persist-credentials: false
+
+      - *dockerhub-auth
+
+      - name: Prepare E2E workspace
+        uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
+
+      - name: Install OpenShell CLI
+        run: bash scripts/install-openshell.sh
+
+      - name: "Run issue #2478 crash-loop recovery live Vitest test"
+        run: |
+          set -euo pipefail
+          export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
+          if command -v openshell >/dev/null 2>&1; then
+            OPENSHELL_BIN="$(command -v openshell)"
+          elif [ -x "$HOME/.local/bin/openshell" ]; then
+            OPENSHELL_BIN="$HOME/.local/bin/openshell"
+          else
+            echo "::error::OpenShell CLI not found after install"
+            ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true
+            exit 1
+          fi
+          export OPENSHELL_BIN
+          echo "Using OPENSHELL_BIN=$OPENSHELL_BIN"
+          "$OPENSHELL_BIN" --version
+          npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/issue-2478-crash-loop-recovery.test.ts
+
+      - name: "Upload issue #2478 crash-loop recovery artifacts"
+        if: always()
+        uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
+
+      - name: Clean up Docker auth
+        if: always()
+        shell: bash
+        run: bash .github/scripts/docker-auth-cleanup.sh
+
+  issue-2478-crash-loop-recovery-soak:
+    needs: generate-matrix
+    if: ${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 0') || (github.event_name == 'workflow_dispatch' && (contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery-soak,') || contains(format(',{0},', inputs.targets), ',issue-2478-crash-loop-recovery-soak,'))) }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 35
+    env:
+      E2E_JOB: "1"
+      E2E_TARGET_ID: "issue-2478-crash-loop-recovery-soak"
+      E2E_DEFAULT_ENABLED: "0"
+      E2E_CHANGE_FOCUSED: "0"
+      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/issue-2478-crash-loop-recovery-soak
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
+      NEMOCLAW_RUN_LIVE_E2E: "1"
+      NEMOCLAW_NON_INTERACTIVE: "1"
+      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+      NEMOCLAW_SANDBOX_NAME: "e2e-2478-soak"
+      NEMOCLAW_E2E_RECOVERY_PROFILE: "soak"
+      NEMOCLAW_E2E_CRASH_CYCLES: "5"
+      NEMOCLAW_E2E_SOAK_SECONDS: "300"
       OPENSHELL_GATEWAY: "nemoclaw"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -5907,6 +5973,7 @@ jobs:
         openclaw-inference-switch,
         bedrock-runtime-compatible-anthropic,
         issue-2478-crash-loop-recovery,
+        issue-2478-crash-loop-recovery-soak,
         device-auth-health,
         channels-add-remove,
         tunnel-lifecycle,

test/e2e/live/issue-2478-crash-loop-recovery.test.ts

@@ -20,11 +20,12 @@ import type { HostCliClient } from "../fixtures/clients/index.ts";
 import { expect, test } from "../fixtures/e2e-test.ts";
 import type { NemoClawInstance } from "../fixtures/phases/onboarding.ts";
 import { ubuntuRepoDocker } from "../registry/matrix.ts";
+import { resolveIssue2478RecoverySettings } from "./issue-2478-recovery-profile.ts";
 
 const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw");
 const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-2478";
-const CRASH_CYCLES = positiveInteger(process.env.NEMOCLAW_E2E_CRASH_CYCLES, 5);
-const SOAK_SECONDS = positiveInteger(process.env.NEMOCLAW_E2E_SOAK_SECONDS, 300);
+const RECOVERY_SETTINGS = resolveIssue2478RecoverySettings(process.env);
+const TARGET_ID = process.env.E2E_TARGET_ID ?? "issue-2478-crash-loop-recovery";
 const COMPATIBLE_MODEL = process.env.NEMOCLAW_COMPAT_MODEL ?? "test-model";
 const COMPATIBLE_AUTH_VALUE = ["nemoclaw", "e2e", "compatible", "mock"].join("-");
 const PROXY_ENV_PATH = "/tmp/nemoclaw-proxy-env.sh";
@@ -36,11 +37,6 @@ const ONBOARD_ARGS = [
   "--yes-i-accept-third-party-software",
 ];
 
-function positiveInteger(raw: string | undefined, fallback: number): number {
-  const parsed = raw ? Number(raw) : fallback;
-  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
-}
-
 function probeEnv(): NodeJS.ProcessEnv {
   return {
     ...buildAvailabilityProbeEnv(),
@@ -440,10 +436,11 @@ test("gateway recovery preserves guard chain and avoids crash loop (#2478)", {
   },
 }, async ({ artifacts, cleanup, environment, gateway, host, progress, runtime, sandbox }) => {
   await artifacts.target.declare({
-    id: "issue-2478-crash-loop-recovery",
+    id: TARGET_ID,
     issues: ["#2478", "#2701"],
-    crashCycles: CRASH_CYCLES,
-    soakSeconds: SOAK_SECONDS,
+    recoveryProfile: RECOVERY_SETTINGS.profile,
+    crashCycles: RECOVERY_SETTINGS.crashCycles,
+    soakSeconds: RECOVERY_SETTINGS.soakSeconds,
     compatibleEndpointModel: COMPATIBLE_MODEL,
   });
 
@@ -479,7 +476,7 @@ test("gateway recovery preserves guard chain and avoids crash loop (#2478)", {
 
   progress.phase("exercise repeated gateway crash recovery");
   let previousPid = initialPid!;
-  for (let cycle = 1; cycle <= CRASH_CYCLES; cycle += 1) {
+  for (let cycle = 1; cycle <= RECOVERY_SETTINGS.crashCycles; cycle += 1) {
     await killGatewayPid(sandbox, instance.sandboxName, previousPid, `cycle-${cycle}-kill-gateway`);
     await runProbeOnly(host, instance.sandboxName, `cycle-${cycle}-connect-probe-only`);
     const nextPid = await waitForGatewayPid(gateway, instance, 45_000);
@@ -523,7 +520,12 @@ test("gateway recovery preserves guard chain and avoids crash loop (#2478)", {
   });
 
   progress.phase("measure gateway and inference stability");
-  const soak = await sampleGatewayStability(gateway, runtime, instance, SOAK_SECONDS);
+  const soak = await sampleGatewayStability(
+    gateway,
+    runtime,
+    instance,
+    RECOVERY_SETTINGS.soakSeconds,
+  );
   await artifacts.writeJson("soak-summary.json", soak);
   const distinctPids = new Set(soak.samples.filter((pid): pid is number => pid !== null));
   const emptySamples = soak.samples.filter((pid) => pid === null).length;

test/e2e/live/issue-2478-recovery-profile.ts

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+export type Issue2478RecoveryProfile = "functional" | "soak";
+
+export interface Issue2478RecoverySettings {
+  profile: Issue2478RecoveryProfile;
+  crashCycles: number;
+  soakSeconds: number;
+}
+
+const PROFILE_DEFAULTS: Record<
+  Issue2478RecoveryProfile,
+  Pick<Issue2478RecoverySettings, "crashCycles" | "soakSeconds">
+> = {
+  functional: {
+    crashCycles: 1,
+    soakSeconds: 15,
+  },
+  soak: {
+    crashCycles: 5,
+    soakSeconds: 300,
+  },
+};
+
+function positiveInteger(name: string, raw: string | undefined, fallback: number): number {
+  if (raw === undefined) return fallback;
+
+  const parsed = Number(raw);
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    throw new Error(`${name} must be a positive integer, got '${raw}'`);
+  }
+  return parsed;
+}
+
+export function resolveIssue2478RecoverySettings(
+  env: NodeJS.ProcessEnv,
+): Issue2478RecoverySettings {
+  const rawProfile = env.NEMOCLAW_E2E_RECOVERY_PROFILE ?? "soak";
+  if (rawProfile !== "functional" && rawProfile !== "soak") {
+    throw new Error(
+      `NEMOCLAW_E2E_RECOVERY_PROFILE must be 'functional' or 'soak', got '${rawProfile}'`,
+    );
+  }
+
+  const defaults = PROFILE_DEFAULTS[rawProfile];
+  return {
+    profile: rawProfile,
+    crashCycles: positiveInteger(
+      "NEMOCLAW_E2E_CRASH_CYCLES",
+      env.NEMOCLAW_E2E_CRASH_CYCLES,
+      defaults.crashCycles,
+    ),
+    soakSeconds: positiveInteger(
+      "NEMOCLAW_E2E_SOAK_SECONDS",
+      env.NEMOCLAW_E2E_SOAK_SECONDS,
+      defaults.soakSeconds,
+    ),
+  };
+}

test/e2e/support/issue-2478-recovery-profile.test.ts

@@ -0,0 +1,73 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { resolveIssue2478RecoverySettings } from "../live/issue-2478-recovery-profile.ts";
+
+describe("gateway crash-loop recovery profiles", () => {
+  it("keeps the functional profile to one crash cycle and one stability sample (#7919)", () => {
+    expect(
+      resolveIssue2478RecoverySettings({
+        NEMOCLAW_E2E_RECOVERY_PROFILE: "functional",
+      }),
+    ).toEqual({
+      profile: "functional",
+      crashCycles: 1,
+      soakSeconds: 15,
+    });
+  });
+
+  it("retains the full repeated-cycle soak contract by default (#7919)", () => {
+    expect(resolveIssue2478RecoverySettings({})).toEqual({
+      profile: "soak",
+      crashCycles: 5,
+      soakSeconds: 300,
+    });
+  });
+
+  it("keeps the Sunday soak as a strict superset of the functional profile (#7919)", () => {
+    const functional = resolveIssue2478RecoverySettings({
+      NEMOCLAW_E2E_RECOVERY_PROFILE: "functional",
+    });
+    const soak = resolveIssue2478RecoverySettings({
+      NEMOCLAW_E2E_RECOVERY_PROFILE: "soak",
+    });
+
+    expect(soak.crashCycles).toBeGreaterThan(functional.crashCycles);
+    expect(soak.soakSeconds).toBeGreaterThan(functional.soakSeconds);
+  });
+
+  it("allows positive explicit cycle and soak overrides for diagnostics (#7919)", () => {
+    expect(
+      resolveIssue2478RecoverySettings({
+        NEMOCLAW_E2E_RECOVERY_PROFILE: "soak",
+        NEMOCLAW_E2E_CRASH_CYCLES: "2",
+        NEMOCLAW_E2E_SOAK_SECONDS: "45",
+      }),
+    ).toEqual({
+      profile: "soak",
+      crashCycles: 2,
+      soakSeconds: 45,
+    });
+  });
+
+  it("rejects unknown profiles instead of silently weakening the soak (#7919)", () => {
+    expect(() =>
+      resolveIssue2478RecoverySettings({
+        NEMOCLAW_E2E_RECOVERY_PROFILE: "quick",
+      }),
+    ).toThrow(/must be 'functional' or 'soak'/);
+  });
+
+  it.each([
+    ["NEMOCLAW_E2E_CRASH_CYCLES", "0"],
+    ["NEMOCLAW_E2E_SOAK_SECONDS", "abc"],
+  ] as const)("rejects malformed %s overrides instead of silently using defaults (#7919)", (name, value) => {
+    expect(() =>
+      resolveIssue2478RecoverySettings({
+        [name]: value,
+      }),
+    ).toThrow(`${name} must be a positive integer`);
+  });
+});

test/e2e/support/issue-2478-recovery-workflow-boundary.test.ts

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts";
+import { readWorkflow } from "../../helpers/e2e-workflow-contract";
+
+describe("recovery workflow scheduling (#7919)", () => {
+  it("rejects recovery schedule, profile, and change-focused soak drift (#7919)", () => {
+    const workflow = readWorkflow() as {
+      on: { schedule: Array<{ cron: string }> };
+      jobs: Record<string, { env: Record<string, string>; if: string }>;
+    };
+    workflow.on.schedule = [{ cron: "0 0 * * *" }];
+    workflow.jobs["issue-2478-crash-loop-recovery"]!.env.NEMOCLAW_E2E_CRASH_CYCLES = "5";
+    workflow.jobs["issue-2478-crash-loop-recovery-soak"]!.env.E2E_CHANGE_FOCUSED = "1";
+    workflow.jobs["issue-2478-crash-loop-recovery-soak"]!.if =
+      "${{ github.event_name != 'workflow_dispatch' }}";
+
+    expect(validateE2eWorkflow(workflow)).toEqual(
+      expect.arrayContaining([
+        "workflow schedule must include 0 0 * * 1-6",
+        "workflow schedule must include 0 0 * * 0",
+        "workflow schedule must separate Monday through Saturday functional runs from the Sunday soak",
+        "issue-2478-crash-loop-recovery job env NEMOCLAW_E2E_CRASH_CYCLES must be 1",
+        'issue-2478-crash-loop-recovery-soak job E2E_CHANGE_FOCUSED must be "0" when set',
+        "issue-2478-crash-loop-recovery-soak job must keep its soak schedule and selector",
+      ]),
+    );
+  });
+});

test/release-e2e-evidence.test.ts

@@ -94,12 +94,13 @@ describe("release E2E evidence", () => {
       targets: "",
     });
     const parallelExplicitJobs = plan.dispatches.parallelExplicit.jobs.split(",");
-    expect(parallelExplicitJobs).toHaveLength(4);
+    expect(parallelExplicitJobs).toHaveLength(5);
     expect(new Set(parallelExplicitJobs)).toEqual(
       new Set([
         "openshell-gateway-auth-contract",
         "mcp-bridge-dev",
         "hermes-gpu-startup",
+        "issue-2478-crash-loop-recovery-soak",
         "sandbox-rlimits-connect",
       ]),
     );