diff --git a/.github/workflows/staging-launchable-full.yaml b/.github/workflows/staging-launchable-full.yaml index 13a4ad8d418..78cbc9b4ae9 100644 --- a/.github/workflows/staging-launchable-full.yaml +++ b/.github/workflows/staging-launchable-full.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: E2E / Staging Launchable Control Plane +name: E2E / Staging Launchable Remote Execution on: workflow_dispatch: @@ -17,7 +17,7 @@ jobs: test: if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' }} runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 175 steps: - name: Check out trusted Vitest controller timeout-minutes: 10 @@ -88,8 +88,8 @@ jobs: brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" printf 'work_dir=%s\n' "$work_dir" >>"$GITHUB_OUTPUT" - - name: Verify staging Launchable control-plane checkpoint - timeout-minutes: 60 + - name: Verify staging Launchable remote execution readiness + timeout-minutes: 75 env: BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} BREV_WORKSPACE_OWNERSHIP_FILE: ${{ steps.prepare.outputs.work_dir }}/workspace-owner.json @@ -127,5 +127,5 @@ jobs: timeout-minutes: 10 uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: - name: staging-launchable-control-plane-${{ github.run_id }}-${{ github.run_attempt }} + name: staging-launchable-remote-execution-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ steps.prepare.outputs.work_dir }}/ diff --git a/test/e2e/RETRY_INVENTORY.md b/test/e2e/RETRY_INVENTORY.md index 16a269c5c9b..90d191a4c8b 100644 --- a/test/e2e/RETRY_INVENTORY.md +++ b/test/e2e/RETRY_INVENTORY.md @@ -7,3 +7,4 @@ | --- | --- | --- | --- | --- | --- | | `external-gateway-health.tcp-readiness` | `openshell-gateway` | 10 attempts, one second apart | The newly started gateway listener rejects a TCP connection with `ECONNREFUSED` | The probe is read-only. Other errors stop without retry. The Blueprint Runner health operation runs once after the listener opens. | `external-gateway-readiness-retry.json` | | `mcp-bridge.tool-discovery` | `test/e2e/live/mcp-bridge-tool-discovery.ts` | 2 attempts, one second apart | Discovery reports `failureClass: connection` before any request reaches the fixture | The operation lists tools without calling them. Any fixture request or other failure class stops the retry. | First-attempt and `retry-2` command artifacts plus `*-mcp-tool-discovery-diagnostics.json` | +| `staging-launchable.remote-execution-readiness` | `test/e2e/fixtures/brev-launchable.ts` | 15 minutes, 15 seconds between attempts, 30 seconds per attempt | `brev exec true` returns a nonzero exit, signal, or timeout while the newly created workspace becomes remotely reachable | The probe is read-only and idempotent. Each attempt first verifies the persisted owned ID still matches the exact-name workspace inventory. | `brev-exec-readiness-failure.json` records the attempt count and final size-bounded, redacted diagnostic on timeout. | diff --git a/test/e2e/live/issue-9880-staging-launchable.test.ts b/test/e2e/live/issue-9880-staging-launchable.test.ts index e9a6b1e8505..3174e93bbd0 100644 --- a/test/e2e/live/issue-9880-staging-launchable.test.ts +++ b/test/e2e/live/issue-9880-staging-launchable.test.ts @@ -7,18 +7,19 @@ import { ISSUE_9880_STAGING_LAUNCHABLE_CLEANUP_TIMEOUT_MS } from "../../../tools import { BrevLaunchableFixture } from "../fixtures/brev-launchable.ts"; import { test } from "../fixtures/e2e-test.ts"; -const CONTROL_PLANE_TEST_TIMEOUT_MS = 45 * 60_000; +const REMOTE_EXECUTION_READINESS_TEST_TIMEOUT_MS = 60 * 60_000; test( - "creates the staging Launchable workspace and records its control-plane identity", + "creates the staging Launchable workspace and proves remote execution readiness", { - timeout: CONTROL_PLANE_TEST_TIMEOUT_MS, + timeout: REMOTE_EXECUTION_READINESS_TEST_TIMEOUT_MS, meta: { e2eCleanupTimeoutMs: ISSUE_9880_STAGING_LAUNCHABLE_CLEANUP_TIMEOUT_MS, e2ePhases: [ "resolve the latest staging handoff", "create the staging workspace", - "record the control-plane checkpoint", + "prove remote execution readiness", + "record remote execution readiness", ], }, }, @@ -33,19 +34,23 @@ test( cleanup.add(`delete Brev workspace ${name}`, () => brevLaunchable.delete(ownership)); const workspace = await brevLaunchable.create(ownership, launchableId); - progress.phase("record the control-plane checkpoint"); - await artifacts.writeJson("staging-launchable-control-plane-checkpoint.json", { + progress.phase("prove remote execution readiness"); + await brevLaunchable.waitForExec(ownership); + + progress.phase("record remote execution readiness"); + await artifacts.writeJson("staging-launchable-remote-execution-readiness.json", { bootImage: handoff.bootImage, candidateSha: handoff.nemoclawSha, imageRepositorySha: handoff.imageRepositorySha, launchableId, producerRunId: handoff.producerRunId, + remoteExecutionReady: true, workspaceId: workspace.id, workspaceName: workspace.name, }); await artifacts.target.complete({ id: "staging-launchable-full", - classification: "control-plane-checkpoint-passed", + classification: "remote-execution-ready", }); }, ); diff --git a/test/e2e/support/brev-launchable-fixture.test.ts b/test/e2e/support/brev-launchable-fixture.test.ts index 0ba795ad06a..784916b3ce8 100644 --- a/test/e2e/support/brev-launchable-fixture.test.ts +++ b/test/e2e/support/brev-launchable-fixture.test.ts @@ -300,6 +300,35 @@ describe("the Brev Launchable fixture binds staging identity and workspace lifec expect(ownership.id).toBe("owned-id"); }); + it("proves Brev exec readiness after bounded failed probes", async () => { + vi.useFakeTimers(); + const root = temporaryRoot(); + let attempts = 0; + const command = vi.fn(async (_binary: string, args: string[]) => { + switch (args[0]) { + case "ls": + return workspaceResult("owned-id"); + case "exec": + attempts += 1; + return { ...result(""), exitCode: attempts < 3 ? 1 : 0 }; + default: + throw new Error(`unexpected command: ${args.join(" ")}`); + } + }); + const fixture = createFixture(root, command); + + const readiness = fixture.waitForExec(recordedOwnership(), 100); + await vi.runAllTimersAsync(); + await expect(readiness).resolves.toBeUndefined(); + + expect(command.mock.calls.filter((call) => call[1][0] === "exec")).toEqual([ + ["brev", ["exec", "owned-id", "true"], expect.any(Object)], + ["brev", ["exec", "owned-id", "true"], expect.any(Object)], + ["brev", ["exec", "owned-id", "true"], expect.any(Object)], + ]); + expect(fs.existsSync(path.join(root, "brev-exec-readiness-failure.json"))).toBe(false); + }); + it("records the last failed Brev exec readiness attempt", async () => { vi.useFakeTimers(); const root = temporaryRoot();