Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/staging-launchable-full.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: E2E / Staging Launchable Control Plane
name: E2E / Staging Launchable Remote Execution

on:
workflow_dispatch:
Expand All @@ -17,7 +17,7 @@ jobs:
test:
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' }}
runs-on: ubuntu-latest
timeout-minutes: 90
timeout-minutes: 160
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
steps:
- name: Check out trusted Vitest controller
timeout-minutes: 10
Expand Down Expand Up @@ -88,8 +88,8 @@ jobs:
brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID"
printf 'work_dir=%s\n' "$work_dir" >>"$GITHUB_OUTPUT"

- name: Verify staging Launchable control-plane checkpoint
timeout-minutes: 60
- name: Verify staging Launchable remote execution readiness
timeout-minutes: 75
env:
BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}
BREV_WORKSPACE_OWNERSHIP_FILE: ${{ steps.prepare.outputs.work_dir }}/workspace-owner.json
Expand Down Expand Up @@ -127,5 +127,5 @@ jobs:
timeout-minutes: 10
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: staging-launchable-control-plane-${{ github.run_id }}-${{ github.run_attempt }}
name: staging-launchable-remote-execution-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ steps.prepare.outputs.work_dir }}/
1 change: 1 addition & 0 deletions test/e2e/RETRY_INVENTORY.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@
| --- | --- | --- | --- | --- | --- |
| `external-gateway-health.tcp-readiness` | `openshell-gateway` | 10 attempts, one second apart | The newly started gateway listener rejects a TCP connection with `ECONNREFUSED` | The probe is read-only. Other errors stop without retry. The Blueprint Runner health operation runs once after the listener opens. | `external-gateway-readiness-retry.json` |
| `mcp-bridge.tool-discovery` | `test/e2e/live/mcp-bridge-tool-discovery.ts` | 2 attempts, one second apart | Discovery reports `failureClass: connection` before any request reaches the fixture | The operation lists tools without calling them. Any fixture request or other failure class stops the retry. | First-attempt and `retry-2` command artifacts plus `*-mcp-tool-discovery-diagnostics.json` |
| `staging-launchable.remote-execution-readiness` | `test/e2e/fixtures/brev-launchable.ts` | 15 minutes, 15 seconds between attempts, 30 seconds per attempt | `brev exec <owned-workspace-id> true` returns a nonzero exit, signal, or timeout while the newly created workspace becomes remotely reachable | The probe is read-only and idempotent. Each attempt first verifies the persisted owned ID still matches the exact-name workspace inventory. | `brev-exec-readiness-failure.json` records the attempt count and final size-bounded, redacted diagnostic on timeout. |
19 changes: 12 additions & 7 deletions test/e2e/live/issue-9880-staging-launchable.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,19 @@ import { ISSUE_9880_STAGING_LAUNCHABLE_CLEANUP_TIMEOUT_MS } from "../../../tools
import { BrevLaunchableFixture } from "../fixtures/brev-launchable.ts";
import { test } from "../fixtures/e2e-test.ts";

const CONTROL_PLANE_TEST_TIMEOUT_MS = 45 * 60_000;
const REMOTE_EXECUTION_READINESS_TEST_TIMEOUT_MS = 60 * 60_000;

test(
"creates the staging Launchable workspace and records its control-plane identity",
"creates the staging Launchable workspace and proves remote execution readiness",
{
timeout: CONTROL_PLANE_TEST_TIMEOUT_MS,
timeout: REMOTE_EXECUTION_READINESS_TEST_TIMEOUT_MS,
meta: {
e2eCleanupTimeoutMs: ISSUE_9880_STAGING_LAUNCHABLE_CLEANUP_TIMEOUT_MS,
e2ePhases: [
"resolve the latest staging handoff",
"create the staging workspace",
"record the control-plane checkpoint",
"prove remote execution readiness",
"record remote execution readiness",
],
},
},
Expand All @@ -33,19 +34,23 @@ test(
cleanup.add(`delete Brev workspace ${name}`, () => brevLaunchable.delete(ownership));
const workspace = await brevLaunchable.create(ownership, launchableId);

progress.phase("record the control-plane checkpoint");
await artifacts.writeJson("staging-launchable-control-plane-checkpoint.json", {
progress.phase("prove remote execution readiness");
await brevLaunchable.waitForExec(ownership);

progress.phase("record remote execution readiness");
await artifacts.writeJson("staging-launchable-remote-execution-readiness.json", {
bootImage: handoff.bootImage,
candidateSha: handoff.nemoclawSha,
imageRepositorySha: handoff.imageRepositorySha,
launchableId,
producerRunId: handoff.producerRunId,
remoteExecutionReady: true,
workspaceId: workspace.id,
workspaceName: workspace.name,
});
await artifacts.target.complete({
id: "staging-launchable-full",
classification: "control-plane-checkpoint-passed",
classification: "remote-execution-ready",
});
},
);
29 changes: 29 additions & 0 deletions test/e2e/support/brev-launchable-fixture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,35 @@ describe("the Brev Launchable fixture binds staging identity and workspace lifec
expect(ownership.id).toBe("owned-id");
});

it("proves Brev exec readiness after bounded failed probes", async () => {
vi.useFakeTimers();
const root = temporaryRoot();
let attempts = 0;
const command = vi.fn(async (_binary: string, args: string[]) => {
switch (args[0]) {
case "ls":
return workspaceResult("owned-id");
case "exec":
attempts += 1;
return { ...result(""), exitCode: attempts < 3 ? 1 : 0 };
default:
throw new Error(`unexpected command: ${args.join(" ")}`);
}
});
const fixture = createFixture(root, command);

const readiness = fixture.waitForExec(recordedOwnership(), 100);
await vi.runAllTimersAsync();
await expect(readiness).resolves.toBeUndefined();

expect(command.mock.calls.filter((call) => call[1][0] === "exec")).toEqual([
["brev", ["exec", "owned-id", "true"], expect.any(Object)],
["brev", ["exec", "owned-id", "true"], expect.any(Object)],
["brev", ["exec", "owned-id", "true"], expect.any(Object)],
]);
expect(fs.existsSync(path.join(root, "brev-exec-readiness-failure.json"))).toBe(false);
});

it("records the last failed Brev exec readiness attempt", async () => {
vi.useFakeTimers();
const root = temporaryRoot();
Expand Down
Loading