Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 10 additions & 40 deletions test/e2e/support/e2e-host-dependency-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,10 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { describe, expect, it } from "vitest";
import YAML from "yaml";

import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts";

const WORKFLOW_PATH = path.join(process.cwd(), ".github/workflows/e2e.yaml");
import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts";
import { readWorkflow as readE2eWorkflow } from "../../helpers/e2e-workflow-contract.ts";

interface WorkflowStep {
name?: string;
Expand All @@ -22,7 +16,7 @@ interface Workflow {
}

function readWorkflow(): Workflow {
return YAML.parse(fs.readFileSync(WORKFLOW_PATH, "utf8")) as Workflow;
return readE2eWorkflow() as unknown as Workflow;
}

function throwMissingStep(stepName: string): never {
Expand Down Expand Up @@ -67,42 +61,24 @@ describe("inline E2E host dependency boundary", () => {
"openclaw-tui-chat-correlation host dependency install must be exactly 'sudo apt-get install -y --no-install-recommends expect'",
},
])("rejects package allowlist drift in $jobName", ({ jobName, stepName, expected }) => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-apt-allowlist-"));
const workflowPath = path.join(tmp, "workflow.yaml");
const workflow = readWorkflow();
const install = workflow.jobs[jobName]?.steps.find((step) => step.name === stepName)!;
install.run = (install.run ?? "").replace(/(sudo apt-get install[^\n]+)/u, "$1 curl");
fs.writeFileSync(workflowPath, YAML.stringify(workflow));

try {
expect(validateE2eWorkflowBoundary(workflowPath)).toContain(expected);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
expect(validateE2eWorkflow(workflow)).toContain(expected);
});

it("rejects installing the OpenClaw TUI host dependency after workspace preparation", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-host-dependency-order-"));
const workflowPath = path.join(tmp, "workflow.yaml");
const workflow = readWorkflow();
const steps = workflow.jobs["openclaw-tui-chat-correlation"].steps;
const installIndex = requireStepIndex(steps, "Install OpenClaw TUI host dependencies");
const prepareIndex = requireStepIndex(steps, "Prepare E2E workspace");
[steps[installIndex], steps[prepareIndex]] = [steps[prepareIndex]!, steps[installIndex]!];
fs.writeFileSync(workflowPath, YAML.stringify(workflow));

try {
expect(validateE2eWorkflowBoundary(workflowPath)).toContain(
"openclaw-tui-chat-correlation host dependencies must be installed before workspace prep",
);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
expect(validateE2eWorkflow(workflow)).toContain(
"openclaw-tui-chat-correlation host dependencies must be installed before workspace prep",
);
});

it("keeps cloud-onboard host dependencies before workspace preparation", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-host-order-"));
const workflowPath = path.join(tmp, "workflow.yaml");
const workflow = readWorkflow();
const steps = workflow.jobs["cloud-onboard"].steps;
const installIndex = requireStepIndex(
Expand All @@ -112,14 +88,8 @@ describe("inline E2E host dependency boundary", () => {
const install = steps.splice(installIndex, 1)[0]!;
const prepareIndex = requireStepIndex(steps, "Prepare E2E workspace");
steps.splice(prepareIndex + 1, 0, install);
fs.writeFileSync(workflowPath, YAML.stringify(workflow));

try {
expect(validateE2eWorkflowBoundary(workflowPath)).toContain(
"cloud-onboard DCode TUI host dependencies must precede workspace prep",
);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
expect(validateE2eWorkflow(workflow)).toContain(
"cloud-onboard DCode TUI host dependencies must precede workspace prep",
);
});
});
78 changes: 50 additions & 28 deletions test/e2e/support/e2e-live-target-gating.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import path from "node:path";

import { describe, expect, it } from "vitest";

import { testTimeoutOptions } from "../../helpers/timeouts.ts";
import { LIVE_E2E_ROOT, REPO_ROOT } from "../fixtures/paths.ts";

const VITEST = path.join(REPO_ROOT, "node_modules", "vitest", "vitest.mjs");
Expand All @@ -30,14 +31,14 @@ function liveTestFiles(root = LIVE_E2E_ROOT): string[] {
function listLiveTests(options: {
enabled: boolean;
env?: NodeJS.ProcessEnv;
file?: string;
files?: readonly string[];
filesOnly?: boolean;
}) {
const args = [
"list",
"--project",
"e2e-live",
...(options.file ? [`test/e2e/live/${options.file}`] : []),
...(options.files ?? []).map((file) => `test/e2e/live/${file}`),
...(options.filesOnly ? ["--filesOnly"] : []),
"--passWithNoTests",
];
Expand All @@ -61,6 +62,10 @@ function listLiveTests(options: {
};
}

function linesForFile(lines: readonly string[], file: string): string[] {
return lines.filter((line) => line.startsWith(`[e2e-live] test/e2e/live/${file} >`));
}

describe("live E2E target gating", () => {
it("collects no live files without project opt-in and all live files with it", () => {
const disabled = listLiveTests({ enabled: false, filesOnly: true });
Expand All @@ -76,35 +81,52 @@ describe("live E2E target gating", () => {
expect(collected).toEqual(discovered);
});

it.each([
["sandbox-rlimits-connect.test.ts", "NEMOCLAW_E2E_CONNECT_RLIMITS"],
["mcp-bridge.test.ts", "NEMOCLAW_MCP_BRIDGE_AGENT_MATRIX"],
["issue-4434-tui-unreachable-inference.test.ts", "NEMOCLAW_ISSUE_4434_LIVE"],
] as const)("applies %s's explicit opt-in at real Vitest collection", (file, gate) => {
const disabled = listLiveTests({ enabled: true, file });
const enabled = listLiveTests({ enabled: true, env: { [gate]: "1" }, file });
it(
"applies each special target's explicit opt-in at real Vitest collection",
testTimeoutOptions(15_000),
() => {
const gatedFiles = [
["sandbox-rlimits-connect.test.ts", "NEMOCLAW_E2E_CONNECT_RLIMITS"],
["mcp-bridge.test.ts", "NEMOCLAW_MCP_BRIDGE_AGENT_MATRIX"],
["issue-4434-tui-unreachable-inference.test.ts", "NEMOCLAW_ISSUE_4434_LIVE"],
] as const;
const files = gatedFiles.map(([file]) => file);
const disabled = listLiveTests({ enabled: true, files });

expect(disabled.status, disabled.stderr || disabled.stdout).toBe(0);
expect(enabled.status, enabled.stderr || enabled.stdout).toBe(0);
expect(
enabled.lines.length,
`${file} should collect more tests when ${gate}=1`,
).toBeGreaterThan(disabled.lines.length);
});
expect(disabled.status, disabled.stderr || disabled.stdout).toBe(0);
for (const [file, gate] of gatedFiles) {
const enabled = listLiveTests({ enabled: true, env: { [gate]: "1" }, files: [file] });

expect(enabled.status, enabled.stderr || enabled.stdout).toBe(0);
expect(
linesForFile(enabled.lines, file).length,
`${file} should collect more tests when ${gate}=1`,
).toBeGreaterThan(linesForFile(disabled.lines, file).length);
}
},
);

it.each([
[
"spark-install.test.ts",
"spark install path: standard non-interactive install leaves NemoClaw and OpenShell usable",
],
[
"openshell-gateway-upgrade.test.ts",
"openshell-gateway-upgrade: upgrades old working OpenClaw claw and restores survivor state",
],
])("applies %s's Linux gate at real Vitest collection", (file, testName) => {
const result = listLiveTests({ enabled: true, file });
it("applies Linux gates at real Vitest collection", () => {
const linuxTests = [
[
"spark-install.test.ts",
"spark install path: standard non-interactive install leaves NemoClaw and OpenShell usable",
],
[
"openshell-gateway-upgrade.test.ts",
"openshell-gateway-upgrade: upgrades old working OpenClaw claw and restores survivor state",
],
] as const;
const result = listLiveTests({
enabled: true,
files: linuxTests.map(([file]) => file),
});

expect(result.status, result.stderr || result.stdout).toBe(0);
expect(result.lines.some((line) => line.endsWith(testName))).toBe(process.platform === "linux");
for (const [file, testName] of linuxTests) {
expect(linesForFile(result.lines, file).some((line) => line.endsWith(testName))).toBe(
process.platform === "linux",
);
}
});
});
21 changes: 14 additions & 7 deletions test/e2e/support/e2e-recovery-helpers.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";

import { GatewayClient, HostCliClient, SandboxClient } from "../fixtures/clients/index.ts";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { CommandRunner } from "../fixtures/clients/index.ts";
import { GatewayClient, HostCliClient, SandboxClient } from "../fixtures/clients/index.ts";
import type { NemoClawInstance } from "../fixtures/phases/onboarding.ts";
import type {
ShellProbeResult,
Expand Down Expand Up @@ -206,6 +205,9 @@ describe("GatewayClient recovery helpers (#2701)", () => {
});

describe("expectPidStable", () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());

it("returns the PID when it is stable across all samples", async () => {
const runner = new ScriptedRunner();
// initial sample + 3 stable samples
Expand All @@ -217,37 +219,42 @@ describe("GatewayClient recovery helpers (#2701)", () => {
);
const gateway = buildGateway(runner);

const pid = await gateway.expectPidStable(fakeInstance(), {
const observation = gateway.expectPidStable(fakeInstance(), {
durationSeconds: 3,
pollIntervalSeconds: 1,
});
expect(pid).toBe(100);
await vi.runAllTimersAsync();
await expect(observation).resolves.toBe(100);
});

it("throws when the PID changes (crash-loop)", async () => {
const runner = new ScriptedRunner();
runner.queue({ stdout: "100\n" }, { stdout: "201\n" });
const gateway = buildGateway(runner);

await expect(
const observation = expect(
gateway.expectPidStable(fakeInstance(), {
durationSeconds: 2,
pollIntervalSeconds: 1,
}),
).rejects.toThrow(/PID changed 100→201.*crash-loop/);
await vi.runAllTimersAsync();
await observation;
});

it("throws when the gateway disappears mid-window", async () => {
const runner = new ScriptedRunner();
runner.queue({ stdout: "100\n" }, { stdout: "" });
const gateway = buildGateway(runner);

await expect(
const observation = expect(
gateway.expectPidStable(fakeInstance(), {
durationSeconds: 2,
pollIntervalSeconds: 1,
}),
).rejects.toThrow(/gateway disappeared/);
await vi.runAllTimersAsync();
await observation;
});

it("throws when no gateway exists at the start of the window", async () => {
Expand Down
73 changes: 13 additions & 60 deletions test/e2e/support/e2e-workflow.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
Expand All @@ -15,53 +14,11 @@ import {
validateE2eWorkflowBoundary,
validateFreeStandingWorkflowInventory,
} from "../../../tools/e2e/workflow-boundary.mts";
import { buildE2eWorkflowPlan } from "../../../tools/e2e/workflow-plan.mts";
import { readWorkflow, removeJobNeed } from "../../helpers/e2e-workflow-contract";
import { testTimeoutOptions } from "../../helpers/timeouts";
import { assertChannelsStopStartSandboxName } from "../live/channels-stop-start-safety.ts";

function generateMatrixScript(): string {
const workflow = readWorkflow();
const jobs = workflow.jobs as Record<string, { steps?: Array<Record<string, unknown>> }>;
const generateStep = jobs["generate-matrix"]?.steps?.find(
(step) => step.name === "Generate E2E target matrix",
);
expect(generateStep?.run).toEqual(expect.any(String));
return generateStep?.run as string;
}

function generateMatrixForDispatch(env: { JOBS: string; TARGETS: string }): Record<string, string> {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-matrix-"));
const outputPath = path.join(tmp, "github-output");
const summaryPath = path.join(tmp, "github-summary");
try {
const result = spawnSync("bash", ["-c", generateMatrixScript()], {
cwd: process.cwd(),
encoding: "utf-8",
timeout: 120_000,
killSignal: "SIGKILL",
env: {
...process.env,
GITHUB_OUTPUT: outputPath,
GITHUB_STEP_SUMMARY: summaryPath,
JOBS: env.JOBS,
TARGETS: env.TARGETS,
},
});
expect(result.signal).toBeNull();
expect(result.stderr).toBe("");
expect(result.status).toBe(0);
return Object.fromEntries(
fs
.readFileSync(outputPath, "utf-8")
.trim()
.split("\n")
.map((line) => line.split(/=(.*)/s).slice(0, 2)),
);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
}

describe("e2e workflow boundary", () => {
it("guards channels-stop-start destructive cleanup to test-owned sandboxes", () => {
expect(() => assertChannelsStopStartSandboxName("personal-dev")).toThrow(
Expand Down Expand Up @@ -346,25 +303,21 @@ jobs:
inventory.allowedJobs.filter((job) => !inventory.explicitOnlyJobs.includes(job)).sort(),
);

expect(
generateMatrixForDispatch({ JOBS: nonHermesJobs.join(","), TARGETS: "" }),
).toMatchObject({
hermes_selected: "false",
matrix: "[]",
expect(buildE2eWorkflowPlan({ jobs: nonHermesJobs.join(",") })).toMatchObject({
hermesSelected: false,
matrix: [],
});
expect(generateMatrixForDispatch({ JOBS: hermesSelector, TARGETS: "" })).toMatchObject({
hermes_selected: "true",
matrix: "[]",
expect(buildE2eWorkflowPlan({ jobs: hermesSelector })).toMatchObject({
hermesSelected: true,
matrix: [],
});
expect(
generateMatrixForDispatch({ JOBS: "", TARGETS: nonHermesTargets.join(",") }),
).toMatchObject({
hermes_selected: "false",
matrix: "[]",
expect(buildE2eWorkflowPlan({ targets: nonHermesTargets.join(",") })).toMatchObject({
hermesSelected: false,
matrix: [],
});
expect(generateMatrixForDispatch({ JOBS: "", TARGETS: hermesSelector })).toMatchObject({
hermes_selected: "true",
matrix: "[]",
expect(buildE2eWorkflowPlan({ targets: hermesSelector })).toMatchObject({
hermesSelected: true,
matrix: [],
});

for (const job of inventory.allowedJobs) {
Expand Down
Loading