Skip to content

test(e2e): derive inventory and selector contracts #6710

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cv merged 2 commits into from
Jul 12, 2026
Merged

test(e2e): derive inventory and selector contracts #6710

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b8d7399
test(e2e): derive inventory and selector contracts
cv Jul 12, 2026
0161c69
test(e2e): keep matrix assertions linear
cv Jul 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 11 additions & 35 deletions test/e2e/support/e2e-manifests.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,24 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";
import path from "node:path";
import { describe, expect, it } from "vitest";

import { loadManifest, loadManifestsFromDir, validateManifest } from "../registry/manifests.ts";
import { loadManifestsFromDir, validateManifest } from "../registry/manifests.ts";
import { listTargets } from "../registry/registry.ts";

const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
const E2E_SUITE_DIR = path.join(REPO_ROOT, "test/e2e");
const MANIFEST_DIR = path.join(E2E_SUITE_DIR, "manifests");

describe("NemoClawInstance manifests", () => {
it("should validate all NemoClaw instance manifests", () => {
it("loads every checked-in instance manifest through validation", () => {
const manifests = loadManifestsFromDir(MANIFEST_DIR);

expect(manifests.length).toBeGreaterThanOrEqual(19);
for (const manifest of manifests) {
expect(() => validateManifest(manifest.document, manifest.filePath)).not.toThrow();
}
expect(manifests).not.toHaveLength(0);
});

it("should reject manifest with assertion or suite IDs", () => {
it("rejects manifest assertion and suite IDs", () => {
const badManifest = {
apiVersion: "nemoclaw.io/v1",
kind: "NemoClawInstance",
Expand All @@ -39,7 +36,7 @@ describe("NemoClawInstance manifests", () => {
);
});

it("should reject raw secret values in manifest", () => {
it("rejects raw secret values", () => {
const badManifest = {
apiVersion: "nemoclaw.io/v1",
kind: "NemoClawInstance",
Expand All @@ -56,37 +53,16 @@ describe("NemoClawInstance manifests", () => {
);
});

it("should cover every typed target manifest need", () => {
const manifestNames = new Set(
loadManifestsFromDir(MANIFEST_DIR).map((manifest) => manifest.document.metadata.name),
it("resolves every typed target manifest path to a validated manifest", () => {
const manifestPaths = new Set(
loadManifestsFromDir(MANIFEST_DIR).map((manifest) => path.resolve(manifest.filePath)),
);
const missingManifests = listTargets()
.map((target) => target.manifestPath)
.filter((manifestPath): manifestPath is string => Boolean(manifestPath))
.map((manifestPath) => path.basename(manifestPath, ".yaml"))
.filter((id) => !manifestNames.has(id));
.map((manifestPath) => path.resolve(REPO_ROOT, manifestPath))
.filter((manifestPath) => !manifestPaths.has(manifestPath));

expect(missingManifests, `missing manifest files: ${missingManifests.join(", ")}`).toEqual([]);
});

it("registry target manifest paths resolve setup and onboarding choices", () => {
const target = listTargets().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw");

expect(target).toBeTruthy();
expect(target!.manifestPath).toBe("test/e2e/manifests/openclaw-nvidia.yaml");
const manifest = loadManifest(path.join(REPO_ROOT, target!.manifestPath as string)).document;
expect(manifest.spec.setup.install.source).toBe("repo-current");
expect(manifest.spec.onboarding.agent).toBe("openclaw");
expect(manifest.spec.onboarding.provider).toBe("nvidia");
});

it("declares observability on the canonical Deep Agents Code live target", () => {
const target = listTargets().find(
(entry) => entry.id === "ubuntu-repo-cloud-langchain-deepagents-code",
);

expect(target).toBeTruthy();
const manifest = loadManifest(path.join(REPO_ROOT, target!.manifestPath as string)).document;
expect(manifest.spec.onboarding.features?.observability).toBe(true);
});
});
117 changes: 39 additions & 78 deletions test/e2e/support/e2e-matrix.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import path from "node:path";

import { describe, expect, it } from "vitest";
import { target } from "../registry/builder.ts";
import { listTargets } from "../registry/registry.ts";
import { buildLiveTargetMatrix } from "../registry/run.ts";
import { resolveRunnerForTarget } from "../registry/runner-routing.ts";
import { liveTargetSupport } from "../registry/runtime-support.ts";

const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
const RUN_TARGETS = path.join(REPO_ROOT, "test/e2e/registry/run.ts");
Expand All @@ -21,6 +23,12 @@ function runEmitLiveMatrix(args: string[] = []) {
});
}

function requireUnsupportedTarget() {
const unsupported = listTargets().find((entry) => !liveTargetSupport(entry).supported);
expect(unsupported, "expected at least one unsupported live E2E target").toBeDefined();
return unsupported!;
}

describe("live E2E target matrix", () => {
it("honors an explicit runs-on:<label> requirement override", () => {
const custom = target("test-runs-on-override")
Expand Down Expand Up @@ -75,76 +83,38 @@ describe("live E2E target matrix", () => {
expect(() => resolveRunnerForTarget(broken)).toThrow(/no default for platform/);
});

it("builds the default live matrix from fixture-supported targets only", () => {
expect(buildLiveTargetMatrix().map((entry) => entry.id)).toEqual([
"ubuntu-policy-custom-missing-presets-negative",
"ubuntu-repo-cloud-langchain-deepagents-code",
"ubuntu-repo-cloud-openclaw",
"ubuntu-repo-docker-post-reboot-recovery",
]);
expect(buildLiveTargetMatrix()[0]).toMatchObject({
id: "ubuntu-policy-custom-missing-presets-negative",
runner: "ubuntu-latest",
platform: "ubuntu-local",
install: "repo-current",
runtime: "docker-running",
onboarding: "cloud-openclaw-policy-custom-missing-presets",
expectedStateId: "onboarding-failure-policy-presets-required",
requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"],
supported: true,
supportReasons: [],
pendingRuntimeSuites: [],
});
expect(buildLiveTargetMatrix()[1]).toMatchObject({
id: "ubuntu-repo-cloud-langchain-deepagents-code",
runner: "ubuntu-latest",
platform: "ubuntu-local",
install: "repo-current",
runtime: "docker-running",
onboarding: "cloud-langchain-deepagents-code",
expectedStateId: "cloud-deepagents-code-ready",
requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"],
supported: true,
supportReasons: [],
pendingRuntimeSuites: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"],
});
expect(buildLiveTargetMatrix()[2]).toMatchObject({
id: "ubuntu-repo-cloud-openclaw",
runner: "ubuntu-latest",
platform: "ubuntu-local",
install: "repo-current",
runtime: "docker-running",
onboarding: "cloud-openclaw",
expectedStateId: "cloud-openclaw-ready",
requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"],
supported: true,
supportReasons: [],
pendingRuntimeSuites: ["smoke", "inference", "credentials"],
});
// Failing-test-first guard for #4423. Pinned in the matrix to
// confirm the lifecycle whitelist + post-reboot-recovery target
// are wired together; the actual RED/GREEN behavior is exercised
// by the live runner (gates on the fix landing in src/lib/).
expect(buildLiveTargetMatrix()[3]).toMatchObject({
id: "ubuntu-repo-docker-post-reboot-recovery",
runner: "ubuntu-latest",
platform: "ubuntu-local",
install: "repo-current",
runtime: "docker-running",
onboarding: "cloud-openclaw",
expectedStateId: "post-reboot-recovery-ready",
requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"],
supported: true,
supportReasons: [],
});
it("builds the default live matrix from every fixture-supported target", () => {
const targets = listTargets();
const supportedTargets = targets.filter((entry) => liveTargetSupport(entry).supported);
const matrix = buildLiveTargetMatrix();

expect(matrix).not.toHaveLength(0);
expect(matrix.map((entry) => entry.id)).toEqual(supportedTargets.map((entry) => entry.id));
expect(new Set(matrix.map((entry) => entry.id)).size).toBe(matrix.length);
for (const entry of matrix) {
const registered = supportedTargets.find((target) => target.id === entry.id);
expect(
registered,
`matrix entry '${entry.id}' must resolve to a supported target`,
).toBeDefined();
expect(entry).toMatchObject({
runner: resolveRunnerForTarget(registered!).runner,
supported: true,
supportReasons: [],
pendingRuntimeSuites: registered!.suiteIds ?? [],
});
}
});

it("keeps explicitly selected unsupported live targets in the matrix with skip reasons", () => {
expect(buildLiveTargetMatrix(["ubuntu-repo-cloud-hermes"])).toEqual([
const unsupported = requireUnsupportedTarget();
const support = liveTargetSupport(unsupported);

expect(buildLiveTargetMatrix([unsupported.id])).toEqual([
expect.objectContaining({
id: "ubuntu-repo-cloud-hermes",
id: unsupported.id,
supported: false,
supportReasons: ["onboarding 'cloud-hermes' is not wired for live fixtures"],
supportReasons: support.reasons,
}),
]);
});
Expand All @@ -155,24 +125,15 @@ describe("live E2E target matrix", () => {
const lines = result.stdout.trim().split("\n");
expect(lines.length, "live matrix output must be a single line").toBe(1);
const parsed = JSON.parse(lines[0]);
expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([
"ubuntu-policy-custom-missing-presets-negative",
"ubuntu-repo-cloud-langchain-deepagents-code",
"ubuntu-repo-cloud-openclaw",
"ubuntu-repo-docker-post-reboot-recovery",
]);
expect(parsed).toEqual(buildLiveTargetMatrix());
});

it("honors explicit target selections for --emit-live-matrix", () => {
const result = runEmitLiveMatrix(["--targets", "ubuntu-repo-cloud-hermes"]);
const unsupported = requireUnsupportedTarget();
const result = runEmitLiveMatrix(["--targets", unsupported.id]);
expect(result.status, result.stderr).toBe(0);
const parsed = JSON.parse(result.stdout.trim());
expect(parsed).toEqual([
expect.objectContaining({
id: "ubuntu-repo-cloud-hermes",
supported: false,
}),
]);
expect(parsed).toEqual(buildLiveTargetMatrix([unsupported.id]));
});

it("rejects retired typed-shell runner flags", () => {
Expand Down
Loading
Loading