From 711463445ee717ce177ae3437c85a7b39b84d178 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 12 Jul 2026 01:46:55 -0700 Subject: [PATCH 01/21] test(config): replace declarative shape mirrors --- test/validate-blueprint.test.ts | 91 +----------- test/validate-config-schemas.test.ts | 215 +++++++++++++++------------ 2 files changed, 121 insertions(+), 185 deletions(-) diff --git a/test/validate-blueprint.test.ts b/test/validate-blueprint.test.ts index 5ecbb77215c..5f0844bf4a8 100644 --- a/test/validate-blueprint.test.ts +++ b/test/validate-blueprint.test.ts @@ -2,10 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Validate blueprint.yaml profile declarations and base sandbox policy. - * - * Catches configuration regressions (missing profiles, empty fields, - * missing policy sections) before merge. + * Protect security and routing semantics declared by the shipping blueprint, + * provider profiles, and policy files. Structural validation belongs to + * scripts/validate-configs.ts. */ import { readFileSync } from "node:fs"; @@ -54,24 +53,10 @@ const OPENCLAW_PERMISSIVE_POLICY_PATH = new URL( "../agents/openclaw/policy-permissive.yaml", import.meta.url, ); -const REQUIRED_PROFILE_FIELDS: ReadonlyArray = [ - "provider_type", - "endpoint", -]; - -type BlueprintProfile = { - provider_type?: string; - endpoint?: string; - dynamic_endpoint?: boolean; -}; - type Blueprint = { - version?: string; digest?: string; - profiles?: string[]; components?: { sandbox?: { image?: string | null }; - inference?: { profiles?: Record }; }; }; @@ -145,23 +130,8 @@ function loadYaml(path: URL): T { } const bp = loadYaml(BLUEPRINT_PATH); -const declared = Array.isArray(bp.profiles) ? bp.profiles : []; -const defined = bp.components?.inference?.profiles; describe("blueprint.yaml", () => { - it("parses as a YAML mapping", () => { - expect(bp).toEqual(expect.objectContaining({})); - }); - - it("has a non-empty top-level profiles list", () => { - expect(declared.length).toBeGreaterThan(0); - }); - - it("has a non-empty components.inference.profiles mapping", () => { - expect(defined).toBeDefined(); - expect(Object.keys(defined ?? {}).length).toBeGreaterThan(0); - }); - it("pins the sandbox image by digest instead of a mutable tag (#1438)", () => { // The blueprint MUST NOT pull a sandbox image by a mutable tag like // ":latest" — a registry compromise or accidental force-push could @@ -202,33 +172,6 @@ describe("blueprint.yaml", () => { // the other, this assertion catches it before merge. expect(topLevelDigest).toBe(imageDigest); }); - - for (const name of declared) { - describe(`profile '${name}'`, () => { - it("has a definition", () => { - expect(defined).toBeDefined(); - expect(name in (defined ?? {})).toBe(true); - }); - - for (const field of REQUIRED_PROFILE_FIELDS) { - it(`has non-empty '${field}'`, () => { - const cfg = defined?.[name]; - if (!cfg) return; // covered by "has a definition" - if (field === "endpoint" && cfg.dynamic_endpoint === true) { - expect(field in cfg).toBe(true); - } else { - expect(cfg[field]).toBeTruthy(); - } - }); - } - }); - } - - for (const name of Object.keys(defined ?? {})) { - it(`defined profile '${name}' is declared in top-level list`, () => { - expect(declared).toContain(name); - }); - } }); describe("Model Router pool config", () => { @@ -258,18 +201,6 @@ describe("Model Router pool config", () => { describe("base sandbox policy", () => { const policy = loadYaml(BASE_POLICY_PATH); - it("parses as a YAML mapping", () => { - expect(policy).toEqual(expect.objectContaining({})); - }); - - it("has 'version'", () => { - expect("version" in policy).toBe(true); - }); - - it("has 'network_policies'", () => { - expect("network_policies" in policy).toBe(true); - }); - it("no endpoint rule uses wildcard method", () => { const np = policy.network_policies ?? {}; const violations: string[] = []; @@ -648,10 +579,6 @@ describe("permissive sandbox policy", () => { const policy = loadYaml(PERMISSIVE_POLICY_PATH); const agentPolicy = loadYaml(OPENCLAW_PERMISSIVE_POLICY_PATH); - it("parses and declares network_policies", () => { - expect(policy.network_policies).toBeDefined(); - }); - it("allows inference.local:443 in the managed_inference block (#2513)", () => { const np = policy.network_policies ?? {}; expect(np.managed_inference).toBeDefined(); @@ -740,23 +667,11 @@ describe("Hermes sandbox policy", () => { }); describe("github preset", () => { - // The fix for #1583 was *only* meaningful if the github preset - // actually exists and is loadable — otherwise users have no way to - // opt in. Verify the preset file is present and well-formed. const PRESET_PATH = new URL( "../nemoclaw-blueprint/policies/presets/github.yaml", import.meta.url, ); - it("parses the existing github preset file (#1583)", () => { - const parsed = loadYaml(PRESET_PATH); - expect(parsed).toEqual(expect.objectContaining({})); - const meta = parsed.preset; - expect(meta?.name).toBe("github"); - const np = parsed.network_policies; - expect(np && "github" in np).toBe(true); - }); - it("only advertises the installed git binary in the github preset (#2179)", () => { const parsed = loadYaml(PRESET_PATH); const meta = parsed.preset; diff --git a/test/validate-config-schemas.test.ts b/test/validate-config-schemas.test.ts index 97d126382bb..577cf7901bf 100644 --- a/test/validate-config-schemas.test.ts +++ b/test/validate-config-schemas.test.ts @@ -2,19 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Validate config files against their JSON Schemas. + * Exercise config JSON Schemas with focused synthetic fixtures. * - * Complements validate-blueprint.test.ts (business-logic invariants) with - * structural/type validation via JSON Schema. Runs as part of the "cli" - * Vitest project. + * Checked-in config files are validated by scripts/validate-configs.ts. This + * suite protects schema behavior without coupling it to those config values. */ -import { existsSync, readFileSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv, { type ValidateFunction } from "ajv/dist/2020.js"; import { describe, expect, it } from "vitest"; -import YAML from "yaml"; import { discoverTargets } from "../scripts/validate-configs"; @@ -52,14 +50,6 @@ function isLooseObject(value: LooseValue | object | undefined): value is LooseOb ); } -function loadYAML(path: string): LooseObject { - const parsed = YAML.parse(readFileSync(path, "utf-8")); - if (!isLooseObject(parsed)) { - throw new Error(`Expected YAML object in ${path}`); - } - return parsed; -} - function loadJSON(path: string): LooseObject { const parsed = parseJson(readFileSync(path, "utf-8")); if (!isLooseObject(parsed)) { @@ -293,15 +283,23 @@ describe("config validation target discovery", () => { describe("onboard-config.schema.json", () => { const validate = compileSchema("schemas/onboard-config.schema.json"); - const data = loadJSON(repoPath("ci/onboard-performance-budget.json")); + const validOnboardConfig = { + $comment: "Schema fixture", + schemaVersion: 1, + mode: "advisory", + scope: "fixture", + totalBudgetMs: 0, + regressionWarning: { minDeltaMs: 0, minPercent: 0 }, + phaseRegressionWarning: { minDeltaMs: 0, minPercent: 0 }, + }; - it("onboard-performance-budget.json passes schema validation", () => { - expectValid(validate, data, "onboard-performance-budget.json"); + it("accepts a minimal onboard performance budget", () => { + expectValid(validate, validOnboardConfig, "minimal onboard config"); }); it("rejects invalid threshold shapes", () => { const bad = { - ...cloneObject(data), + ...validOnboardConfig, regressionWarning: { minDeltaMs: -1, minPercent: 20 }, }; expect(validate(bad)).toBe(false); @@ -312,30 +310,41 @@ describe("onboard-config.schema.json", () => { describe("blueprint.schema.json", () => { const validate = compileSchema("schemas/blueprint.schema.json"); - const data = loadYAML(repoPath("nemoclaw-blueprint/blueprint.yaml")); + const validBlueprint = { + version: "1.0.0", + profiles: ["default"], + components: { + sandbox: { image: "example.invalid/nemoclaw:fixture", name: "fixture" }, + inference: { + profiles: { + default: { provider_type: "openai", endpoint: "https://api.example.com" }, + }, + }, + }, + }; - it("blueprint.yaml passes schema validation", () => { - expectValid(validate, data, "blueprint.yaml"); + it("accepts a minimal blueprint", () => { + expectValid(validate, validBlueprint, "minimal blueprint"); }); it("rejects blueprint with missing required field", () => { - const bad = cloneObject(data); + const bad = cloneObject(validBlueprint); delete bad.version; expect(validate(bad)).toBe(false); }); it("rejects blueprint with wrong type for version", () => { - const bad = { ...cloneObject(data), version: 123 }; + const bad = { ...validBlueprint, version: 123 }; expect(validate(bad)).toBe(false); }); it("rejects blueprint with unknown top-level property", () => { - const bad = { ...cloneObject(data), unknownField: true }; + const bad = { ...validBlueprint, unknownField: true }; expect(validate(bad)).toBe(false); }); it("rejects blueprint with unknown nested component property", () => { - const root = asRecord(data); + const root = asRecord(validBlueprint); const components = asRecord(root.components); const inference = asRecord(components.inference); const bad = { @@ -352,7 +361,7 @@ describe("blueprint.schema.json", () => { }); it("rejects blueprint inference profile with unknown property", () => { - const root = asRecord(data); + const root = asRecord(validBlueprint); const components = asRecord(root.components); const inference = asRecord(components.inference); const profiles = asRecord(inference.profiles); @@ -406,20 +415,38 @@ describe("blueprint.schema.json", () => { describe("router-pool-config.schema.json", () => { const validate = compileSchema("schemas/router-pool-config.schema.json"); - const data = loadYAML(repoPath("nemoclaw-blueprint/router/pool-config.yaml")); + const validRouterPoolConfig = { + routing: { + method: "fixture", + checkpoint: "fixture", + tolerance: 0.5, + encoder: "fixture", + encoder_backend: "fixture", + }, + models: [ + { + name: "fixture", + display_name: "Fixture", + litellm_model: "openai/fixture", + cost_per_m_input_tokens: 0, + cost_per_m_output_tokens: 0, + api_base: "https://api.example.com/v1", + }, + ], + }; - it("pool-config.yaml passes schema validation", () => { - expectValid(validate, data, "pool-config.yaml"); + it("accepts a minimal router pool config", () => { + expectValid(validate, validRouterPoolConfig, "minimal router pool config"); }); it("rejects router pool config without routing settings", () => { - const bad = cloneObject(data); + const bad = cloneObject(validRouterPoolConfig); delete bad.routing; expect(validate(bad)).toBe(false); }); it("rejects router pool config models without LiteLLM model IDs", () => { - const root = asRecord(data); + const root = asRecord(validRouterPoolConfig); const firstModel = asRecord(Array.isArray(root.models) ? root.models[0] : undefined); const { litellm_model: _litellmModel, ...modelWithoutId } = firstModel; const bad = { ...root, models: [modelWithoutId] }; @@ -427,7 +454,7 @@ describe("router-pool-config.schema.json", () => { }); it("rejects router pool config api_base without HTTPS", () => { - const root = asRecord(data); + const root = asRecord(validRouterPoolConfig); const firstModel = asRecord(Array.isArray(root.models) ? root.models[0] : undefined); const bad = { ...root, @@ -442,40 +469,29 @@ describe("router-pool-config.schema.json", () => { describe("sandbox-policy.schema.json", () => { const validate = compileSchema("schemas/sandbox-policy.schema.json"); registerOpenShellJsonRpcMcpMatcherTests("sandbox", validate); - const data = loadYAML(repoPath("nemoclaw-blueprint/policies/openclaw-sandbox.yaml")); + const validSandboxPolicy = { + version: 1, + network_policies: { + test_service: { + name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], + endpoints: [{ host: "api.example.com", port: 443, access: "full" }], + }, + }, + }; - it("openclaw-sandbox.yaml passes schema validation", () => { - expectValid(validate, data, "openclaw-sandbox.yaml"); + it("accepts a minimal sandbox policy", () => { + expectValid(validate, validSandboxPolicy, "minimal sandbox policy"); }); - it("openclaw-sandbox-permissive.yaml passes schema validation", () => { - expectValid( - validate, - loadYAML(repoPath("nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml")), - "openclaw-sandbox-permissive.yaml", - ); - }); - - for (const file of [ - "agents/openclaw/policy-permissive.yaml", - "agents/hermes/policy-additions.yaml", - "agents/hermes/policy-permissive.yaml", - ]) { - if (existsSync(repoPath(file))) { - it(`${file} passes schema validation`, () => { - expectValid(validate, loadYAML(repoPath(file)), file); - }); - } - } - it("rejects policy with missing network_policies", () => { - const bad = cloneObject(data); + const bad = cloneObject(validSandboxPolicy); delete bad.network_policies; expect(validate(bad)).toBe(false); }); it("rejects policy with unknown top-level property", () => { - const bad = { ...cloneObject(data), extra: true }; + const bad = { ...validSandboxPolicy, extra: true }; expect(validate(bad)).toBe(false); }); @@ -852,16 +868,20 @@ describe("sandbox-policy.schema.json", () => { describe("policy-preset.schema.json", () => { const validate = compileSchema("schemas/policy-preset.schema.json"); registerOpenShellJsonRpcMcpMatcherTests("preset", validate); - const presetFiles = - discoverTargets().find((target) => target.schema === "schemas/policy-preset.schema.json") - ?.files ?? []; - - for (const file of presetFiles) { - it(`${file} passes schema validation`, () => { - const data = loadYAML(repoPath(file)); - expectValid(validate, data, file); - }); - } + const validPolicyPreset = { + preset: { name: "test", description: "Test preset" }, + network_policies: { + test_service: { + name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], + endpoints: [{ host: "api.example.com", port: 443, access: "full" }], + }, + }, + }; + + it("accepts a minimal policy preset", () => { + expectValid(validate, validPolicyPreset, "minimal policy preset"); + }); it("rejects preset without preset metadata", () => { const bad = { @@ -1255,7 +1275,6 @@ describe("policy-preset.schema.json", () => { describe("openclaw-plugin.schema.json", () => { const validate = compileSchema("schemas/openclaw-plugin.schema.json"); - const data = loadJSON(repoPath("nemoclaw/openclaw.plugin.json")); const validPluginFixture = { id: "fixture-plugin", name: "Fixture Plugin", @@ -1266,12 +1285,8 @@ describe("openclaw-plugin.schema.json", () => { activation: { onStartup: true }, }; - it("openclaw.plugin.json passes schema validation", () => { - expectValid(validate, data, "openclaw.plugin.json"); - }); - - it("accepts runtime slash activation metadata", () => { - expectValid(validate, validPluginFixture, "runtime slash activation fixture"); + it("accepts a minimal plugin manifest with runtime slash activation", () => { + expectValid(validate, validPluginFixture, "minimal plugin manifest"); }); it("rejects command alias without kind", () => { @@ -1308,29 +1323,35 @@ describe("openclaw-plugin.schema.json", () => { describe("model-specific-setup/schema.json", () => { const validate = compileSchema("nemoclaw-blueprint/model-specific-setup/schema.json"); - const data = loadJSON( - repoPath("nemoclaw-blueprint/model-specific-setup/openclaw/kimi-k2.6-managed-inference.json"), - ); - const familyData = loadJSON( - repoPath( - "nemoclaw-blueprint/model-specific-setup/openclaw/gpt-5-o-series-managed-inference.json", - ), - ); + const exactModelFixture = { + id: "fixture-openclaw-exact", + agent: "openclaw", + description: "Fixture OpenClaw setup", + match: { modelIds: ["fixture/model"] }, + effects: { openclawCompat: {} }, + }; + const modelFamilyFixture = { + id: "fixture-openclaw-family", + agent: "openclaw", + description: "Fixture OpenClaw model family setup", + match: { modelIdPrefixes: ["fixture"] }, + effects: { openclawCompat: {} }, + }; - it("accepts the OpenClaw Kimi manifest", () => { - expectValid(validate, data, "kimi-k2.6-managed-inference.json"); + it("accepts an exact OpenClaw model selector", () => { + expectValid(validate, exactModelFixture, "exact OpenClaw model selector"); }); - it("accepts bounded model-family prefixes for OpenClaw", () => { - expectValid(validate, familyData, "gpt-5-o-series-managed-inference.json"); + it("accepts a bounded OpenClaw model-family prefix", () => { + expectValid(validate, modelFamilyFixture, "OpenClaw model-family prefix"); }); it("rejects ambiguous exact and prefix model selectors", () => { const bad = { - ...cloneObject(familyData), + ...cloneObject(modelFamilyFixture), match: { - ...asRecord(familyData.match), - modelIds: ["gpt-5"], + ...asRecord(modelFamilyFixture.match), + modelIds: ["fixture/model"], }, }; expect(validate(bad)).toBe(false); @@ -1338,10 +1359,10 @@ describe("model-specific-setup/schema.json", () => { it("rejects namespaced model-family prefixes", () => { const bad = { - ...cloneObject(familyData), + ...cloneObject(modelFamilyFixture), match: { - ...asRecord(familyData.match), - modelIdPrefixes: ["azure/gpt-5"], + ...asRecord(modelFamilyFixture.match), + modelIdPrefixes: ["provider/fixture"], }, }; expect(validate(bad)).toBe(false); @@ -1349,7 +1370,7 @@ describe("model-specific-setup/schema.json", () => { it("rejects OpenClaw manifests with Hermes effects", () => { const bad = { - ...cloneObject(data), + ...cloneObject(exactModelFixture), effects: { hermesCompat: { future: true, @@ -1361,7 +1382,7 @@ describe("model-specific-setup/schema.json", () => { it("rejects manifests with empty match objects", () => { const bad = { - ...cloneObject(data), + ...cloneObject(exactModelFixture), match: {}, }; expect(validate(bad)).toBe(false); @@ -1369,7 +1390,7 @@ describe("model-specific-setup/schema.json", () => { it("rejects whitespace-only manifest strings", () => { const bad = { - ...cloneObject(data), + ...cloneObject(exactModelFixture), description: " ", match: { modelIds: [" "], @@ -1387,7 +1408,7 @@ describe("model-specific-setup/schema.json", () => { ["openclaw-plugins/fixture", "/usr/local/share/nemoclaw/openclaw-plugins/subdir/../escape"], ]) { const bad = { - ...cloneObject(data), + ...cloneObject(exactModelFixture), effects: { openclawPlugins: [ { @@ -1404,7 +1425,7 @@ describe("model-specific-setup/schema.json", () => { it("accepts OpenClaw plugin paths inside the staged plugin trees", () => { const valid = { - ...cloneObject(data), + ...cloneObject(exactModelFixture), effects: { openclawPlugins: [ { From 60f895f2372338be706b49497713e3532238aa8b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 12 Jul 2026 02:10:25 -0700 Subject: [PATCH 02/21] test(e2e): derive inventory and selector contracts (#6710) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary E2E support tests now derive manifest, matrix, and dispatch expectations from the production registries and workflow planner instead of copying the current inventory. Focused malformed-selector, secret-bearing-job, artifact, runner, and unsupported-target behavior remains explicit. ## Related Issue Part of #6708. ## Changes - Remove literal manifest count, target-field, and observability snapshots. - Verify every typed target resolves to a validated manifest path. - Replace the hard-coded live matrix with supported-target, uniqueness, runner-resolution, unsupported-selection, and CLI parity behavior. - Collapse hundreds of repeated selector assertions into registry-derived coverage while retaining invalid and mixed selector cases. - Replace exact workflow target-to-job literals with referential-integrity checks against actual workflow jobs. ## Type of Change - [x] Code change (feature, bug fix, or refactor) - [ ] Code change with doc updates - [ ] Doc only (prose changes, no code sample modifications) - [ ] Doc only (includes code sample changes) ## Quality Gates - [x] Tests added or updated for changed behavior - [ ] Existing tests cover changed behavior — justification: - [ ] Tests not applicable — justification: - [ ] Docs updated for user-facing behavior changes - [x] Docs not applicable — justification: Test methodology changes only; E2E targets, workflows, commands, and runtime behavior are unchanged. - [x] Sensitive paths changed (security, policy, credentials, preflight, onboarding, inference, runner, sandbox, or messaging) - [x] Sensitive-path review completed or maintainer-approved waiver recorded — reviewer/approval link/justification: Test-only cleanup preserves malformed selector rejection, secret-bearing job gating, artifact-path rejection, runner resolution, and unsupported-target evidence. - [ ] Non-success, skipped, or missing CI check accepted by maintainer — check name, approval link, and follow-up issue: ## Verification - [x] PR description includes the DCO sign-off declaration and every commit appears as `Verified` in GitHub - [x] Normal `pre-commit`, `commit-msg`, and `pre-push` hooks passed, or `npm run check:diff` passed when hooks were skipped or unavailable - [x] Targeted behavior tests pass for the current change set, or tests are marked not applicable above — `vitest --project e2e-support`: 3 files and 30 tests passed; `npm run test:titles:check` passed. - [ ] Applicable broad gate passed — `npm test` for broad runtime/test-harness changes; `npm run check` for repo-wide validation/coverage changes — command/result: - [x] Quality Gates section completed with required justifications or waivers - [x] No secrets, API keys, or credentials committed - [ ] `npm run docs` builds without warnings (doc changes only) - [ ] Doc pages follow the [style guide](https://github.com/NVIDIA/NemoClaw/blob/main/docs/CONTRIBUTING.md) (doc changes only) - [ ] New doc pages include SPDX header and frontmatter (new pages only) --- Signed-off-by: Carlos Villela --- test/e2e/support/e2e-manifests.test.ts | 46 +-- test/e2e/support/e2e-matrix.test.ts | 117 ++---- test/e2e/support/e2e-workflow.test.ts | 483 +------------------------ 3 files changed, 58 insertions(+), 588 deletions(-) diff --git a/test/e2e/support/e2e-manifests.test.ts b/test/e2e/support/e2e-manifests.test.ts index 8a2a3bc7198..05bc2407fcf 100644 --- a/test/e2e/support/e2e-manifests.test.ts +++ b/test/e2e/support/e2e-manifests.test.ts @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; import path from "node:path"; +import { describe, expect, it } from "vitest"; -import { loadManifest, loadManifestsFromDir, validateManifest } from "../registry/manifests.ts"; +import { loadManifestsFromDir, validateManifest } from "../registry/manifests.ts"; import { listTargets } from "../registry/registry.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); @@ -12,16 +12,13 @@ const E2E_SUITE_DIR = path.join(REPO_ROOT, "test/e2e"); const MANIFEST_DIR = path.join(E2E_SUITE_DIR, "manifests"); describe("NemoClawInstance manifests", () => { - it("should validate all NemoClaw instance manifests", () => { + it("loads every checked-in instance manifest through validation", () => { const manifests = loadManifestsFromDir(MANIFEST_DIR); - expect(manifests.length).toBeGreaterThanOrEqual(19); - for (const manifest of manifests) { - expect(() => validateManifest(manifest.document, manifest.filePath)).not.toThrow(); - } + expect(manifests).not.toHaveLength(0); }); - it("should reject manifest with assertion or suite IDs", () => { + it("rejects manifest assertion and suite IDs", () => { const badManifest = { apiVersion: "nemoclaw.io/v1", kind: "NemoClawInstance", @@ -39,7 +36,7 @@ describe("NemoClawInstance manifests", () => { ); }); - it("should reject raw secret values in manifest", () => { + it("rejects raw secret values", () => { const badManifest = { apiVersion: "nemoclaw.io/v1", kind: "NemoClawInstance", @@ -56,37 +53,16 @@ describe("NemoClawInstance manifests", () => { ); }); - it("should cover every typed target manifest need", () => { - const manifestNames = new Set( - loadManifestsFromDir(MANIFEST_DIR).map((manifest) => manifest.document.metadata.name), + it("resolves every typed target manifest path to a validated manifest", () => { + const manifestPaths = new Set( + loadManifestsFromDir(MANIFEST_DIR).map((manifest) => path.resolve(manifest.filePath)), ); const missingManifests = listTargets() .map((target) => target.manifestPath) .filter((manifestPath): manifestPath is string => Boolean(manifestPath)) - .map((manifestPath) => path.basename(manifestPath, ".yaml")) - .filter((id) => !manifestNames.has(id)); + .map((manifestPath) => path.resolve(REPO_ROOT, manifestPath)) + .filter((manifestPath) => !manifestPaths.has(manifestPath)); expect(missingManifests, `missing manifest files: ${missingManifests.join(", ")}`).toEqual([]); }); - - it("registry target manifest paths resolve setup and onboarding choices", () => { - const target = listTargets().find((entry) => entry.id === "ubuntu-repo-cloud-openclaw"); - - expect(target).toBeTruthy(); - expect(target!.manifestPath).toBe("test/e2e/manifests/openclaw-nvidia.yaml"); - const manifest = loadManifest(path.join(REPO_ROOT, target!.manifestPath as string)).document; - expect(manifest.spec.setup.install.source).toBe("repo-current"); - expect(manifest.spec.onboarding.agent).toBe("openclaw"); - expect(manifest.spec.onboarding.provider).toBe("nvidia"); - }); - - it("declares observability on the canonical Deep Agents Code live target", () => { - const target = listTargets().find( - (entry) => entry.id === "ubuntu-repo-cloud-langchain-deepagents-code", - ); - - expect(target).toBeTruthy(); - const manifest = loadManifest(path.join(REPO_ROOT, target!.manifestPath as string)).document; - expect(manifest.spec.onboarding.features?.observability).toBe(true); - }); }); diff --git a/test/e2e/support/e2e-matrix.test.ts b/test/e2e/support/e2e-matrix.test.ts index 8ac1eb5435f..9543db98ba6 100644 --- a/test/e2e/support/e2e-matrix.test.ts +++ b/test/e2e/support/e2e-matrix.test.ts @@ -6,8 +6,10 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { target } from "../registry/builder.ts"; +import { listTargets } from "../registry/registry.ts"; import { buildLiveTargetMatrix } from "../registry/run.ts"; import { resolveRunnerForTarget } from "../registry/runner-routing.ts"; +import { liveTargetSupport } from "../registry/runtime-support.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const RUN_TARGETS = path.join(REPO_ROOT, "test/e2e/registry/run.ts"); @@ -21,6 +23,12 @@ function runEmitLiveMatrix(args: string[] = []) { }); } +function requireUnsupportedTarget() { + const unsupported = listTargets().find((entry) => !liveTargetSupport(entry).supported); + expect(unsupported, "expected at least one unsupported live E2E target").toBeDefined(); + return unsupported!; +} + describe("live E2E target matrix", () => { it("honors an explicit runs-on: