From 86a59c2cd185ebab6db967a75b9b5d125680ea0f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 6 Jul 2026 19:36:54 -0400 Subject: [PATCH 1/5] refactor(e2e): add typed target evidence --- test/e2e/fixtures/artifacts.ts | 70 ++++++++++++++++ test/e2e/live/agent-turn-latency.test.ts | 2 +- ...drock-runtime-compatible-anthropic.test.ts | 7 +- test/e2e/live/brave-search.test.ts | 3 +- test/e2e/live/channels-add-remove.test.ts | 3 +- test/e2e/live/channels-stop-start-helpers.ts | 2 +- test/e2e/live/cloud-inference.test.ts | 7 +- test/e2e/live/cloud-onboard.test.ts | 4 +- test/e2e/live/common-egress-agent.test.ts | 12 +-- .../e2e/live/concurrent-gateway-ports.test.ts | 5 +- test/e2e/live/credential-migration.test.ts | 5 +- test/e2e/live/credential-sanitization.test.ts | 3 +- .../cron-preflight-inference-local.test.ts | 3 +- test/e2e/live/dashboard-remote-bind.test.ts | 3 +- test/e2e/live/device-auth-health.test.ts | 3 +- test/e2e/live/diagnostics.test.ts | 5 +- test/e2e/live/docs-validation.test.ts | 3 +- test/e2e/live/double-onboard.test.ts | 5 +- test/e2e/live/full-e2e.test.ts | 4 +- test/e2e/live/gateway-guard-recovery.test.ts | 3 +- test/e2e/live/gateway-health-honest.test.ts | 3 +- test/e2e/live/gpu-double-onboard.test.ts | 4 +- test/e2e/live/gpu-e2e.test.ts | 2 +- test/e2e/live/hermes-discord.test.ts | 4 +- test/e2e/live/hermes-e2e.test.ts | 5 +- test/e2e/live/hermes-gpu-startup.test.ts | 5 +- test/e2e/live/hermes-inference-switch.test.ts | 2 +- .../live/hermes-root-entrypoint-smoke.test.ts | 5 +- .../hermes-sandbox-secret-boundary.test.ts | 5 +- test/e2e/live/hermes-slack-e2e-helpers.ts | 5 +- test/e2e/live/inference-routing.test.ts | 21 ++--- .../issue-2478-crash-loop-recovery.test.ts | 2 +- ...sue-4434-tui-unreachable-inference.test.ts | 5 +- .../issue-4462-scope-upgrade-approval.test.ts | 4 +- test/e2e/live/jetson-nvmap-gpu.test.ts | 2 +- test/e2e/live/kimi-inference-compat.test.ts | 2 +- test/e2e/live/launchable-smoke.test.ts | 3 +- .../messaging-compatible-endpoint.test.ts | 5 +- ...l-router-provider-routed-inference.test.ts | 5 +- test/e2e/live/network-policy.test.ts | 5 +- test/e2e/live/ollama-auth-proxy.test.ts | 3 +- test/e2e/live/onboard-negative-paths.test.ts | 5 +- test/e2e/live/onboard-repair.test.ts | 4 +- .../e2e/live/openclaw-discord-pairing.test.ts | 2 +- .../live/openclaw-inference-switch.test.ts | 11 ++- .../openclaw-plugin-runtime-exdev.test.ts | 5 +- test/e2e/live/openclaw-skill-cli.test.ts | 5 +- test/e2e/live/openclaw-slack-pairing.test.ts | 2 +- .../openclaw-tui-chat-correlation.test.ts | 3 +- test/e2e/live/openshell-version-pin.test.ts | 3 +- test/e2e/live/registry-targets.test.ts | 5 +- test/e2e/live/runtime-overrides.test.ts | 7 +- test/e2e/live/sandbox-operations.test.ts | 5 +- test/e2e/live/sandbox-rlimits-connect.test.ts | 2 +- test/e2e/live/sandbox-survival.test.ts | 5 +- test/e2e/live/sessions-agents-cli.test.ts | 3 +- test/e2e/live/shields-config.test.ts | 5 +- test/e2e/live/skill-agent.test.ts | 9 +- test/e2e/live/snapshot-commands.test.ts | 5 +- test/e2e/live/spark-install.test.ts | 2 +- test/e2e/live/state-backup-restore.test.ts | 4 +- test/e2e/live/telegram-injection.test.ts | 2 +- test/e2e/live/token-rotation.test.ts | 5 +- test/e2e/live/ubuntu-repo-cli-smoke.test.ts | 3 +- test/e2e/live/upgrade-stale-sandbox.test.ts | 3 +- test/e2e/support/e2e-target-evidence.test.ts | 84 +++++++++++++++++++ 66 files changed, 271 insertions(+), 167 deletions(-) create mode 100644 test/e2e/support/e2e-target-evidence.test.ts diff --git a/test/e2e/fixtures/artifacts.ts b/test/e2e/fixtures/artifacts.ts index 680f355a014..863d0665b11 100644 --- a/test/e2e/fixtures/artifacts.ts +++ b/test/e2e/fixtures/artifacts.ts @@ -6,6 +6,74 @@ import path from "node:path"; import { redactString } from "./redaction.ts"; +export type TargetContract = string | readonly string[]; + +export type TargetMetadata> = { + id: string; + contract?: TargetContract; + contracts?: readonly string[]; +} & Extension; + +export type TargetResult> = { + id: string; + status?: string; +} & Extension; + +type TargetEvidenceKind = "metadata" | "result"; + +function normalizeTargetEvidence( + kind: TargetEvidenceKind, + value: TargetMetadata | TargetResult, +): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError(`target ${kind} must be an object`); + } + if (typeof value.id !== "string" || value.id.trim() === "") { + throw new TypeError(`target ${kind} id must be a non-empty string`); + } + if ( + kind === "result" && + "status" in value && + (typeof value.status !== "string" || value.status.trim() === "") + ) { + throw new TypeError("target result status must be a non-empty string"); + } + + const record = { ...value } as Record; + const singular = record.contract; + const plural = record.contracts; + if (singular !== undefined && plural !== undefined) { + throw new TypeError("target metadata must use either contract or contracts, not both"); + } + const contracts = singular ?? plural; + if (contracts !== undefined) { + const normalized = typeof contracts === "string" ? [contracts] : contracts; + if (!Array.isArray(normalized) || normalized.some((contract) => typeof contract !== "string")) { + throw new TypeError("target contracts must be a string or an array of strings"); + } + record.contracts = normalized; + } + delete record.contract; + if (kind === "result") record.status ??= "passed"; + record.runner = "vitest"; + return record; +} + +export class TargetEvidenceWriter { + constructor(private readonly artifacts: ArtifactSink) {} + + async declare(metadata: TargetMetadata): Promise { + return this.artifacts.writeJson("target.json", normalizeTargetEvidence("metadata", metadata)); + } + + async complete(result: TargetResult): Promise { + return this.artifacts.writeJson( + "target-result.json", + normalizeTargetEvidence("result", result), + ); + } +} + /** * The publication boundary for live E2E evidence. * @@ -15,10 +83,12 @@ import { redactString } from "./redaction.ts"; */ export class ArtifactSink { readonly rootDir: string; + readonly target: TargetEvidenceWriter; private readonly redactionValues = new Set(); constructor(rootDir: string, redactionValues: Iterable = []) { this.rootDir = path.resolve(rootDir); + this.target = new TargetEvidenceWriter(this); this.addRedactionValues(redactionValues); } diff --git a/test/e2e/live/agent-turn-latency.test.ts b/test/e2e/live/agent-turn-latency.test.ts index 46f124ee6ea..ede6c2a5793 100644 --- a/test/e2e/live/agent-turn-latency.test.ts +++ b/test/e2e/live/agent-turn-latency.test.ts @@ -40,7 +40,7 @@ test.skipIf(!shouldRunLiveE2E())( async ({ artifacts, cleanup, host, sandbox, secrets }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const results: Record = { model: MODEL, maxTurnSeconds: MAX_TURN_SECONDS }; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "agent-turn-latency", boundary: "two real sandboxes + hosted inference + OpenClaw agent turn + Hermes API turn", openclawSandbox: OPENCLAW_SANDBOX, diff --git a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts index 1f53550ef4c..858407afe50 100644 --- a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts +++ b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts @@ -1149,7 +1149,7 @@ async function skipPreContractEndpointValidationRateLimit(options: { redactedStdoutTail: evidenceTail(options.onboarding.redactedStdout), redactedStderrTail: evidenceTail(options.onboarding.redactedStderr), }); - await options.artifacts.writeJson("target-result.json", { + await options.artifacts.target.complete({ id: "bedrock-runtime-compatible-anthropic", status: "skipped", reason: BEDROCK_PRE_CONTRACT_ENDPOINT_VALIDATION_SKIP_REASON, @@ -1253,9 +1253,8 @@ RUN_BEDROCK_TEST( } }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "bedrock-runtime-compatible-anthropic", - runner: "vitest", refs: ["#3767", "#5098"], agent: AGENT, sandboxName: SANDBOX_NAME, @@ -1364,7 +1363,7 @@ RUN_BEDROCK_TEST( redact: (text, extraValues) => secrets.redact(text, extraValues), }); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "bedrock-runtime-compatible-anthropic", agent: AGENT, assertions: { diff --git a/test/e2e/live/brave-search.test.ts b/test/e2e/live/brave-search.test.ts index 060363fe35f..651ad88c4fc 100644 --- a/test/e2e/live/brave-search.test.ts +++ b/test/e2e/live/brave-search.test.ts @@ -30,9 +30,8 @@ test.skipIf(!shouldRunLiveE2E())( const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const redactionValues = [braveKey, inferenceKey]; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "brave-search", - runner: "vitest", boundary: "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", sandboxName: SANDBOX_NAME, diff --git a/test/e2e/live/channels-add-remove.test.ts b/test/e2e/live/channels-add-remove.test.ts index ff99bd54774..b61eec650cf 100644 --- a/test/e2e/live/channels-add-remove.test.ts +++ b/test/e2e/live/channels-add-remove.test.ts @@ -398,9 +398,8 @@ liveTest( onboarding: "cloud-openclaw", }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "channels-add-remove", - runner: "vitest", sandboxName: SANDBOX_NAME, contract: [ "onboard creates an OpenClaw sandbox with no Telegram channel", diff --git a/test/e2e/live/channels-stop-start-helpers.ts b/test/e2e/live/channels-stop-start-helpers.ts index a7737eb20eb..abb593abd38 100644 --- a/test/e2e/live/channels-stop-start-helpers.ts +++ b/test/e2e/live/channels-stop-start-helpers.ts @@ -429,7 +429,7 @@ export async function runChannelsStopStartTarget({ }); const redactions = redactionValues(apiKey, tokens); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "channels-stop-start", boundary: "install.sh messaging onboard + channels stop/start CLI + rebuild + sandbox config probes", diff --git a/test/e2e/live/cloud-inference.test.ts b/test/e2e/live/cloud-inference.test.ts index 5f2574f0653..2bf15fc34ed 100644 --- a/test/e2e/live/cloud-inference.test.ts +++ b/test/e2e/live/cloud-inference.test.ts @@ -82,7 +82,7 @@ async function writePreContractExternalProviderSkip( ): Promise { const evidence = buildPreContractExternalProviderSkipEvidence(install, classification); await artifacts.writeJson("transient-provider-validation.skip.json", evidence); - await artifacts.writeJson("target-result.json", evidence); + await artifacts.target.complete(evidence); } function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { @@ -243,9 +243,8 @@ test.skipIf(!shouldRunLiveE2E())( `missing sandbox skill validator: ${SANDBOX_SKILL_VALIDATOR}`, ).toBe(true); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "cloud-inference", - runner: "vitest", boundary: "install-sh-onboard-sandbox-inference-local-skill-filesystem", contracts: [ "Docker is running before install/onboard", @@ -339,7 +338,7 @@ test.skipIf(!shouldRunLiveE2E())( : "unknown"; expect(sandboxSkillStatus, resultText(sandboxSkills)).not.toBe("unknown"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "cloud-inference", status: "passed", assertions: { diff --git a/test/e2e/live/cloud-onboard.test.ts b/test/e2e/live/cloud-onboard.test.ts index c342caa6d79..552084ab911 100644 --- a/test/e2e/live/cloud-onboard.test.ts +++ b/test/e2e/live/cloud-onboard.test.ts @@ -84,7 +84,7 @@ liveTest( const installCwd = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-public-install-")); const redactionValues = [hosted.apiKey]; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "cloud-onboard", sandboxName: SANDBOX_NAME, installUrl, @@ -176,6 +176,6 @@ liveTest( } await cleanup(host, sandbox, { label: "final-cleanup", verify: true }); - await artifacts.writeJson("target-result.json", { id: "cloud-onboard", status: "passed" }); + await artifacts.target.complete({ id: "cloud-onboard", status: "passed" }); }, ); diff --git a/test/e2e/live/common-egress-agent.test.ts b/test/e2e/live/common-egress-agent.test.ts index 5e3da826d74..01f47685ea3 100644 --- a/test/e2e/live/common-egress-agent.test.ts +++ b/test/e2e/live/common-egress-agent.test.ts @@ -574,7 +574,7 @@ describe.sequential("common-egress agent live targets", () => { const hosted = await assertPrerequisites(host, secrets, skip); const apiKey = hosted.apiKey; const braveApiKey = secrets.required("BRAVE_API_KEY"); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "common-egress-agent", case: "openclaw-balanced-weather", sandboxName: OPENCLAW_BALANCED_SANDBOX, @@ -687,7 +687,7 @@ After it returns, reply with only WEATHER_AGENT_OK. Do not fetch any other URL.` ); expect(weatherProof.exitCode, text(weatherProof)).toBe(0); expect(weatherProof.stdout.trim()).toMatch(/^[a-f0-9]{64}\s+/); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "common-egress-agent", case: "openclaw-balanced-weather", status: "passed", @@ -701,7 +701,7 @@ After it returns, reply with only WEATHER_AGENT_OK. Do not fetch any other URL.` async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { const hosted = await assertPrerequisites(host, secrets, skip); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "common-egress-agent", case: "openclaw-open-public-reference", sandboxName: OPENCLAW_OPEN_SANDBOX, @@ -733,7 +733,7 @@ After it returns, reply with only WEATHER_AGENT_OK. Do not fetch any other URL.` https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q30&props=labels&languages=en&format=json After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched response says entity Q30 has the English label United States. Do not fetch any other URL.`, }); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "common-egress-agent", case: "openclaw-open-public-reference", status: "passed", @@ -747,7 +747,7 @@ After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched respons async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { const hosted = await assertPrerequisites(host, secrets, skip); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "common-egress-agent", case: "hermes-open-public-reference", sandboxName: HERMES_SANDBOX, @@ -783,7 +783,7 @@ After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched respons prompt: buildHermesReferencePrompt(), sandboxName: HERMES_SANDBOX, }); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "common-egress-agent", case: "hermes-open-public-reference", status: "passed", diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index 1003644e22c..02e997bbf7d 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -297,9 +297,8 @@ liveTest( const fake = await startFakeOpenAiCompatibleServer({ port: Number(process.env.NEMOCLAW_E2E_FAKE_PORT ?? 0), }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "concurrent-gateway-ports", - runner: "vitest", boundary: "direct-cli-docker-openshell-multiple-gateways-dashboard-forwards", contract: [ "sandbox A onboards on the default NemoClaw gateway and dashboard port", @@ -401,7 +400,7 @@ liveTest( expect(["Ready", "Running"]).toContain(phaseAAfterDestroyB); await expectPortListening(host, GATEWAY_PORT_A, "phase-4-gateway-port-a-still-listening"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "concurrent-gateway-ports", assertions: { sandboxAOnboarded: onboardA.exitCode === 0, diff --git a/test/e2e/live/credential-migration.test.ts b/test/e2e/live/credential-migration.test.ts index aa564bafeff..8cef42cafd0 100644 --- a/test/e2e/live/credential-migration.test.ts +++ b/test/e2e/live/credential-migration.test.ts @@ -181,9 +181,8 @@ runCredentialMigrationTest( fs.rmSync(home, { recursive: true, force: true }); }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "credential-migration", - runner: "vitest", boundary: "real-onboard-openshell-gateway", sandboxName: SANDBOX_NAME, contracts: [ @@ -294,7 +293,7 @@ runCredentialMigrationTest( expect(fs.existsSync(victimFile), "symlink target must remain present").toBe(true); expect(fs.readFileSync(victimFile, "utf-8")).toBe(victimPayload); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "credential-migration", sandboxName: SANDBOX_NAME, model: hostedInference.model || CREDENTIAL_MIGRATION_MODEL, diff --git a/test/e2e/live/credential-sanitization.test.ts b/test/e2e/live/credential-sanitization.test.ts index ceca327dee9..4c1135ec221 100644 --- a/test/e2e/live/credential-sanitization.test.ts +++ b/test/e2e/live/credential-sanitization.test.ts @@ -297,9 +297,8 @@ runCredentialSanitizationTest( "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "credential-sanitization", - runner: "vitest", boundary: "install-sh-onboard-and-sandbox-exec", sandboxName: SANDBOX_NAME, contracts: [ diff --git a/test/e2e/live/cron-preflight-inference-local.test.ts b/test/e2e/live/cron-preflight-inference-local.test.ts index dd299adf4cf..d4cfe9e6eb5 100644 --- a/test/e2e/live/cron-preflight-inference-local.test.ts +++ b/test/e2e/live/cron-preflight-inference-local.test.ts @@ -212,9 +212,8 @@ test.skipIf(!shouldRunLiveE2E())( const hosted = requireHostedInferenceConfig(secrets, process.env, { model: MODEL }); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "cron-preflight-inference-local", - runner: "vitest", boundary: "install.sh + in-sandbox OpenClaw cron preflight runtime helper", sandboxName: SANDBOX_NAME, model: MODEL, diff --git a/test/e2e/live/dashboard-remote-bind.test.ts b/test/e2e/live/dashboard-remote-bind.test.ts index c43beab9ece..fcaf782a4c1 100644 --- a/test/e2e/live/dashboard-remote-bind.test.ts +++ b/test/e2e/live/dashboard-remote-bind.test.ts @@ -53,9 +53,8 @@ runDashboardRemoteBindTest( const dashboardPort = process.env.NEMOCLAW_DASHBOARD_PORT || "18789"; const remoteHost = remoteHostCandidate(); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "dashboard-remote-bind", - runner: "vitest", boundary: "remote-dashboard-forward", optIn: "NEMOCLAW_E2E_DASHBOARD_REMOTE_BIND=1", sandboxName, diff --git a/test/e2e/live/device-auth-health.test.ts b/test/e2e/live/device-auth-health.test.ts index e5596b02783..01d75861cd7 100644 --- a/test/e2e/live/device-auth-health.test.ts +++ b/test/e2e/live/device-auth-health.test.ts @@ -57,9 +57,8 @@ test.skipIf(!shouldRunLiveE2E())( model: INFERENCE_MODEL, }; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "device-auth-health", - runner: "vitest", boundary: "install.sh + OpenShell sandbox exec + NemoClaw status + host curl", sandboxName: SANDBOX_NAME, dashboardPort: DASHBOARD_PORT, diff --git a/test/e2e/live/diagnostics.test.ts b/test/e2e/live/diagnostics.test.ts index 041b87f7ed1..5fe002310ab 100644 --- a/test/e2e/live/diagnostics.test.ts +++ b/test/e2e/live/diagnostics.test.ts @@ -137,9 +137,8 @@ runDiagnosticsTest( const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "diagnostics", - runner: "vitest", boundary: "debug-archive-install-sh-docker-openshell-sandbox-exec-credentials", sandboxName: SANDBOX_NAME, contracts: [ @@ -413,7 +412,7 @@ runDiagnosticsTest( }); } - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "diagnostics", sandboxName: SANDBOX_NAME, model: hosted.model, diff --git a/test/e2e/live/docs-validation.test.ts b/test/e2e/live/docs-validation.test.ts index 3a10055603b..cfb40ab5683 100644 --- a/test/e2e/live/docs-validation.test.ts +++ b/test/e2e/live/docs-validation.test.ts @@ -71,9 +71,8 @@ runDocsValidationTest( "docs validation matches CLI help and local documentation links", { timeout: BUILD_TIMEOUT_MS + DOCS_CHECK_TIMEOUT_MS * 2 }, async ({ artifacts, host }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "docs-validation", - runner: "vitest", boundary: "checkout-local-docs-checks", phases: ["cli-docs-parity", "local-markdown-links"], }); diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index 89f3c8249e0..2cc6a130f1f 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -483,9 +483,8 @@ liveTest( await cleanupDoubleOnboardState(host, sandbox); }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "double-onboard", - runner: "vitest", boundary: "direct-cli-openshell-lifecycle", contract: [ "first onboard creates a sandbox and NemoClaw gateway", @@ -750,7 +749,7 @@ liveTest( "registry still contains test entries", ).toBe(false); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "double-onboard", fakeOpenAiRequests: fake.requests(), assertions: { diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 1663b4ccb6d..aa74514c281 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -229,7 +229,7 @@ liveTest( async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, secrets, skip }) => { const hosted = requireHostedInferenceConfig(secrets); const redactionValues = [hosted.apiKey]; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "full-e2e", sandboxName: SANDBOX_NAME, endpointUrl: hosted.endpointUrl, @@ -384,7 +384,7 @@ liveTest( const registryText = fs.existsSync(registry) ? fs.readFileSync(registry, "utf8") : ""; expect(registryText).not.toContain(SANDBOX_NAME); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "full-e2e", securityPosture, status: "passed", diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index 8eb3c94fb3a..03ae0b1eba3 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -72,9 +72,8 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) }) => { secrets.required("NVIDIA_INFERENCE_API_KEY"); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "gateway-guard-recovery", - runner: "vitest", boundary: "sandbox-lifecycle", issues: ["#2701", "#2478"], acceptanceCoverage: { diff --git a/test/e2e/live/gateway-health-honest.test.ts b/test/e2e/live/gateway-health-honest.test.ts index a0f23770f02..f3f0b6adb97 100644 --- a/test/e2e/live/gateway-health-honest.test.ts +++ b/test/e2e/live/gateway-health-honest.test.ts @@ -41,9 +41,8 @@ test.skipIf(!shouldRunLiveE2E())( const gatewayLog = path.join(stateDir, "openshell-gateway.log"); const gatewayPidFile = path.join(stateDir, "openshell-gateway.pid"); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "gateway-health-honest", - runner: "vitest", boundary: "real-startGateway-openshell-docker-driver-process", contracts: [ "startGateway() invokes a real OpenShell Docker-driver gateway child process", diff --git a/test/e2e/live/gpu-double-onboard.test.ts b/test/e2e/live/gpu-double-onboard.test.ts index 49e122f02f6..dc466c70eb0 100644 --- a/test/e2e/live/gpu-double-onboard.test.ts +++ b/test/e2e/live/gpu-double-onboard.test.ts @@ -151,7 +151,7 @@ liveTest( "gpu double onboard keeps Ollama auth proxy token consistent after re-onboard", { timeout: LIVE_TIMEOUT_MS }, async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "gpu-double-onboard", sandboxName: SANDBOX_NAME, proxyPort: PROXY_PORT, @@ -287,7 +287,7 @@ liveTest( const registryFile = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); const registryText = fs.existsSync(registryFile) ? fs.readFileSync(registryFile, "utf8") : ""; expect(registryText).not.toContain(SANDBOX_NAME); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "gpu-double-onboard", status: "passed", }); diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index a5308ff49d5..b9eff25280f 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -94,7 +94,7 @@ test.skipIf(!shouldRunLiveE2E())( "GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { timeout: TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "gpu-e2e", boundary: "GPU host + install.sh Ollama provider + OpenShell sandbox + auth proxy + inference.local", diff --git a/test/e2e/live/hermes-discord.test.ts b/test/e2e/live/hermes-discord.test.ts index 09d71ae9118..d4f7809bfa4 100644 --- a/test/e2e/live/hermes-discord.test.ts +++ b/test/e2e/live/hermes-discord.test.ts @@ -363,7 +363,7 @@ test.skipIf(!shouldRunLiveE2E())( const env = commandEnv(apiKey); const redactionValues = redactions(apiKey); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "hermes-discord", boundary: "install.sh --non-interactive Hermes sandbox + Discord config + OpenShell provider rewrite + sandbox leak probes + rebuild credential reuse", @@ -749,7 +749,7 @@ done`, expect(registryProbe.stdout.trim()).toBe("ABSENT"); })(); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "hermes-discord", assertions: { dockerAndNonInteractivePrereqs: true, diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index 0463235494b..0efb26cfbe0 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -244,9 +244,8 @@ test.skipIf(!shouldRunLiveE2E())( const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "hermes-e2e", - runner: "vitest", boundary: "install.sh --non-interactive --fresh + Hermes sandbox runtime", sandboxName: SANDBOX_NAME, dashboardEnabled: hermesDashboardE2eEnabled(), @@ -1394,7 +1393,7 @@ test.skipIf(!shouldRunLiveE2E())( ).toBeUndefined(); } - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "hermes-e2e", assertions: { installShNonInteractiveHermes: true, diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index b8c43d8bb69..78c3547d36a 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -175,9 +175,8 @@ test.skipIf(!shouldRunLiveE2E())( "hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready state", { timeout: LIVE_TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "hermes-gpu-startup", - runner: "vitest", boundary: "install.sh --non-interactive --fresh + Hermes GPU-supervised startup", sandboxName: SANDBOX_NAME, inference: "hermetic fake OpenAI-compatible endpoint", @@ -297,7 +296,7 @@ test.skipIf(!shouldRunLiveE2E())( expect(JSON.stringify(fakeRequests)).not.toContain(EXTRA_PLACEHOLDER_TOKEN_A); expect(JSON.stringify(fakeRequests)).not.toContain(EXTRA_PLACEHOLDER_TOKEN_B); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "hermes-gpu-startup", assertions: { selectedGpuRouteVerified: true, diff --git a/test/e2e/live/hermes-inference-switch.test.ts b/test/e2e/live/hermes-inference-switch.test.ts index 8a664b2c5fd..cfbc7fee080 100644 --- a/test/e2e/live/hermes-inference-switch.test.ts +++ b/test/e2e/live/hermes-inference-switch.test.ts @@ -82,7 +82,7 @@ test.skipIf(!shouldRunLiveE2E())( "Hermes inference set updates route/config and preserves live runtime", { timeout: TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, secrets }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "hermes-inference-switch", boundary: "install.sh + Hermes sandbox + inference set + in-sandbox health/chat + hermes -z probes", diff --git a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts index 8f12a02add5..601cb43688a 100644 --- a/test/e2e/live/hermes-root-entrypoint-smoke.test.ts +++ b/test/e2e/live/hermes-root-entrypoint-smoke.test.ts @@ -418,9 +418,8 @@ liveTest( const baseImage = `nemoclaw-hermes-sandbox-base-local:root-entrypoint-${runId}`; const containers: string[] = []; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "hermes-root-entrypoint-smoke", - runner: "vitest", boundary: "docker-root-entrypoint", image, prebuiltImage: Boolean(process.env.NEMOCLAW_HERMES_TEST_IMAGE), @@ -461,7 +460,7 @@ liveTest( throw error; } - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "hermes-root-entrypoint-smoke", image, assertions: { diff --git a/test/e2e/live/hermes-sandbox-secret-boundary.test.ts b/test/e2e/live/hermes-sandbox-secret-boundary.test.ts index 1f261e44115..b6d2a106ea5 100644 --- a/test/e2e/live/hermes-sandbox-secret-boundary.test.ts +++ b/test/e2e/live/hermes-sandbox-secret-boundary.test.ts @@ -747,9 +747,8 @@ liveTest( let removeManagedImage = false; let removeBaseImage = false; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "hermes-sandbox-secret-boundary", - runner: "vitest", boundary: "docker-hermes-image-and-startup", image, baseImage, @@ -843,7 +842,7 @@ liveTest( RAW_REFRESH_TOKEN, ); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "hermes-sandbox-secret-boundary", image, managedImage, diff --git a/test/e2e/live/hermes-slack-e2e-helpers.ts b/test/e2e/live/hermes-slack-e2e-helpers.ts index b857747d5d7..2fbc5db158b 100644 --- a/test/e2e/live/hermes-slack-e2e-helpers.ts +++ b/test/e2e/live/hermes-slack-e2e-helpers.ts @@ -221,9 +221,8 @@ export async function runHermesSlackE2E({ await cleanupHermesSlack({ host, apiKey, artifactPrefix: "cleanup-hermes-slack" }); }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "hermes-slack-e2e", - runner: "vitest", boundary: "bash install.sh --non-interactive + Hermes Slack sandbox runtime", sandboxName: SANDBOX_NAME, providerNames: [`${SANDBOX_NAME}-slack-bridge`, `${SANDBOX_NAME}-slack-app`], @@ -638,7 +637,7 @@ PY`, } } - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "hermes-slack-e2e", assertions: { installerAndCliAvailable: true, diff --git a/test/e2e/live/inference-routing.test.ts b/test/e2e/live/inference-routing.test.ts index a68589dea58..3bafcc51355 100644 --- a/test/e2e/live/inference-routing.test.ts +++ b/test/e2e/live/inference-routing.test.ts @@ -560,9 +560,8 @@ liveTest( ); await cleanupSandbox(host, sandbox, sandboxName); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "inference-routing-invalid-api-key", - runner: "vitest", contract: [ "invalid NVIDIA key exits non-zero", "output contains credential classification", @@ -602,9 +601,8 @@ liveTest( ); await cleanupSandbox(host, sandbox, sandboxName); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "inference-routing-unreachable-endpoint", - runner: "vitest", contract: [ "unreachable custom endpoint exits non-zero", "output contains transport classification", @@ -674,9 +672,8 @@ liveTest( "", ].join("\n"), ); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "https-dns-backed-endpoint-fail-closed", - runner: "vitest", issue: 4684, contract: [ "DNS-backed HTTPS endpoint validation fails closed before handing config to OpenShell", @@ -743,9 +740,8 @@ liveTest( ); await cleanupSandbox(host, sandbox, sandboxName); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "inference-routing-credential-isolation", - runner: "vitest", contract: [ "real NVIDIA_INFERENCE_API_KEY does not appear in sandbox environment", "real NVIDIA_INFERENCE_API_KEY does not appear in sandbox process list when ps is available", @@ -925,9 +921,8 @@ liveTest( ); await cleanupSandbox(host, sandbox, sandboxName); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "inference-routing-openai", - runner: "vitest", contract: ["OpenAI provider onboards", "sandbox inference.local routes chat to OpenAI"], model, }); @@ -973,9 +968,8 @@ liveTest( ); await cleanupSandbox(host, sandbox, sandboxName); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "inference-routing-anthropic", - runner: "vitest", contract: [ "Anthropic provider onboards", "sandbox inference.local routes Messages API to Anthropic", @@ -1026,9 +1020,8 @@ liveTest( ); await cleanupSandbox(host, sandbox, sandboxName); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "inference-routing-compatible-endpoint", - runner: "vitest", contract: [ "custom OpenAI-compatible endpoint onboards", "sandbox inference.local routes chat to compatible endpoint", diff --git a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts index 9e19876a7e4..ee25d330a4a 100644 --- a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts +++ b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts @@ -431,7 +431,7 @@ test("issue-2478: gateway recovery preserves guard chain and avoids crash loop", runtime, sandbox, }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "issue-2478-crash-loop-recovery", issues: ["#2478", "#2701"], crashCycles: CRASH_CYCLES, diff --git a/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts b/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts index 941bfad51a1..bd1e09f29f5 100644 --- a/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts +++ b/test/e2e/live/issue-4434-tui-unreachable-inference.test.ts @@ -160,9 +160,8 @@ runIssue4434LiveTest( const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "issue-4434-tui-unreachable-inference", - runner: "vitest", boundary: [ "real cloud OpenClaw sandbox", "host DOCKER-USER iptables DROP rules", @@ -517,7 +516,7 @@ runIssue4434LiveTest( fs.writeFileSync(captureFile, redactedRawCapture, "utf8"); const analysis = analyzeIssue4434TuiCapture(redactedRawCapture); await artifacts.writeText("openclaw-tui-capture.plain.log", analysis.plain); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "issue-4434-tui-unreachable-inference", expectExitCode: tui.exitCode, visibleError: analysis.visibleError, diff --git a/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts b/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts index 7a1fa89e5ad..a3c4ff817ee 100644 --- a/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts +++ b/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts @@ -1217,7 +1217,7 @@ liveTest( { timeout: LIVE_TIMEOUT_MS }, async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, secrets, skip }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "issue-4462-scope-upgrade-approval", sandboxName: SANDBOX_NAME, contracts: [ @@ -1476,7 +1476,7 @@ liveTest( expect(adminConnectOutput).toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); await cleanup(host, sandbox); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "issue-4462-scope-upgrade-approval", status: "passed", }); diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index daa0980b55b..eec458fcfe8 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -70,7 +70,7 @@ liveTest( "Jetson nvmap GPU onboard grants device-node group and reports verified CUDA", { timeout: TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "jetson-nvmap-gpu", issue: 4231, boundary: diff --git a/test/e2e/live/kimi-inference-compat.test.ts b/test/e2e/live/kimi-inference-compat.test.ts index ff1af187a1e..e7ee1a1419f 100644 --- a/test/e2e/live/kimi-inference-compat.test.ts +++ b/test/e2e/live/kimi-inference-compat.test.ts @@ -40,7 +40,7 @@ test.skipIf(!shouldRunLiveE2E())( maybeRegisterKimiMockCleanup(cleanup, fake); cleanup.add("destroy Kimi sandbox", () => cleanupKimi(host, sandbox)); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "kimi-inference-compat", boundary: kimiBoundary(mode), inferenceClassification: "public-nvidia required with mock/hermetic fallback", diff --git a/test/e2e/live/launchable-smoke.test.ts b/test/e2e/live/launchable-smoke.test.ts index 885dc7db89f..3107c9acb9f 100644 --- a/test/e2e/live/launchable-smoke.test.ts +++ b/test/e2e/live/launchable-smoke.test.ts @@ -222,9 +222,8 @@ runLaunchableSmokeTest( async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { validateSandboxName(SANDBOX_NAME); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "launchable-smoke", - runner: "vitest", boundary: "ubuntu-launchable-install-flow", refs: ["#2599", "#5098"], phases: [ diff --git a/test/e2e/live/messaging-compatible-endpoint.test.ts b/test/e2e/live/messaging-compatible-endpoint.test.ts index f81dfc1b664..5f932afd5e2 100644 --- a/test/e2e/live/messaging-compatible-endpoint.test.ts +++ b/test/e2e/live/messaging-compatible-endpoint.test.ts @@ -624,9 +624,8 @@ liveTest( skip("Docker is required for messaging compatible endpoint E2E"); } - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "messaging-compatible-endpoint", - runner: "vitest", boundary: "direct-cli-onboard-openshell-compatible-endpoint", refs: ["#2766", "#2572", "#5098"], contract: [ @@ -704,7 +703,7 @@ liveTest( : "Live Telegram-compatible round trip secrets not fully set", }); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "messaging-compatible-endpoint", runner, endpointUrl, diff --git a/test/e2e/live/model-router-provider-routed-inference.test.ts b/test/e2e/live/model-router-provider-routed-inference.test.ts index d49f058bd5c..3b8ad6606f4 100644 --- a/test/e2e/live/model-router-provider-routed-inference.test.ts +++ b/test/e2e/live/model-router-provider-routed-inference.test.ts @@ -98,9 +98,8 @@ test.skipIf(!shouldRunLiveE2E())( const apiKey = requireModelRouterPublicKey(secrets); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "model-router-provider-routed-inference", - runner: "vitest", boundary: "direct-cli-onboard-and-sandbox-exec", contract: [ "Docker is available before onboarding", @@ -209,7 +208,7 @@ test.skipIf(!shouldRunLiveE2E())( `Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure: ${lastCompletion.slice(0, 500)}`, ).toBe("ok"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "model-router-provider-routed-inference", assertions: { dockerRunning: docker.exitCode === 0, diff --git a/test/e2e/live/network-policy.test.ts b/test/e2e/live/network-policy.test.ts index ef4bb17e019..4e748a7b910 100644 --- a/test/e2e/live/network-policy.test.ts +++ b/test/e2e/live/network-policy.test.ts @@ -426,9 +426,8 @@ RUN_NETWORK_POLICY_TEST( "network-policy: restricted sandbox enforces live allow/deny policy probes", { timeout: TEST_TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "network-policy", - runner: "vitest", boundary: "live-sandbox-network-policy", contracts: [ "deny-by-default egress", @@ -933,7 +932,7 @@ nemoclaw-start node /tmp/nemoclaw-web-fetch-e2e.mjs 'http://host.openshell.inter }); expect(text(npmPing)).toContain("NPM_OK"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "network-policy", sandboxName: SANDBOX_NAME, assertions: { diff --git a/test/e2e/live/ollama-auth-proxy.test.ts b/test/e2e/live/ollama-auth-proxy.test.ts index e22360eea4e..af258c9de25 100644 --- a/test/e2e/live/ollama-auth-proxy.test.ts +++ b/test/e2e/live/ollama-auth-proxy.test.ts @@ -166,9 +166,8 @@ test.skipIf(!shouldRunLiveE2E())( "Ollama auth proxy enforces tokens, proxies inference, persists tokens, and recovers", { timeout: LIVE_TIMEOUT_MS }, async ({ artifacts, cleanup, host }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "ollama-auth-proxy", - runner: "vitest", boundary: "real host Ollama + real Node auth proxy + curl + optional Docker reachability", ollamaPort: OLLAMA_PORT, proxyPort: PROXY_PORT, diff --git a/test/e2e/live/onboard-negative-paths.test.ts b/test/e2e/live/onboard-negative-paths.test.ts index b16f106fc58..87101cd5609 100644 --- a/test/e2e/live/onboard-negative-paths.test.ts +++ b/test/e2e/live/onboard-negative-paths.test.ts @@ -101,9 +101,8 @@ liveTest( }); await cleanupInvalidKeyState(host, sandboxName); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "onboard-invalid-nvidia-key", - runner: "vitest", boundary: "direct-cli-onboard", contract: [ "invalid NVIDIA key exits non-zero", @@ -134,7 +133,7 @@ liveTest( expect(text).toContain("Must start with nvapi-"); expect(hasStackTrace(text), text).toBe(false); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "onboard-invalid-nvidia-key", exitCode: result.exitCode, assertions: { diff --git a/test/e2e/live/onboard-repair.test.ts b/test/e2e/live/onboard-repair.test.ts index feb4e3165c1..7977771a8e0 100644 --- a/test/e2e/live/onboard-repair.test.ts +++ b/test/e2e/live/onboard-repair.test.ts @@ -112,7 +112,7 @@ liveTest( "onboard repair resumes missing sandbox and rejects conflicting resume inputs", { timeout: LIVE_TIMEOUT_MS }, async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "onboard-repair", sandboxName: SANDBOX_NAME, otherSandboxName: OTHER_SANDBOX_NAME, @@ -228,6 +228,6 @@ liveTest( await cleanup(host, sandbox); expect(fs.existsSync(SESSION_FILE)).toBe(false); - await artifacts.writeJson("target-result.json", { id: "onboard-repair", status: "passed" }); + await artifacts.target.complete({ id: "onboard-repair", status: "passed" }); }, ); diff --git a/test/e2e/live/openclaw-discord-pairing.test.ts b/test/e2e/live/openclaw-discord-pairing.test.ts index ec654d8be59..813a12a186e 100644 --- a/test/e2e/live/openclaw-discord-pairing.test.ts +++ b/test/e2e/live/openclaw-discord-pairing.test.ts @@ -46,7 +46,7 @@ test.skipIf(!shouldRunLiveE2E())( }); const redactions = pairingRedactions({ apiKey, discordToken: DISCORD_TOKEN }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "openclaw-discord-pairing", boundary: "install.sh Discord OpenClaw sandbox + fake Discord Gateway token rewrite + runtime pairing request + connect-shell approval", diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index e4b4e23488e..1a0382d928c 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -901,9 +901,8 @@ RUN_OPENCLAW_INFERENCE_SWITCH_TEST( "openclaw-inference-switch: switches route and preserves live OpenClaw behavior", { timeout: TEST_TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "openclaw-inference-switch", - runner: "vitest", boundary: "install-sh-openclaw-inference-set-and-live-agent-turn", sandboxName: SANDBOX_NAME, switchProvider: SWITCH_PROVIDER, @@ -989,7 +988,7 @@ RUN_OPENCLAW_INFERENCE_SWITCH_TEST( ); const installText = resultText(install); if (install.exitCode !== 0 && isExternalProviderValidationFailure(installText)) { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "openclaw-inference-switch", status: "skipped", reason: "external-provider-validation-unavailable-before-inference-switch", @@ -1056,7 +1055,7 @@ RUN_OPENCLAW_INFERENCE_SWITCH_TEST( const inference = await checkSandboxInference(sandbox, home); if (inference !== "ok") { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "openclaw-inference-switch", status: "skipped", reason: inference.skipped, @@ -1067,7 +1066,7 @@ RUN_OPENCLAW_INFERENCE_SWITCH_TEST( const agentTurn = await checkOpenClawAgentTurn(host, home); if (agentTurn !== "ok") { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "openclaw-inference-switch", status: "skipped", reason: agentTurn.skipped, @@ -1083,7 +1082,7 @@ RUN_OPENCLAW_INFERENCE_SWITCH_TEST( expect(registryText).not.toContain(`"${SANDBOX_NAME}"`); } - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "openclaw-inference-switch", status: "passed", assertions: { diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts index b3dfbd86456..a52a2cface7 100644 --- a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts +++ b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts @@ -161,9 +161,8 @@ liveTest( "OpenClaw plugin runtime deps replacement survives cross-filesystem EXDEV layout", { timeout: ONBOARD_TIMEOUT_MS + PROBE_TIMEOUT_MS + 5 * 60_000 }, async ({ artifacts, cleanup, host, sandbox, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "openclaw-plugin-runtime-exdev", - runner: "vitest", boundary: "fresh-openclaw-sandbox-exec", regressionTargets: ["#3513", "#3127"], contract: [ @@ -291,7 +290,7 @@ liveTest( expect(probeText).toContain("source-side staging failure self-check completed"); expect(probeText).toContain("runtime deps replacement completed"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "openclaw-plugin-runtime-exdev", onboardExitCode: onboard.exitCode, filesystemProbeExitCode: df.exitCode, diff --git a/test/e2e/live/openclaw-skill-cli.test.ts b/test/e2e/live/openclaw-skill-cli.test.ts index b9e4eb46547..ffb42121d28 100644 --- a/test/e2e/live/openclaw-skill-cli.test.ts +++ b/test/e2e/live/openclaw-skill-cli.test.ts @@ -149,9 +149,8 @@ runOpenClawSkillCliTest( "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "openclaw-skill-cli", - runner: "vitest", boundary: "install-sh-onboard-and-openclaw-skills-cli-in-sandbox", sandboxName: SANDBOX_NAME, contracts: [ @@ -278,7 +277,7 @@ runOpenClawSkillCliTest( ); expect(resultText(check)).toContain(`"${SKILL_ID}"`); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "openclaw-skill-cli", status: "passed", sandboxName: SANDBOX_NAME, diff --git a/test/e2e/live/openclaw-slack-pairing.test.ts b/test/e2e/live/openclaw-slack-pairing.test.ts index 124f301e3d0..3e071d16fa8 100644 --- a/test/e2e/live/openclaw-slack-pairing.test.ts +++ b/test/e2e/live/openclaw-slack-pairing.test.ts @@ -85,7 +85,7 @@ test.skipIf(!shouldRunLiveE2E())( slackApp: SLACK_APP_TOKEN, }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "openclaw-slack-pairing", boundary: "install.sh Slack OpenClaw sandbox + fake Slack REST/websocket token rewrite + runtime pairing request + connect-shell approval", diff --git a/test/e2e/live/openclaw-tui-chat-correlation.test.ts b/test/e2e/live/openclaw-tui-chat-correlation.test.ts index 025cc35f0c5..cf4fbdd44bc 100644 --- a/test/e2e/live/openclaw-tui-chat-correlation.test.ts +++ b/test/e2e/live/openclaw-tui-chat-correlation.test.ts @@ -488,9 +488,8 @@ test( async ({ artifacts, environment, onboard, sandbox, secrets }) => { secrets.required("NVIDIA_INFERENCE_API_KEY"); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "openclaw-tui-chat-correlation", - runner: "vitest", boundary: "openclaw-gateway-websocket", issues: ["#2603", "#3145"], ownerIssue: "#4347", diff --git a/test/e2e/live/openshell-version-pin.test.ts b/test/e2e/live/openshell-version-pin.test.ts index c627a9f1a4d..7999aeb907a 100644 --- a/test/e2e/live/openshell-version-pin.test.ts +++ b/test/e2e/live/openshell-version-pin.test.ts @@ -310,9 +310,8 @@ async function runVersionPinTarget( artifacts: ArtifactSink, options: { ghDownloadMode: GhDownloadMode }, ): Promise { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "openshell-version-pin", - runner: "vitest", boundary: "installer-script-unit", regressionTarget: "#3474", ghDownloadMode: options.ghDownloadMode, diff --git a/test/e2e/live/registry-targets.test.ts b/test/e2e/live/registry-targets.test.ts index 7a26048e680..e14e5c1a3c6 100644 --- a/test/e2e/live/registry-targets.test.ts +++ b/test/e2e/live/registry-targets.test.ts @@ -68,9 +68,8 @@ for (const target of listTargets()) { throw new Error(`target '${target.id}' is missing expectedStateId`); } - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: target.id, - runner: "vitest", boundary: "typed-registry", pendingRuntimeSuites: support.pendingRuntimeSuites, }); @@ -125,7 +124,7 @@ for (const target of listTargets()) { secrets, }); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: target.id, expectedStateId: validation.state.id, probes: validation.probes.map((probe) => probe.id), diff --git a/test/e2e/live/runtime-overrides.test.ts b/test/e2e/live/runtime-overrides.test.ts index d32eba631f2..e127bb98646 100644 --- a/test/e2e/live/runtime-overrides.test.ts +++ b/test/e2e/live/runtime-overrides.test.ts @@ -244,9 +244,8 @@ runtimeOverridesTest( const cleanupImage = process.env.NEMOCLAW_TEST_IMAGE === undefined; try { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "runtime-overrides", - runner: "vitest", boundary: "docker-image-entrypoint", image, contract: [ @@ -261,7 +260,7 @@ runtimeOverridesTest( const docker = dockerAvailable(); dockerLog.push(formatLog("docker info", docker)); if (docker.status !== 0) { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "runtime-overrides", status: "skipped", reason: DOCKER_REQUIRED_MESSAGE, @@ -385,7 +384,7 @@ runtimeOverridesTest( expect(primaryModel(rejected)).toBe(baselineModel); expect(firstProviderModel(rejected).contextWindow).toBe(baselineContextWindow); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "runtime-overrides", status: "passed", image, diff --git a/test/e2e/live/sandbox-operations.test.ts b/test/e2e/live/sandbox-operations.test.ts index d0472fa7657..365eadc5c1a 100644 --- a/test/e2e/live/sandbox-operations.test.ts +++ b/test/e2e/live/sandbox-operations.test.ts @@ -604,9 +604,8 @@ liveTest( async ({ artifacts, cleanup, environment, host, sandbox, secrets, skip }) => { const hosted = requireHostedInferenceConfig(secrets); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "sandbox-operations", - runner: "vitest", boundary: "repo-cli-docker-openshell-sandbox", contracts: [ "TC-SBX-01 list shows onboarded sandbox", @@ -669,7 +668,7 @@ liveTest( const gatewayRecovery = await assertGatewayRecovery(host, SANDBOX_A); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "sandbox-operations", status: "passed", gatewayRecovery, diff --git a/test/e2e/live/sandbox-rlimits-connect.test.ts b/test/e2e/live/sandbox-rlimits-connect.test.ts index 1ea957274f6..9ad6b5de2a2 100644 --- a/test/e2e/live/sandbox-rlimits-connect.test.ts +++ b/test/e2e/live/sandbox-rlimits-connect.test.ts @@ -58,7 +58,7 @@ runConnectRlimitTest( async ({ artifacts, cleanup, host, secrets }) => { const apiKey = secrets.required("NVIDIA_API_KEY"); const redactionValues = secrets.redactionValues([apiKey]); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "sandbox-rlimits-connect", issue: 2173, optIn: "NEMOCLAW_RUN_LIVE_E2E=1 NEMOCLAW_E2E_CONNECT_RLIMITS=1", diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index b1dca793958..181423e9593 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -88,9 +88,8 @@ test.skipIf(!shouldRunLiveE2E())( const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "sandbox-survival", - runner: "vitest", boundary: "install-sh-docker-openshell-gateway-sandbox-inference", contracts: [ "install.sh --non-interactive creates the named OpenClaw sandbox", @@ -304,7 +303,7 @@ test.skipIf(!shouldRunLiveE2E())( new RegExp(`(^|\\s)${SANDBOX_NAME}(\\s|$)`, "m"), ); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "sandbox-survival", status: "passed", assertions: { diff --git a/test/e2e/live/sessions-agents-cli.test.ts b/test/e2e/live/sessions-agents-cli.test.ts index 0fbe816d9d3..9b4900adb11 100644 --- a/test/e2e/live/sessions-agents-cli.test.ts +++ b/test/e2e/live/sessions-agents-cli.test.ts @@ -320,9 +320,8 @@ runSessionsAgentsCliTest( "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "sessions-agents-cli", - runner: "vitest", boundary: "host-cli-openclaw-sessions-agents-gateway", sandboxName: SANDBOX_NAME, contracts: [ diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index d9a8c1e8ef1..5a2c1771f04 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -228,9 +228,8 @@ RUN_SHIELDS_TEST( "shields-config: live shields up/down locks config and detects drift", { timeout: TEST_TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "shields-config", - runner: "vitest", boundary: "live-sandbox-shields-config", contracts: [ "source install creates a live OpenClaw sandbox", @@ -632,7 +631,7 @@ RUN_SHIELDS_TEST( expect(doubleDown.exitCode, resultText(doubleDown)).not.toBe(0); expect(resultText(doubleDown)).toContain("already unlocked"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "shields-config", sandboxName: SANDBOX_NAME, assertions: { diff --git a/test/e2e/live/skill-agent.test.ts b/test/e2e/live/skill-agent.test.ts index 0b567a1a03d..9c247db2423 100644 --- a/test/e2e/live/skill-agent.test.ts +++ b/test/e2e/live/skill-agent.test.ts @@ -139,9 +139,8 @@ runSkillAgentTest( const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "skill-agent", - runner: "vitest", boundary: "direct-cli-onboard-sandbox-skill-and-agent-turn", contract: [ "Docker is available before onboarding", @@ -227,7 +226,7 @@ runSkillAgentTest( ); const onboardText = resultText(onboard); if (onboard.exitCode !== 0 && isExternalProviderValidationFailure(onboardText)) { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "skill-agent", status: "skipped", reason: "external-provider-validation-unavailable-before-sandbox-skill-check", @@ -293,7 +292,7 @@ runSkillAgentTest( if (!agentOk) { const fixturePresent = await verifySkillFixturePresent(sandbox, SANDBOX_NAME); if (shouldSkipExternalAgentVerificationFailure(lastAgentOutput, fixturePresent)) { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "skill-agent", status: "skipped", reason: "external-agent-verification-flake-after-fixture-present", @@ -310,7 +309,7 @@ runSkillAgentTest( `Agent did not return ${VERIFY_PHRASE}; last exit ${lastExitCode}\n${lastAgentOutput.slice(-12_000)}`, ).toBe(true); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "skill-agent", status: "passed", assertions: { diff --git a/test/e2e/live/snapshot-commands.test.ts b/test/e2e/live/snapshot-commands.test.ts index e2b0856490e..30de92a9b5f 100644 --- a/test/e2e/live/snapshot-commands.test.ts +++ b/test/e2e/live/snapshot-commands.test.ts @@ -114,9 +114,8 @@ test.skipIf(!shouldRunLiveE2E())( { timeout: LIVE_TIMEOUT_MS }, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "snapshot-commands", - runner: "vitest", boundary: "install.sh + nemoclaw snapshot commands + openshell sandbox exec", sandboxName: SANDBOX_NAME, backupDir: BACKUP_DIR, @@ -340,7 +339,7 @@ test.skipIf(!shouldRunLiveE2E())( expect(resultText(help)).toContain("snapshot list"); expect(resultText(help)).toContain("snapshot restore"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "snapshot-commands", status: "passed", firstSnapshotTimestamp: timestamp, diff --git a/test/e2e/live/spark-install.test.ts b/test/e2e/live/spark-install.test.ts index 27ce9ecff29..1d0edeec078 100644 --- a/test/e2e/live/spark-install.test.ts +++ b/test/e2e/live/spark-install.test.ts @@ -75,7 +75,7 @@ liveTest( "spark install path: standard non-interactive install leaves NemoClaw and OpenShell usable", { timeout: LIVE_TIMEOUT_MS }, async ({ artifacts, cleanup, host, secrets }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "spark-install", sandboxName: SANDBOX_NAME, contracts: [ diff --git a/test/e2e/live/state-backup-restore.test.ts b/test/e2e/live/state-backup-restore.test.ts index afc16809608..2e61ac59527 100644 --- a/test/e2e/live/state-backup-restore.test.ts +++ b/test/e2e/live/state-backup-restore.test.ts @@ -231,7 +231,7 @@ test.skipIf(!shouldRunLiveE2E())( } catch (error) { const text = errorText(error); if (isNvidiaEndpointValidationUnavailable(text)) { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "state-backup-restore", status: "skipped", reason: "external-provider-validation-unavailable-before-state-backup-contract", @@ -340,7 +340,7 @@ test.skipIf(!shouldRunLiveE2E())( } catch (error) { const text = errorText(error); if (isNvidiaEndpointValidationUnavailable(text)) { - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "state-backup-restore", status: "skipped", reason: "external-provider-validation-unavailable-during-reonboard", diff --git a/test/e2e/live/telegram-injection.test.ts b/test/e2e/live/telegram-injection.test.ts index 4d4692dd240..8ba65ecb7e3 100644 --- a/test/e2e/live/telegram-injection.test.ts +++ b/test/e2e/live/telegram-injection.test.ts @@ -204,7 +204,7 @@ test.skipIf(!shouldRunLiveE2E())( }); const redactions = redactionValues(apiKey); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "telegram-injection", boundary: "install.sh OpenClaw sandbox + OpenShell sandbox exec and ssh-config stdin paths + process table and validateName probes", diff --git a/test/e2e/live/token-rotation.test.ts b/test/e2e/live/token-rotation.test.ts index 825a369b5d2..f6fccd40e91 100644 --- a/test/e2e/live/token-rotation.test.ts +++ b/test/e2e/live/token-rotation.test.ts @@ -299,9 +299,8 @@ liveTest( await fakeOpenAI.close(); }); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "token-rotation", - runner: "vitest", boundary: "direct-cli-onboard-openshell", workflow: { workflow: "e2e.yaml", @@ -493,7 +492,7 @@ liveTest( expect(afterSlackSameText).toContain(`Sandbox '${SANDBOX_NAME}' exists and is ready`); expect(afterSlackSameText).toContain("reusing it"); - await artifacts.writeJson("target-result.json", { + await artifacts.target.complete({ id: "token-rotation", sandboxName: SANDBOX_NAME, assertions: { diff --git a/test/e2e/live/ubuntu-repo-cli-smoke.test.ts b/test/e2e/live/ubuntu-repo-cli-smoke.test.ts index 98d75757eac..8942676b48c 100644 --- a/test/e2e/live/ubuntu-repo-cli-smoke.test.ts +++ b/test/e2e/live/ubuntu-repo-cli-smoke.test.ts @@ -10,9 +10,8 @@ const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); test("ubuntu repo cli smoke", async ({ artifacts, host }) => { - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "ubuntu-repo-cli-smoke", - runner: "vitest", boundary: "repo-local-cli", }); diff --git a/test/e2e/live/upgrade-stale-sandbox.test.ts b/test/e2e/live/upgrade-stale-sandbox.test.ts index 8cb6f7916f3..ba3a0506ae9 100644 --- a/test/e2e/live/upgrade-stale-sandbox.test.ts +++ b/test/e2e/live/upgrade-stale-sandbox.test.ts @@ -38,9 +38,8 @@ test.skipIf(!shouldRunLiveE2E())( async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { const hosted = requireHostedInferenceConfig(secrets); - await artifacts.writeJson("target.json", { + await artifacts.target.declare({ id: "upgrade-stale-sandbox", - runner: "vitest", boundary: "install.sh + Docker old base image + OpenShell sandbox create + NemoClaw rebuild", sandboxName: SANDBOX_NAME, oldOpenClawVersion: OLD_OPENCLAW_VERSION, diff --git a/test/e2e/support/e2e-target-evidence.test.ts b/test/e2e/support/e2e-target-evidence.test.ts new file mode 100644 index 00000000000..1c9492eab12 --- /dev/null +++ b/test/e2e/support/e2e-target-evidence.test.ts @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { ArtifactSink } from "../fixtures/artifacts.ts"; + +function liveTypescriptFiles(): string[] { + const liveRoot = path.resolve(import.meta.dirname, "../live"); + return fs + .readdirSync(liveRoot) + .filter((file) => file.endsWith(".ts")) + .map((file) => path.join(liveRoot, file)); +} + +describe("target evidence", () => { + it("emits normalized, redacted metadata and result files", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-target-evidence-")); + const secret = "target-evidence-secret"; + try { + const artifacts = new ArtifactSink(root, [secret]); + + await artifacts.target.declare({ + id: "typed-target", + contract: ["first contract", "second contract"], + detail: `contains ${secret}`, + }); + await artifacts.target.complete({ + id: "typed-target", + assertionCount: 2, + }); + + expect(JSON.parse(fs.readFileSync(path.join(root, "target.json"), "utf8"))).toEqual({ + id: "typed-target", + detail: "contains [REDACTED]", + contracts: ["first contract", "second contract"], + runner: "vitest", + }); + expect(JSON.parse(fs.readFileSync(path.join(root, "target-result.json"), "utf8"))).toEqual({ + id: "typed-target", + status: "passed", + assertionCount: 2, + runner: "vitest", + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("rejects invalid identifiers, results, and conflicting contract names", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-target-evidence-invalid-")); + try { + const target = new ArtifactSink(root).target; + + await expect(target.declare({ id: "" })).rejects.toThrow(/id must be a non-empty string/); + await expect(target.complete({ id: "typed-target", status: "" })).rejects.toThrow( + /status must be a non-empty string/, + ); + await expect( + target.declare({ + id: "typed-target", + contract: "singular", + contracts: ["plural"], + }), + ).rejects.toThrow(/either contract or contracts/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("keeps live target evidence behind the typed API", () => { + const violations = liveTypescriptFiles() + .filter((file) => + /\.writeJson\(["']target(?:-result)?\.json["']/.test(fs.readFileSync(file, "utf8")), + ) + .map((file) => path.basename(file)); + + expect(violations).toEqual([]); + }); +}); From 02f84801128f0aca0de83f1dcb23a2612ee576eb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 6 Jul 2026 21:02:14 -0400 Subject: [PATCH 2/5] fix(e2e): preserve result evidence extension fields --- test/e2e/fixtures/artifacts.ts | 29 ++++++++++++-------- test/e2e/support/e2e-target-evidence.test.ts | 2 ++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/test/e2e/fixtures/artifacts.ts b/test/e2e/fixtures/artifacts.ts index 863d0665b11..bf91ead59c0 100644 --- a/test/e2e/fixtures/artifacts.ts +++ b/test/e2e/fixtures/artifacts.ts @@ -40,20 +40,25 @@ function normalizeTargetEvidence( } const record = { ...value } as Record; - const singular = record.contract; - const plural = record.contracts; - if (singular !== undefined && plural !== undefined) { - throw new TypeError("target metadata must use either contract or contracts, not both"); - } - const contracts = singular ?? plural; - if (contracts !== undefined) { - const normalized = typeof contracts === "string" ? [contracts] : contracts; - if (!Array.isArray(normalized) || normalized.some((contract) => typeof contract !== "string")) { - throw new TypeError("target contracts must be a string or an array of strings"); + if (kind === "metadata") { + const singular = record.contract; + const plural = record.contracts; + if (singular !== undefined && plural !== undefined) { + throw new TypeError("target metadata must use either contract or contracts, not both"); + } + const contracts = singular ?? plural; + if (contracts !== undefined) { + const normalized = typeof contracts === "string" ? [contracts] : contracts; + if ( + !Array.isArray(normalized) || + normalized.some((contract) => typeof contract !== "string") + ) { + throw new TypeError("target contracts must be a string or an array of strings"); + } + record.contracts = normalized; } - record.contracts = normalized; + delete record.contract; } - delete record.contract; if (kind === "result") record.status ??= "passed"; record.runner = "vitest"; return record; diff --git a/test/e2e/support/e2e-target-evidence.test.ts b/test/e2e/support/e2e-target-evidence.test.ts index 1c9492eab12..03e6ee63a22 100644 --- a/test/e2e/support/e2e-target-evidence.test.ts +++ b/test/e2e/support/e2e-target-evidence.test.ts @@ -32,6 +32,7 @@ describe("target evidence", () => { await artifacts.target.complete({ id: "typed-target", assertionCount: 2, + contract: "result extension field", }); expect(JSON.parse(fs.readFileSync(path.join(root, "target.json"), "utf8"))).toEqual({ @@ -44,6 +45,7 @@ describe("target evidence", () => { id: "typed-target", status: "passed", assertionCount: 2, + contract: "result extension field", runner: "vitest", }); } finally { From b36e465b359015f86613e6f904a77cb517bcf3f9 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 6 Jul 2026 21:42:42 -0400 Subject: [PATCH 3/5] fix(e2e): harden target evidence migration guard --- test/e2e/support/e2e-target-evidence.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/support/e2e-target-evidence.test.ts b/test/e2e/support/e2e-target-evidence.test.ts index 03e6ee63a22..350c60a8c18 100644 --- a/test/e2e/support/e2e-target-evidence.test.ts +++ b/test/e2e/support/e2e-target-evidence.test.ts @@ -77,7 +77,7 @@ describe("target evidence", () => { it("keeps live target evidence behind the typed API", () => { const violations = liveTypescriptFiles() .filter((file) => - /\.writeJson\(["']target(?:-result)?\.json["']/.test(fs.readFileSync(file, "utf8")), + /\.writeJson\(\s*[`'"]target(?:-result)?\.json[`'"]/.test(fs.readFileSync(file, "utf8")), ) .map((file) => path.basename(file)); From b1d7147be872ca742bdd3a698c50cbced99335d8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 6 Jul 2026 23:32:56 -0400 Subject: [PATCH 4/5] test(e2e): cover target evidence validation edges Signed-off-by: Julie Yaunches --- test/e2e/fixtures/artifacts.ts | 5 +++++ test/e2e/support/e2e-target-evidence.test.ts | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/test/e2e/fixtures/artifacts.ts b/test/e2e/fixtures/artifacts.ts index bf91ead59c0..85a67b7f046 100644 --- a/test/e2e/fixtures/artifacts.ts +++ b/test/e2e/fixtures/artifacts.ts @@ -16,6 +16,11 @@ export type TargetMetadata> = export type TargetResult> = { id: string; + /** + * Optional for the normal success path: reaching `complete()` after the live + * assertions have passed records `passed`. Skipped or non-success evidence + * must set an explicit status at the call site. + */ status?: string; } & Extension; diff --git a/test/e2e/support/e2e-target-evidence.test.ts b/test/e2e/support/e2e-target-evidence.test.ts index 350c60a8c18..8f21115fe1b 100644 --- a/test/e2e/support/e2e-target-evidence.test.ts +++ b/test/e2e/support/e2e-target-evidence.test.ts @@ -28,6 +28,7 @@ describe("target evidence", () => { id: "typed-target", contract: ["first contract", "second contract"], detail: `contains ${secret}`, + extensionField: "metadata extension field", }); await artifacts.target.complete({ id: "typed-target", @@ -38,6 +39,7 @@ describe("target evidence", () => { expect(JSON.parse(fs.readFileSync(path.join(root, "target.json"), "utf8"))).toEqual({ id: "typed-target", detail: "contains [REDACTED]", + extensionField: "metadata extension field", contracts: ["first contract", "second contract"], runner: "vitest", }); @@ -59,9 +61,13 @@ describe("target evidence", () => { const target = new ArtifactSink(root).target; await expect(target.declare({ id: "" })).rejects.toThrow(/id must be a non-empty string/); + await expect(target.declare({ id: " " })).rejects.toThrow(/id must be a non-empty string/); await expect(target.complete({ id: "typed-target", status: "" })).rejects.toThrow( /status must be a non-empty string/, ); + await expect(target.complete({ id: "typed-target", status: " " })).rejects.toThrow( + /status must be a non-empty string/, + ); await expect( target.declare({ id: "typed-target", @@ -69,6 +75,18 @@ describe("target evidence", () => { contracts: ["plural"], }), ).rejects.toThrow(/either contract or contracts/); + await expect( + target.declare({ + id: "typed-target", + contract: 42, + } as never), + ).rejects.toThrow(/contracts must be a string or an array of strings/); + await expect( + target.declare({ + id: "typed-target", + contracts: ["valid", 42], + } as never), + ).rejects.toThrow(/contracts must be a string or an array of strings/); } finally { fs.rmSync(root, { recursive: true, force: true }); } From 92f94b21c6f21caf9b3713306b0fb74ee2da6daf Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 6 Jul 2026 23:55:00 -0400 Subject: [PATCH 5/5] fix(e2e): canonicalize artifact sink roots Signed-off-by: Julie Yaunches --- test/e2e/fixtures/artifacts.ts | 9 +++++-- test/e2e/support/e2e-target-evidence.test.ts | 25 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/test/e2e/fixtures/artifacts.ts b/test/e2e/fixtures/artifacts.ts index 85a67b7f046..833f4f823d3 100644 --- a/test/e2e/fixtures/artifacts.ts +++ b/test/e2e/fixtures/artifacts.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; @@ -19,7 +20,9 @@ export type TargetResult> = { /** * Optional for the normal success path: reaching `complete()` after the live * assertions have passed records `passed`. Skipped or non-success evidence - * must set an explicit status at the call site. + * must set an explicit status at the call site. Omit the key to use the + * default; an explicit `undefined` value is rejected like any other invalid + * status payload. */ status?: string; } & Extension; @@ -97,7 +100,9 @@ export class ArtifactSink { private readonly redactionValues = new Set(); constructor(rootDir: string, redactionValues: Iterable = []) { - this.rootDir = path.resolve(rootDir); + const resolvedRoot = path.resolve(rootDir); + fsSync.mkdirSync(resolvedRoot, { recursive: true }); + this.rootDir = fsSync.realpathSync(resolvedRoot); this.target = new TargetEvidenceWriter(this); this.addRedactionValues(redactionValues); } diff --git a/test/e2e/support/e2e-target-evidence.test.ts b/test/e2e/support/e2e-target-evidence.test.ts index 8f21115fe1b..b899d4f06ed 100644 --- a/test/e2e/support/e2e-target-evidence.test.ts +++ b/test/e2e/support/e2e-target-evidence.test.ts @@ -68,6 +68,9 @@ describe("target evidence", () => { await expect(target.complete({ id: "typed-target", status: " " })).rejects.toThrow( /status must be a non-empty string/, ); + await expect( + target.complete({ id: "typed-target", status: undefined } as never), + ).rejects.toThrow(/status must be a non-empty string/); await expect( target.declare({ id: "typed-target", @@ -92,7 +95,29 @@ describe("target evidence", () => { } }); + it("contains symlinked artifact roots before writing", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-target-evidence-symlink-")); + try { + const realRoot = path.join(root, "real-root"); + const linkRoot = path.join(root, "link-root"); + fs.mkdirSync(realRoot); + fs.symlinkSync(realRoot, linkRoot, "dir"); + + const artifacts = new ArtifactSink(linkRoot); + const targetPath = await artifacts.writeJson("target.json", { ok: true }); + + expect(targetPath).toBe(fs.realpathSync(path.join(realRoot, "target.json"))); + expect(JSON.parse(fs.readFileSync(path.join(realRoot, "target.json"), "utf8"))).toEqual({ + ok: true, + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("keeps live target evidence behind the typed API", () => { + // This static guard intentionally catches literal legacy target filenames; + // dynamic target filename construction should not be introduced in live tests. const violations = liveTypescriptFiles() .filter((file) => /\.writeJson\(\s*[`'"]target(?:-result)?\.json[`'"]/.test(fs.readFileSync(file, "utf8")),