From 6d4389d245f644dd1a6189df2e66e649dccee876 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 08:55:26 -0700 Subject: [PATCH 01/10] test(e2e): define protected inference qualification Signed-off-by: Aaron Erickson --- test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md | 72 ++ test/e2e/docs/README.md | 8 + test/e2e/registry/activation-qualification.ts | 684 ++++++++++++++++++ .../e2e-native-runtime-qualification.test.ts | 439 +++++++++++ .../native-runtime-qualification-fixtures.ts | 104 +++ 5 files changed, 1307 insertions(+) create mode 100644 test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md create mode 100644 test/e2e/registry/activation-qualification.ts create mode 100644 test/e2e/support/e2e-native-runtime-qualification.test.ts create mode 100644 test/e2e/support/native-runtime-qualification-fixtures.ts diff --git a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md new file mode 100644 index 00000000000..e4a53ce983e --- /dev/null +++ b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md @@ -0,0 +1,72 @@ + + + +# Native Runtime Activation Qualification + +`registry/activation-qualification.ts` is a dormant release gate for a native +container-runtime candidate. It does not register a runtime, add a live target, +alter the production runtime registry, or advertise support. + +The compiler has one provider-neutral contract. A candidate supplies an open +provider ID and fixture bindings; the compiler does not branch on Podman, MXC, +or any other engine name. The inert Podman candidate in +`support/native-runtime-qualification-fixtures.ts` proves the intended scope, +while the fake MXC-style test proves that the same contract accepts another +provider without a central switch. + +## Required protected matrix + +Compilation requires 24 exact cases: + +- OpenClaw, Hermes, and DCode; +- Linux `amd64` and `arm64`; +- rootless CPU with host-local Ollama; +- rootless NVIDIA GPU with CDI-backed Ollama, NIM, and vLLM; +- the release installer with Docker unavailable; and +- protected E2E for every case. + +Every case must declare installation, Docker-unavailable proof, onboarding, an +agent turn, stop/start, snapshot/restore, rebuild, restart/reconciliation, and +exact cleanup. Removing one case or obligation is a compile error, not a skip. +The agent identity also binds its application-facing name: OpenClaw, Hermes, +or `langchain-deepagents-code` for DCode. + +## Exact evidence + +Activation evidence is complete only when every compiled case has: + +- the exact protected workflow run, job, attempt, head SHA, and base SHA; +- hashed installer script and invocation artifacts with a successful result; +- an exact provider/profile/architecture/acceleration identity and persisted + host-local engine authority; +- immutable agent and probe image references, plus an immutable inference image + reference for provider-managed NIM and vLLM; +- the exact provider-native host, port, network, gateway provider URL, and the + canonical `https://inference.local/v1` application route; +- the serialized host-local inference authority digest, including the exact + provider-owned runtime/container identity and specification digest for NIM + and vLLM; +- the exercised model ID and a hashed inference-result artifact; +- hashed artifacts for every lifecycle obligation, all bound to that same + durable authority; +- a reconciliation receipt proving recovery retained the same authority; +- an NVIDIA CDI `nvidia.com/gpu=all` receipt for GPU cases; and +- exact cleanup proving external Ollama was retained or provider-owned NIM and + vLLM were removed, with no provider-owned runtime IDs remaining. + +Evidence paths must be relative and traversal-free. SHA and SHA-256 fields are +strict lowercase hexadecimal values. Missing, duplicate, unknown, or inexact +case evidence fails the aggregate qualification check. All cases must use the +configured protected workflow and one exact head/base pair. + +These are evidence requirements, not generated evidence. A later protected +collector must publish the receipts from real runners before activation can +consume them. + +## Activation boundary + +Keep this contract inert until an implementation PR supplies executable +provider adapters and the entire protected matrix passes on one exact head/base +pair. Public support, installer selection, production registry wiring, and +workflow dispatch are separate activation work and must not infer support from +the existence of this contract. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 026bfe0b68d..49a6ab08924 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -28,6 +28,7 @@ Direct E2E implementations now live in Vitest. The former | Expected-state probes | `test/e2e/registry/expected-states.ts` | | Product-facing setup/onboarding state | `test/e2e/manifests/*.yaml` | | Migration status and retirement decisions | GitHub issues and pull requests | +| Native runtime activation qualification | `registry/activation-qualification.ts`, `docs/NATIVE_RUNTIME_QUALIFICATION.md` | ## Target Model @@ -107,6 +108,13 @@ mechanics in obligation bindings, and support facts in capabilities. A binding must cover every obligation explicitly; a missing adapter or capability is a compile error rather than a skip. +The dormant native-runtime activation contract builds on this foundation. It +requires the complete all-agent, multiarch, CPU/GPU, host-local inference, +installer, lifecycle, recovery, cleanup, and exact-evidence matrix before a +provider can be activated. See +[`NATIVE_RUNTIME_QUALIFICATION.md`](./NATIVE_RUNTIME_QUALIFICATION.md). The +contract is not production runtime registration or a support claim. + ## How To Run ```bash diff --git a/test/e2e/registry/activation-qualification.ts b/test/e2e/registry/activation-qualification.ts new file mode 100644 index 00000000000..aab61dda141 --- /dev/null +++ b/test/e2e/registry/activation-qualification.ts @@ -0,0 +1,684 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + defineExecutionProfile, + type ExecutionAcceleration, + type ExecutionArchitecture, + type ExecutionCapability, + type ExecutionProfile, + type ExecutionProviderId, +} from "./execution-profile.ts"; +import { compareCodeUnits, type RuntimeAgent } from "./scenario.ts"; + +export const NATIVE_RUNTIME_QUALIFICATION_AGENTS = ["openclaw", "hermes", "dcode"] as const; +export const NATIVE_RUNTIME_QUALIFICATION_ARCHITECTURES = ["amd64", "arm64"] as const; +export const NATIVE_RUNTIME_QUALIFICATION_ACCELERATIONS = ["cpu", "nvidia-gpu"] as const; +export const NATIVE_RUNTIME_QUALIFICATION_INFERENCE = { + cpu: ["ollama"], + "nvidia-gpu": ["ollama", "nim", "vllm"], +} as const satisfies Readonly>; + +export type LocalInferenceProvider = "ollama" | "nim" | "vllm"; +export type QualificationApplication = "openclaw" | "hermes" | "langchain-deepagents-code"; +export type QualificationManagedImageRole = "agent" | "inference" | "probe"; +export type QualificationObligation = + | "installer.install" + | "runtime.docker-unavailable" + | "agent.onboard" + | "agent.turn" + | "sandbox.stop-start" + | "sandbox.snapshot-restore" + | "sandbox.rebuild" + | "runtime.restart-reconcile" + | "cleanup.exact"; +export type QualificationEvidenceKind = + | "protected-run" + | "source-identity" + | "installer-result" + | "docker-unavailable-guard" + | "managed-images" + | "agent-turn" + | "local-inference" + | "lifecycle" + | "recovery" + | "cleanup" + | "nvidia-cdi"; + +export const NATIVE_RUNTIME_QUALIFICATION_OBLIGATIONS = [ + "installer.install", + "runtime.docker-unavailable", + "agent.onboard", + "agent.turn", + "sandbox.stop-start", + "sandbox.snapshot-restore", + "sandbox.rebuild", + "runtime.restart-reconcile", + "cleanup.exact", +] as const satisfies readonly QualificationObligation[]; + +const BASE_EVIDENCE_KINDS = [ + "protected-run", + "source-identity", + "installer-result", + "docker-unavailable-guard", + "managed-images", + "agent-turn", + "local-inference", + "lifecycle", + "recovery", + "cleanup", +] as const satisfies readonly QualificationEvidenceKind[]; + +const REQUIRED_CAPABILITIES = [ + "agent.configure", + "agent.turn", + "evidence.collect", + "sandbox.lifecycle", + "state.observe", + "transport.socket-free", +] as const satisfies readonly ExecutionCapability[]; + +export interface NativeRuntimeQualificationCase { + id: string; + agent: RuntimeAgent; + profile: ExecutionProfile; + inference: LocalInferenceProvider; + gate: "protected-e2e"; + install: "release-installer"; + dockerAvailability: "unavailable"; + obligations: readonly QualificationObligation[]; + evidenceKinds: readonly QualificationEvidenceKind[]; +} + +export interface NativeRuntimeQualificationDefinition { + id: string; + repository: string; + protectedWorkflow: string; + provider: ExecutionProviderId; + engineName: string; + cases: readonly NativeRuntimeQualificationCase[]; +} + +export interface CompiledNativeRuntimeQualification { + id: string; + repository: string; + protectedWorkflow: string; + provider: ExecutionProviderId; + engineName: string; + cases: readonly Readonly[]; +} + +export interface QualificationArtifactReceipt { + path: string; + sha256: string; +} + +export interface NativeRuntimeQualificationEvidence { + schemaVersion: 1; + caseId: string; + protectedRun: { + repository: string; + workflow: string; + runId: number; + attempt: number; + jobId: number; + headSha: string; + baseSha: string; + }; + installer: { + provider: ExecutionProviderId; + architecture: ExecutionArchitecture; + dockerAvailability: "unavailable"; + exitCode: 0; + invocation: QualificationArtifactReceipt; + script: QualificationArtifactReceipt; + }; + runtime: { + provider: ExecutionProviderId; + profileId: string; + agent: RuntimeAgent; + application: QualificationApplication; + inference: LocalInferenceProvider; + architecture: ExecutionArchitecture; + acceleration: ExecutionAcceleration; + rootMode: "rootless"; + engineName: string; + engineVersion: string; + engineAuthority: { + schemaVersion: 1; + providerId: ExecutionProviderId; + operation: "host-local-inference"; + engineId: string; + authorityId: string; + bindingSha256: string; + }; + managedImages: readonly { + role: QualificationManagedImageRole; + imageRef: string; + }[]; + route: { + service: LocalInferenceProvider; + endpoint: { + host: string; + port: number; + networkName: string; + gatewayProviderBaseUrl: string; + applicationBaseUrl: "https://inference.local/v1"; + }; + authority: { + receiptSha256: string; + kind: "host" | "container"; + runtimeId: string | null; + containerName: string | null; + specSha256: string | null; + }; + }; + modelId: string; + inferenceResult: QualificationArtifactReceipt; + }; + operations: readonly { + id: QualificationObligation; + authoritySha256: string; + artifact: QualificationArtifactReceipt; + }[]; + recovery: { + status: "reconciled"; + authoritySha256: string; + artifact: QualificationArtifactReceipt; + }; + cleanup: { + status: "retained-external" | "removed-owned"; + authoritySha256: string; + providerOwnedRuntimeIds: readonly string[]; + artifact: QualificationArtifactReceipt; + }; + nvidiaCdi?: { + devices: readonly ["nvidia.com/gpu=all"]; + artifact: QualificationArtifactReceipt; + }; +} + +const compiledQualifications = new WeakSet(); +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const IMAGE_REFERENCE_PATTERN = + /^(?:[A-Za-z0-9._-]+(?::[0-9]+)?\/)*(?:[A-Za-z0-9._-]+)@sha256:[a-f0-9]{64}$/u; +const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const SAFE_HOST_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$/u; +const SAFE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/=+-]{0,511}$/u; + +const APPLICATION_BY_AGENT = { + openclaw: "openclaw", + hermes: "hermes", + dcode: "langchain-deepagents-code", +} as const satisfies Readonly>; + +function assertSingleLine(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized || /[\r\n]/u.test(normalized)) { + throw new Error(`${label} must be a non-empty single-line string`); + } + return normalized; +} + +function assertExactSet( + actual: readonly T[], + expected: readonly T[], + label: string, +): void { + const actualSet = new Set(actual); + if (actualSet.size !== actual.length) { + throw new Error(`${label} contains duplicate values`); + } + const missing = expected.filter((value) => !actualSet.has(value)); + const unknown = actual.filter((value) => !expected.includes(value)); + if (missing.length > 0 || unknown.length > 0) { + throw new Error( + `${label} is incomplete (missing: ${missing.join(", ") || "none"}; unknown: ${unknown.join(", ") || "none"})`, + ); + } +} + +export function requiredQualificationEvidenceKinds( + acceleration: ExecutionAcceleration, +): readonly QualificationEvidenceKind[] { + return acceleration === "nvidia-gpu" + ? Object.freeze([...BASE_EVIDENCE_KINDS, "nvidia-cdi"]) + : BASE_EVIDENCE_KINDS; +} + +export function qualificationCaseId(input: { + provider: ExecutionProviderId; + agent: RuntimeAgent; + architecture: ExecutionArchitecture; + acceleration: ExecutionAcceleration; + inference: LocalInferenceProvider; +}): string { + return [ + input.provider, + input.agent, + "linux", + input.architecture, + input.acceleration, + input.inference, + ] + .join("-") + .replace("nvidia-gpu", "gpu"); +} + +function coverageKey(input: { + agent: RuntimeAgent; + architecture: ExecutionArchitecture; + acceleration: ExecutionAcceleration; + inference: LocalInferenceProvider; +}): string { + return [input.agent, input.architecture, input.acceleration, input.inference].join("|"); +} + +function requiredCoverageKeys(): string[] { + return NATIVE_RUNTIME_QUALIFICATION_AGENTS.flatMap((agent) => + NATIVE_RUNTIME_QUALIFICATION_ARCHITECTURES.flatMap((architecture) => + NATIVE_RUNTIME_QUALIFICATION_ACCELERATIONS.flatMap((acceleration) => + NATIVE_RUNTIME_QUALIFICATION_INFERENCE[acceleration].map((inference) => + coverageKey({ agent, architecture, acceleration, inference }), + ), + ), + ), + ).sort(compareCodeUnits); +} + +function compileCase( + definition: NativeRuntimeQualificationDefinition, + input: NativeRuntimeQualificationCase, +): Readonly { + const profile = defineExecutionProfile(input.profile); + if (profile.provider !== definition.provider) { + throw new Error( + `Qualification case '${input.id}' profile provider '${profile.provider}' does not match '${definition.provider}'`, + ); + } + if (profile.platform !== "linux" || profile.rootMode !== "rootless") { + throw new Error(`Qualification case '${input.id}' must use a rootless Linux profile`); + } + const capabilities = new Set(profile.capabilities); + const missingCapabilities = REQUIRED_CAPABILITIES.filter((value) => !capabilities.has(value)); + if (missingCapabilities.length > 0 || capabilities.has("transport.docker-socket")) { + throw new Error( + `Qualification case '${input.id}' must be socket-free and declares invalid capabilities (missing: ${missingCapabilities.join(", ") || "none"})`, + ); + } + if (input.gate !== "protected-e2e") { + throw new Error(`Qualification case '${input.id}' must run through protected E2E`); + } + if (input.install !== "release-installer") { + throw new Error(`Qualification case '${input.id}' must exercise the release installer`); + } + if (input.dockerAvailability !== "unavailable") { + throw new Error(`Qualification case '${input.id}' must prove Docker is unavailable`); + } + const allowedInference = NATIVE_RUNTIME_QUALIFICATION_INFERENCE[profile.acceleration]; + if (!(allowedInference as readonly string[]).includes(input.inference)) { + throw new Error( + `Qualification case '${input.id}' cannot use ${input.inference} with ${profile.acceleration}`, + ); + } + assertExactSet( + input.obligations, + NATIVE_RUNTIME_QUALIFICATION_OBLIGATIONS, + `Qualification case '${input.id}' obligations`, + ); + assertExactSet( + input.evidenceKinds, + requiredQualificationEvidenceKinds(profile.acceleration), + `Qualification case '${input.id}' evidence kinds`, + ); + const expectedId = qualificationCaseId({ + provider: definition.provider, + agent: input.agent, + architecture: profile.architecture, + acceleration: profile.acceleration, + inference: input.inference, + }); + if (input.id !== expectedId) { + throw new Error(`Qualification case id '${input.id}' must be '${expectedId}'`); + } + return Object.freeze({ + ...input, + profile, + obligations: Object.freeze([...input.obligations]), + evidenceKinds: Object.freeze([...input.evidenceKinds]), + }); +} + +export function compileNativeRuntimeQualification( + input: NativeRuntimeQualificationDefinition, +): CompiledNativeRuntimeQualification { + const id = assertSingleLine(input.id, "Native runtime qualification id"); + const repository = assertSingleLine(input.repository, "Native runtime qualification repository"); + if (!REPOSITORY_PATTERN.test(repository)) { + throw new Error(`Native runtime qualification repository '${repository}' must be owner/name`); + } + const protectedWorkflow = assertSingleLine( + input.protectedWorkflow, + "Native runtime qualification protected workflow", + ); + const engineName = assertSingleLine(input.engineName, "Native runtime engine name"); + const cases = input.cases.map((entry) => compileCase(input, entry)); + const casesByCoverage = new Map>(); + for (const entry of cases) { + const key = coverageKey({ + agent: entry.agent, + architecture: entry.profile.architecture, + acceleration: entry.profile.acceleration, + inference: entry.inference, + }); + if (casesByCoverage.has(key)) { + throw new Error(`Native runtime qualification repeats case coverage '${key}'`); + } + casesByCoverage.set(key, entry); + } + const expectedCoverage = requiredCoverageKeys(); + const missing = expectedCoverage.filter((key) => !casesByCoverage.has(key)); + const unknown = [...casesByCoverage.keys()].filter((key) => !expectedCoverage.includes(key)); + if (missing.length > 0 || unknown.length > 0) { + throw new Error( + `Native runtime qualification coverage is incomplete (missing: ${missing.join(", ") || "none"}; unknown: ${unknown.join(", ") || "none"})`, + ); + } + const compiled = Object.freeze({ + id, + repository, + protectedWorkflow, + provider: input.provider, + engineName, + cases: Object.freeze([...cases].sort((left, right) => compareCodeUnits(left.id, right.id))), + }); + compiledQualifications.add(compiled); + return compiled; +} + +function assertArtifact(receipt: QualificationArtifactReceipt, label: string): void { + const artifactPath = assertSingleLine(receipt.path, `${label} path`); + if ( + artifactPath.startsWith("/") || + artifactPath.startsWith("\\") || + artifactPath.split(/[\\/]/u).some((part) => part === "..") + ) { + throw new Error(`${label} path must be repository-relative and traversal-free`); + } + if (!SHA256_PATTERN.test(receipt.sha256)) { + throw new Error(`${label} sha256 must be an exact lowercase SHA-256 digest`); + } +} + +function assertPositiveInteger(value: number, label: string): void { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`${label} must be a positive integer`); + } +} + +function assertSha256(value: string, label: string): void { + if (!SHA256_PATTERN.test(value)) { + throw new Error(`${label} must be an exact lowercase SHA-256 digest`); + } +} + +function assertSafeId(value: string, label: string): void { + if (!SAFE_ID_PATTERN.test(value)) { + throw new Error(`${label} must be an exact non-empty runtime identifier`); + } +} + +function requiredManagedImageRoles( + inference: LocalInferenceProvider, +): readonly QualificationManagedImageRole[] { + return inference === "ollama" ? ["agent", "probe"] : ["agent", "inference", "probe"]; +} + +function assertExactUrl(value: string, label: string): URL { + const text = assertSingleLine(value, label); + let parsed: URL; + try { + parsed = new URL(text); + } catch { + throw new Error(`${label} must be an exact absolute URL`); + } + if (parsed.username || parsed.password || parsed.search || parsed.hash || parsed.href !== text) { + throw new Error(`${label} must not contain credentials, query parameters, or fragments`); + } + return parsed; +} + +function assertEngineAuthority( + definition: CompiledNativeRuntimeQualification, + evidence: NativeRuntimeQualificationEvidence, +): void { + const authority = evidence.runtime.engineAuthority; + if ( + authority.schemaVersion !== 1 || + authority.providerId !== definition.provider || + authority.operation !== "host-local-inference" + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' has invalid engine authority`); + } + assertSafeId(authority.engineId, "Runtime engine id"); + assertSafeId(authority.authorityId, "Runtime authority id"); + assertSha256(authority.bindingSha256, "Runtime authority binding"); +} + +function assertInferenceRoute( + qualificationCase: Readonly, + evidence: NativeRuntimeQualificationEvidence, +): string { + const route = evidence.runtime.route; + if (route.service !== qualificationCase.inference) { + throw new Error(`Qualification evidence '${evidence.caseId}' has a different route service`); + } + const endpoint = route.endpoint; + if (!SAFE_HOST_PATTERN.test(endpoint.host)) { + throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid endpoint host`); + } + assertPositiveInteger(endpoint.port, "Inference endpoint port"); + if (endpoint.port > 65_535) { + throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid endpoint port`); + } + assertSafeId(endpoint.networkName, "Inference endpoint network"); + const gateway = assertExactUrl(endpoint.gatewayProviderBaseUrl, "Gateway provider base URL"); + if (gateway.protocol !== "http:" || Number(gateway.port) !== endpoint.port) { + throw new Error( + `Qualification evidence '${evidence.caseId}' has a mismatched gateway endpoint`, + ); + } + if (endpoint.applicationBaseUrl !== "https://inference.local/v1") { + throw new Error( + `Qualification evidence '${evidence.caseId}' has a noncanonical application route`, + ); + } + + const authority = route.authority; + assertSha256(authority.receiptSha256, "Inference authority receipt"); + if (qualificationCase.inference === "ollama") { + if ( + authority.kind !== "host" || + authority.runtimeId !== null || + authority.containerName !== null || + authority.specSha256 !== null + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' must retain external Ollama`); + } + } else { + if ( + authority.kind !== "container" || + authority.runtimeId === null || + authority.containerName === null || + authority.specSha256 === null + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' must name managed inference`); + } + assertSafeId(authority.runtimeId, "Managed inference runtime id"); + assertSafeId(authority.containerName, "Managed inference container name"); + assertSha256(authority.specSha256, "Managed inference specification"); + } + return authority.receiptSha256; +} + +function assertLifecycleEvidence( + qualificationCase: Readonly, + evidence: NativeRuntimeQualificationEvidence, + authoritySha256: string, +): void { + const operationIds = evidence.operations.map((operation) => operation.id); + assertExactSet( + operationIds, + qualificationCase.obligations, + `Qualification evidence '${evidence.caseId}' operations`, + ); + for (const operation of evidence.operations) { + if (operation.authoritySha256 !== authoritySha256) { + throw new Error(`Operation '${operation.id}' is bound to different runtime authority`); + } + assertArtifact(operation.artifact, `Operation '${operation.id}' artifact`); + } + if ( + evidence.recovery.status !== "reconciled" || + evidence.recovery.authoritySha256 !== authoritySha256 + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' has invalid recovery evidence`); + } + assertArtifact(evidence.recovery.artifact, "Recovery artifact"); + + const cleanupStatus = + qualificationCase.inference === "ollama" ? "retained-external" : "removed-owned"; + if ( + evidence.cleanup.status !== cleanupStatus || + evidence.cleanup.authoritySha256 !== authoritySha256 || + evidence.cleanup.providerOwnedRuntimeIds.length !== 0 + ) { + throw new Error( + `Qualification evidence '${evidence.caseId}' has invalid exact cleanup evidence`, + ); + } + assertArtifact(evidence.cleanup.artifact, "Cleanup artifact"); +} + +function assertCaseEvidence( + definition: CompiledNativeRuntimeQualification, + qualificationCase: Readonly, + evidence: NativeRuntimeQualificationEvidence, +): void { + if (evidence.schemaVersion !== 1) { + throw new Error(`Qualification evidence '${evidence.caseId}' has unsupported schemaVersion`); + } + if (evidence.protectedRun.repository !== definition.repository) { + throw new Error(`Qualification evidence '${evidence.caseId}' belongs to the wrong repository`); + } + if (evidence.protectedRun.workflow !== definition.protectedWorkflow) { + throw new Error(`Qualification evidence '${evidence.caseId}' belongs to the wrong workflow`); + } + assertPositiveInteger(evidence.protectedRun.runId, "Protected run id"); + assertPositiveInteger(evidence.protectedRun.attempt, "Protected run attempt"); + assertPositiveInteger(evidence.protectedRun.jobId, "Protected run job id"); + if ( + !SHA_PATTERN.test(evidence.protectedRun.headSha) || + !SHA_PATTERN.test(evidence.protectedRun.baseSha) + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' must name exact head/base SHAs`); + } + if (evidence.protectedRun.headSha === evidence.protectedRun.baseSha) { + throw new Error(`Qualification evidence '${evidence.caseId}' head/base SHAs must differ`); + } + + const profile = qualificationCase.profile; + if ( + evidence.installer.provider !== definition.provider || + evidence.installer.architecture !== profile.architecture || + evidence.installer.dockerAvailability !== "unavailable" || + evidence.installer.exitCode !== 0 + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid installer receipt`); + } + assertArtifact(evidence.installer.invocation, "Installer invocation artifact"); + assertArtifact(evidence.installer.script, "Installer script artifact"); + + if ( + evidence.runtime.provider !== definition.provider || + evidence.runtime.profileId !== profile.id || + evidence.runtime.agent !== qualificationCase.agent || + evidence.runtime.application !== APPLICATION_BY_AGENT[qualificationCase.agent] || + evidence.runtime.inference !== qualificationCase.inference || + evidence.runtime.architecture !== profile.architecture || + evidence.runtime.acceleration !== profile.acceleration || + evidence.runtime.rootMode !== "rootless" + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid runtime identity`); + } + if (evidence.runtime.engineName !== definition.engineName) { + throw new Error(`Qualification evidence '${evidence.caseId}' names the wrong runtime engine`); + } + assertSingleLine(evidence.runtime.engineVersion, "Runtime engine version"); + assertEngineAuthority(definition, evidence); + const imageRoles: QualificationManagedImageRole[] = []; + for (const image of evidence.runtime.managedImages) { + imageRoles.push(image.role); + if (!IMAGE_REFERENCE_PATTERN.test(image.imageRef)) { + throw new Error( + `Qualification evidence '${evidence.caseId}' must use exact image references`, + ); + } + } + assertExactSet( + imageRoles, + requiredManagedImageRoles(qualificationCase.inference), + `Qualification evidence '${evidence.caseId}' managed image roles`, + ); + const authoritySha256 = assertInferenceRoute(qualificationCase, evidence); + assertSingleLine(evidence.runtime.modelId, "Inference model id"); + assertArtifact(evidence.runtime.inferenceResult, "Inference result artifact"); + assertLifecycleEvidence(qualificationCase, evidence, authoritySha256); + + if (profile.acceleration === "nvidia-gpu") { + if ( + evidence.nvidiaCdi?.devices.length !== 1 || + evidence.nvidiaCdi.devices[0] !== "nvidia.com/gpu=all" + ) { + throw new Error(`Qualification evidence '${evidence.caseId}' must prove NVIDIA CDI access`); + } + assertArtifact(evidence.nvidiaCdi.artifact, "NVIDIA CDI artifact"); + } else if (evidence.nvidiaCdi !== undefined) { + throw new Error(`CPU qualification evidence '${evidence.caseId}' must not claim NVIDIA CDI`); + } +} + +export function assertNativeRuntimeQualificationEvidence( + definition: CompiledNativeRuntimeQualification, + evidence: readonly NativeRuntimeQualificationEvidence[], +): void { + if (!compiledQualifications.has(definition)) { + throw new Error("Native runtime qualification evidence requires a compiled definition"); + } + const casesById = new Map(definition.cases.map((entry) => [entry.id, entry])); + const evidenceById = new Map(); + const sourcePairs = new Set(); + for (const entry of evidence) { + if (evidenceById.has(entry.caseId)) { + throw new Error(`Native runtime qualification evidence repeats case '${entry.caseId}'`); + } + const qualificationCase = casesById.get(entry.caseId); + if (!qualificationCase) { + throw new Error(`Native runtime qualification evidence names unknown case '${entry.caseId}'`); + } + assertCaseEvidence(definition, qualificationCase, entry); + sourcePairs.add(`${entry.protectedRun.headSha}:${entry.protectedRun.baseSha}`); + evidenceById.set(entry.caseId, entry); + } + const missing = definition.cases.filter((entry) => !evidenceById.has(entry.id)); + if (missing.length > 0) { + throw new Error( + `Native runtime qualification evidence is incomplete: ${missing.map((entry) => entry.id).join(", ")}`, + ); + } + if (sourcePairs.size !== 1) { + throw new Error("Native runtime qualification evidence must use one exact head/base pair"); + } +} diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts new file mode 100644 index 00000000000..433ab5f633d --- /dev/null +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -0,0 +1,439 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + assertNativeRuntimeQualificationEvidence, + compileNativeRuntimeQualification, + type NativeRuntimeQualificationEvidence, +} from "../registry/activation-qualification.ts"; +import { hasRegisteredRuntimeProfile } from "../registry/registry.ts"; +import { + nativeRuntimeQualificationDefinition, + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, +} from "./native-runtime-qualification-fixtures.ts"; + +const HEAD_SHA = "1".repeat(40); +const BASE_SHA = "2".repeat(40); +const ARTIFACT_SHA = "a".repeat(64); +const AUTHORITY_SHA = "b".repeat(64); +const BINDING_SHA = "c".repeat(64); +const SPEC_SHA = "d".repeat(64); +const IMAGE_REFS = { + agent: `nvcr.io/nvidia/nemoclaw-agent@sha256:${"1".repeat(64)}`, + inference: `nvcr.io/nvidia/nemoclaw-inference@sha256:${"2".repeat(64)}`, + probe: `quay.io/curl/curl@sha256:${"3".repeat(64)}`, +} as const; + +const APPLICATIONS = { + openclaw: "openclaw", + hermes: "hermes", + dcode: "langchain-deepagents-code", +} as const; + +function artifact(label: string) { + return { path: `qualification/${label}.json`, sha256: ARTIFACT_SHA }; +} + +function completeEvidence(): NativeRuntimeQualificationEvidence[] { + return PODMAN_NATIVE_ACTIVATION_QUALIFICATION.cases.map((qualificationCase, index) => { + const managed = qualificationCase.inference !== "ollama"; + return { + schemaVersion: 1, + caseId: qualificationCase.id, + protectedRun: { + repository: "NVIDIA/NemoClaw", + workflow: "E2E / PR Gate", + runId: 1000 + index, + attempt: 1, + jobId: 2000 + index, + headSha: HEAD_SHA, + baseSha: BASE_SHA, + }, + installer: { + provider: PODMAN_NATIVE_ACTIVATION_QUALIFICATION.provider, + architecture: qualificationCase.profile.architecture, + dockerAvailability: "unavailable", + exitCode: 0, + invocation: artifact(`${qualificationCase.id}-installer-invocation`), + script: artifact(`${qualificationCase.id}-installer-script`), + }, + runtime: { + provider: PODMAN_NATIVE_ACTIVATION_QUALIFICATION.provider, + profileId: qualificationCase.profile.id, + agent: qualificationCase.agent, + application: APPLICATIONS[qualificationCase.agent], + inference: qualificationCase.inference, + architecture: qualificationCase.profile.architecture, + acceleration: qualificationCase.profile.acceleration, + rootMode: "rootless", + engineName: "podman", + engineVersion: "5.6.2", + engineAuthority: { + schemaVersion: 1, + providerId: PODMAN_NATIVE_ACTIVATION_QUALIFICATION.provider, + operation: "host-local-inference", + engineId: "podman-rootless", + authorityId: "podman:host-local-inference", + bindingSha256: BINDING_SHA, + }, + managedImages: [ + { role: "agent" as const, imageRef: IMAGE_REFS.agent }, + { role: "probe" as const, imageRef: IMAGE_REFS.probe }, + ...(managed ? [{ role: "inference" as const, imageRef: IMAGE_REFS.inference }] : []), + ], + route: { + service: qualificationCase.inference, + endpoint: { + host: "podman.internal", + port: 8000, + networkName: "podman-inference", + gatewayProviderBaseUrl: "http://host.openshell.internal:8000/v1", + applicationBaseUrl: "https://inference.local/v1", + }, + authority: { + receiptSha256: AUTHORITY_SHA, + kind: managed ? ("container" as const) : ("host" as const), + runtimeId: managed ? `podman-${qualificationCase.inference}` : null, + containerName: managed ? `nemoclaw-${qualificationCase.inference}` : null, + specSha256: managed ? SPEC_SHA : null, + }, + }, + modelId: `${qualificationCase.inference}-qualification-model`, + inferenceResult: artifact(`${qualificationCase.id}-inference-result`), + }, + operations: qualificationCase.obligations.map((id) => ({ + id, + authoritySha256: AUTHORITY_SHA, + artifact: artifact(`${qualificationCase.id}-${id}`), + })), + recovery: { + status: "reconciled" as const, + authoritySha256: AUTHORITY_SHA, + artifact: artifact(`${qualificationCase.id}-recovery`), + }, + cleanup: { + status: managed ? ("removed-owned" as const) : ("retained-external" as const), + authoritySha256: AUTHORITY_SHA, + providerOwnedRuntimeIds: [], + artifact: artifact(`${qualificationCase.id}-cleanup`), + }, + ...(qualificationCase.profile.acceleration === "nvidia-gpu" + ? { + nvidiaCdi: { + devices: ["nvidia.com/gpu=all"] as const, + artifact: artifact(`${qualificationCase.id}-nvidia-cdi`), + }, + } + : {}), + }; + }); +} + +describe("native runtime activation qualification", () => { + it("compiles every required all-agent, multiarch, CPU/GPU, and local-inference case", () => { + const qualification = PODMAN_NATIVE_ACTIVATION_QUALIFICATION; + + expect(qualification.cases).toHaveLength(24); + expect(new Set(qualification.cases.map((entry) => entry.agent))).toEqual( + new Set(["openclaw", "hermes", "dcode"]), + ); + expect(new Set(qualification.cases.map((entry) => entry.profile.architecture))).toEqual( + new Set(["amd64", "arm64"]), + ); + expect(new Set(qualification.cases.map((entry) => entry.profile.acceleration))).toEqual( + new Set(["cpu", "nvidia-gpu"]), + ); + expect(new Set(qualification.cases.map((entry) => entry.inference))).toEqual( + new Set(["ollama", "nim", "vllm"]), + ); + for (const entry of qualification.cases) { + expect(entry).toMatchObject({ + gate: "protected-e2e", + install: "release-installer", + dockerAvailability: "unavailable", + profile: { + platform: "linux", + rootMode: "rootless", + provider: "podman", + }, + }); + expect(entry.profile.capabilities).toContain("transport.socket-free"); + expect(entry.profile.capabilities).not.toContain("transport.docker-socket"); + expect(entry.obligations).toEqual( + expect.arrayContaining([ + "installer.install", + "runtime.docker-unavailable", + "agent.turn", + "sandbox.stop-start", + "sandbox.snapshot-restore", + "sandbox.rebuild", + "runtime.restart-reconcile", + "cleanup.exact", + ]), + ); + expect(hasRegisteredRuntimeProfile(entry.profile.id)).toBe(false); + } + }); + + it("accepts an MXC-style provider without a provider-specific compiler branch", () => { + const mxc = compileNativeRuntimeQualification( + nativeRuntimeQualificationDefinition("test-mxc-native"), + ); + + expect(mxc.cases).toHaveLength(PODMAN_NATIVE_ACTIVATION_QUALIFICATION.cases.length); + expect(new Set(mxc.cases.map((entry) => entry.profile.provider))).toEqual( + new Set(["test-mxc-native"]), + ); + expect(mxc.engineName).toBe("test-mxc-native"); + expect(mxc.cases.every((entry) => entry.id.startsWith("test-mxc-native-"))).toBe(true); + }); + + it("fails closed when one required case is missing", () => { + const definition = nativeRuntimeQualificationDefinition("missing-case-runtime"); + + expect(() => + compileNativeRuntimeQualification({ + ...definition, + cases: definition.cases.slice(1), + }), + ).toThrow(/coverage is incomplete.*missing/u); + }); + + it("fails closed when a case omits lifecycle or exact-evidence requirements", () => { + const missingLifecycle = nativeRuntimeQualificationDefinition("missing-lifecycle-runtime"); + const firstLifecycle = missingLifecycle.cases[0]!; + expect(() => + compileNativeRuntimeQualification({ + ...missingLifecycle, + cases: [ + { + ...firstLifecycle, + obligations: firstLifecycle.obligations.filter( + (entry) => entry !== "runtime.restart-reconcile", + ), + }, + ...missingLifecycle.cases.slice(1), + ], + }), + ).toThrow(/obligations is incomplete/u); + + const missingEvidence = nativeRuntimeQualificationDefinition("missing-evidence-runtime"); + const firstEvidence = missingEvidence.cases[0]!; + expect(() => + compileNativeRuntimeQualification({ + ...missingEvidence, + cases: [ + { + ...firstEvidence, + evidenceKinds: firstEvidence.evidenceKinds.filter( + (entry) => entry !== "installer-result", + ), + }, + ...missingEvidence.cases.slice(1), + ], + }), + ).toThrow(/evidence kinds is incomplete/u); + }); + + it("rejects Docker availability and Docker-socket substitutions", () => { + const dockerPresent = nativeRuntimeQualificationDefinition("docker-present-runtime"); + const first = dockerPresent.cases[0]!; + expect(() => + compileNativeRuntimeQualification({ + ...dockerPresent, + cases: [ + { ...first, dockerAvailability: "available" }, + ...dockerPresent.cases.slice(1), + ] as typeof dockerPresent.cases, + }), + ).toThrow(/must prove Docker is unavailable/u); + + const dockerSocket = nativeRuntimeQualificationDefinition("docker-socket-runtime"); + const firstSocket = dockerSocket.cases[0]!; + expect(() => + compileNativeRuntimeQualification({ + ...dockerSocket, + cases: [ + { + ...firstSocket, + profile: { + ...firstSocket.profile, + capabilities: [...firstSocket.profile.capabilities, "transport.docker-socket"], + }, + }, + ...dockerSocket.cases.slice(1), + ], + }), + ).toThrow(/must be socket-free/u); + }); + + it("accepts only a complete exact evidence set for the compiled candidate", () => { + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + completeEvidence(), + ), + ).not.toThrow(); + + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + completeEvidence().slice(1), + ), + ).toThrow(/evidence is incomplete/u); + }); + + it("rejects inexact source, image, operation, and CDI receipts", () => { + const badSource = completeEvidence(); + badSource[0]!.protectedRun.headSha = "main"; + expect(() => + assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, badSource), + ).toThrow(/exact head\/base SHAs/u); + + const wrongWorkflow = completeEvidence(); + wrongWorkflow[0]!.protectedRun.workflow = "Unprotected runtime test"; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + wrongWorkflow, + ), + ).toThrow(/wrong workflow/u); + + const mixedSourcePair = completeEvidence(); + mixedSourcePair[0]!.protectedRun.headSha = "3".repeat(40); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + mixedSourcePair, + ), + ).toThrow(/one exact head\/base pair/u); + + const badImage = completeEvidence(); + badImage[0]!.runtime.managedImages = [ + { role: "agent", imageRef: "nvcr.io/nvidia/nemoclaw-agent:latest" }, + { role: "probe", imageRef: IMAGE_REFS.probe }, + ]; + expect(() => + assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, badImage), + ).toThrow(/exact image references/u); + + const badOperations = completeEvidence(); + badOperations[0]!.operations = badOperations[0]!.operations.slice(1); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + badOperations, + ), + ).toThrow(/operations is incomplete/u); + + const gpuEvidence = completeEvidence(); + const gpu = gpuEvidence.find((entry) => entry.runtime.acceleration === "nvidia-gpu")!; + gpu.nvidiaCdi = undefined; + expect(() => + assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, gpuEvidence), + ).toThrow(/must prove NVIDIA CDI access/u); + }); + + it("binds application, engine, endpoint, and managed-runtime identity", () => { + const wrongApplication = completeEvidence(); + wrongApplication[0]!.runtime.application = "hermes"; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + wrongApplication, + ), + ).toThrow(/invalid runtime identity/u); + + const wrongEngine = completeEvidence(); + wrongEngine[0]!.runtime.engineName = "docker"; + expect(() => + assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, wrongEngine), + ).toThrow(/wrong runtime engine/u); + + const missingAuthority = completeEvidence(); + missingAuthority[0]!.runtime.engineAuthority.authorityId = ""; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + missingAuthority, + ), + ).toThrow(/exact non-empty runtime identifier/u); + + const wrongEndpoint = completeEvidence(); + wrongEndpoint[0]!.runtime.route.endpoint.gatewayProviderBaseUrl = + "http://host.openshell.internal:9000/v1"; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + wrongEndpoint, + ), + ).toThrow(/mismatched gateway endpoint/u); + + const missingManagedImage = completeEvidence(); + const managedImageCase = missingManagedImage.find( + (entry) => entry.runtime.inference === "nim", + )!; + managedImageCase.runtime.managedImages = managedImageCase.runtime.managedImages.filter( + (image) => image.role !== "inference", + ); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + missingManagedImage, + ), + ).toThrow(/managed image roles is incomplete/u); + + const missingRuntimeId = completeEvidence(); + const managedRuntimeCase = missingRuntimeId.find( + (entry) => entry.runtime.inference === "vllm", + )!; + managedRuntimeCase.runtime.route.authority.runtimeId = null; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + missingRuntimeId, + ), + ).toThrow(/must name managed inference/u); + }); + + it("binds lifecycle, recovery, and exact cleanup to one durable authority", () => { + const wrongOperation = completeEvidence(); + wrongOperation[0]!.operations[0]!.authoritySha256 = "e".repeat(64); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + wrongOperation, + ), + ).toThrow(/different runtime authority/u); + + const wrongRecovery = completeEvidence(); + wrongRecovery[0]!.recovery.authoritySha256 = "e".repeat(64); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + wrongRecovery, + ), + ).toThrow(/invalid recovery evidence/u); + + const wrongCleanup = completeEvidence(); + const managedCleanupCase = wrongCleanup.find((entry) => entry.runtime.inference === "nim")!; + managedCleanupCase.cleanup.status = "retained-external"; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + wrongCleanup, + ), + ).toThrow(/invalid exact cleanup evidence/u); + + const residualRuntime = completeEvidence(); + residualRuntime[0]!.cleanup.providerOwnedRuntimeIds = ["podman-stale-container"]; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + residualRuntime, + ), + ).toThrow(/invalid exact cleanup evidence/u); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification-fixtures.ts b/test/e2e/support/native-runtime-qualification-fixtures.ts new file mode 100644 index 00000000000..ddd5cfcf811 --- /dev/null +++ b/test/e2e/support/native-runtime-qualification-fixtures.ts @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + compileNativeRuntimeQualification, + NATIVE_RUNTIME_QUALIFICATION_ACCELERATIONS, + NATIVE_RUNTIME_QUALIFICATION_AGENTS, + NATIVE_RUNTIME_QUALIFICATION_ARCHITECTURES, + NATIVE_RUNTIME_QUALIFICATION_INFERENCE, + NATIVE_RUNTIME_QUALIFICATION_OBLIGATIONS, + type NativeRuntimeQualificationDefinition, + qualificationCaseId, + requiredQualificationEvidenceKinds, +} from "../registry/activation-qualification.ts"; +import { + defineExecutionProfile, + type ExecutionAcceleration, + type ExecutionArchitecture, + type ExecutionProfile, + type ExecutionProviderId, + executionProviderId, +} from "../registry/execution-profile.ts"; + +const QUALIFICATION_CAPABILITIES = [ + "agent.configure", + "agent.turn", + "evidence.collect", + "sandbox.lifecycle", + "state.observe", + "transport.socket-free", +] as const; + +function profileId( + provider: ExecutionProviderId, + architecture: ExecutionArchitecture, + acceleration: ExecutionAcceleration, +): string { + return `${provider}-linux-${architecture}-${acceleration}`; +} + +function qualificationProfiles(provider: ExecutionProviderId): ExecutionProfile[] { + return NATIVE_RUNTIME_QUALIFICATION_ARCHITECTURES.flatMap((architecture) => + NATIVE_RUNTIME_QUALIFICATION_ACCELERATIONS.map((acceleration) => + defineExecutionProfile({ + id: profileId(provider, architecture, acceleration), + provider, + platform: "linux", + architecture, + rootMode: "rootless", + acceleration, + capabilities: [...QUALIFICATION_CAPABILITIES], + runner: { + hostId: `protected-${architecture}-${acceleration}`, + label: `protected Linux ${architecture} ${acceleration}`, + maxShards: 1, + }, + }), + ), + ); +} + +export function nativeRuntimeQualificationDefinition( + providerName: string, +): NativeRuntimeQualificationDefinition { + const provider = executionProviderId(providerName); + const profiles = qualificationProfiles(provider); + const cases = NATIVE_RUNTIME_QUALIFICATION_AGENTS.flatMap((agent) => + profiles.flatMap((profile) => + NATIVE_RUNTIME_QUALIFICATION_INFERENCE[profile.acceleration].map((inference) => ({ + id: qualificationCaseId({ + provider, + agent, + architecture: profile.architecture, + acceleration: profile.acceleration, + inference, + }), + agent, + profile, + inference, + gate: "protected-e2e" as const, + install: "release-installer" as const, + dockerAvailability: "unavailable" as const, + obligations: [...NATIVE_RUNTIME_QUALIFICATION_OBLIGATIONS], + evidenceKinds: [...requiredQualificationEvidenceKinds(profile.acceleration)], + })), + ), + ); + return { + id: `${provider}-native-activation`, + repository: "NVIDIA/NemoClaw", + protectedWorkflow: "E2E / PR Gate", + provider, + engineName: provider, + cases, + }; +} + +/** + * Dormant candidate contract. Importing this fixture compiles scope; it does + * not register a production runtime, a canonical live target, or a workflow. + */ +export const PODMAN_NATIVE_ACTIVATION_QUALIFICATION = compileNativeRuntimeQualification( + nativeRuntimeQualificationDefinition("podman"), +); From da403b07fd3edeaf3daea0693bf60e26bd222743 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 09:15:17 -0700 Subject: [PATCH 02/10] test(e2e): bind qualification to workflow revision Signed-off-by: Aaron Erickson --- test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md | 5 +++-- test/e2e/registry/activation-qualification.ts | 15 +++++++++++-- .../e2e-native-runtime-qualification.test.ts | 21 ++++++++++++++++++- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md index e4a53ce983e..51c35a56ec5 100644 --- a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md +++ b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md @@ -35,7 +35,8 @@ or `langchain-deepagents-code` for DCode. Activation evidence is complete only when every compiled case has: -- the exact protected workflow run, job, attempt, head SHA, and base SHA; +- the exact protected workflow revision, run, job, attempt, head SHA, and base + SHA; - hashed installer script and invocation artifacts with a successful result; - an exact provider/profile/architecture/acceleration identity and persisted host-local engine authority; @@ -57,7 +58,7 @@ Activation evidence is complete only when every compiled case has: Evidence paths must be relative and traversal-free. SHA and SHA-256 fields are strict lowercase hexadecimal values. Missing, duplicate, unknown, or inexact case evidence fails the aggregate qualification check. All cases must use the -configured protected workflow and one exact head/base pair. +configured protected workflow and one exact head/base/workflow source. These are evidence requirements, not generated evidence. A later protected collector must publish the receipts from real runners before activation can diff --git a/test/e2e/registry/activation-qualification.ts b/test/e2e/registry/activation-qualification.ts index aab61dda141..4a14116bba1 100644 --- a/test/e2e/registry/activation-qualification.ts +++ b/test/e2e/registry/activation-qualification.ts @@ -120,6 +120,8 @@ export interface NativeRuntimeQualificationEvidence { protectedRun: { repository: string; workflow: string; + /** Exact trusted revision from which GitHub loaded the protected workflow. */ + workflowSha: string; runId: number; attempt: number; jobId: number; @@ -576,6 +578,11 @@ function assertCaseEvidence( if (evidence.protectedRun.workflow !== definition.protectedWorkflow) { throw new Error(`Qualification evidence '${evidence.caseId}' belongs to the wrong workflow`); } + if (!SHA_PATTERN.test(evidence.protectedRun.workflowSha)) { + throw new Error( + `Qualification evidence '${evidence.caseId}' must name the exact protected workflow SHA`, + ); + } assertPositiveInteger(evidence.protectedRun.runId, "Protected run id"); assertPositiveInteger(evidence.protectedRun.attempt, "Protected run attempt"); assertPositiveInteger(evidence.protectedRun.jobId, "Protected run job id"); @@ -669,7 +676,9 @@ export function assertNativeRuntimeQualificationEvidence( throw new Error(`Native runtime qualification evidence names unknown case '${entry.caseId}'`); } assertCaseEvidence(definition, qualificationCase, entry); - sourcePairs.add(`${entry.protectedRun.headSha}:${entry.protectedRun.baseSha}`); + sourcePairs.add( + `${entry.protectedRun.headSha}:${entry.protectedRun.baseSha}:${entry.protectedRun.workflowSha}`, + ); evidenceById.set(entry.caseId, entry); } const missing = definition.cases.filter((entry) => !evidenceById.has(entry.id)); @@ -679,6 +688,8 @@ export function assertNativeRuntimeQualificationEvidence( ); } if (sourcePairs.size !== 1) { - throw new Error("Native runtime qualification evidence must use one exact head/base pair"); + throw new Error( + "Native runtime qualification evidence must use one exact head/base/workflow source", + ); } } diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index 433ab5f633d..c9d20cecc74 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -45,6 +45,7 @@ function completeEvidence(): NativeRuntimeQualificationEvidence[] { protectedRun: { repository: "NVIDIA/NemoClaw", workflow: "E2E / PR Gate", + workflowSha: BASE_SHA, runId: 1000 + index, attempt: 1, jobId: 2000 + index, @@ -301,6 +302,15 @@ describe("native runtime activation qualification", () => { ), ).toThrow(/wrong workflow/u); + const wrongWorkflowSource = structuredClone(completeEvidence()); + wrongWorkflowSource[0]!.protectedRun.workflowSha = "main"; + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + wrongWorkflowSource, + ), + ).toThrow(/protected workflow SHA/u); + const mixedSourcePair = completeEvidence(); mixedSourcePair[0]!.protectedRun.headSha = "3".repeat(40); expect(() => @@ -308,7 +318,16 @@ describe("native runtime activation qualification", () => { PODMAN_NATIVE_ACTIVATION_QUALIFICATION, mixedSourcePair, ), - ).toThrow(/one exact head\/base pair/u); + ).toThrow(/one exact head\/base\/workflow source/u); + + const mixedWorkflowSource = structuredClone(completeEvidence()); + mixedWorkflowSource[0]!.protectedRun.workflowSha = "4".repeat(40); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + mixedWorkflowSource, + ), + ).toThrow(/one exact head\/base\/workflow source/u); const badImage = completeEvidence(); badImage[0]!.runtime.managedImages = [ From 350f0750a1078fec42994276f38b2a043523494b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 09:41:01 -0700 Subject: [PATCH 03/10] test(e2e): authenticate qualification evidence Signed-off-by: Aaron Erickson --- test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md | 20 +- test/e2e/registry/activation-qualification.ts | 342 +++++++++++++++++- .../e2e-native-runtime-qualification.test.ts | 219 ++++++++++- 3 files changed, 549 insertions(+), 32 deletions(-) diff --git a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md index 51c35a56ec5..5c3f947074f 100644 --- a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md +++ b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md @@ -60,9 +60,25 @@ strict lowercase hexadecimal values. Missing, duplicate, unknown, or inexact case evidence fails the aggregate qualification check. All cases must use the configured protected workflow and one exact head/base/workflow source. +Raw worker receipts are never sufficient. The trusted protected-E2E controller +must supply one or more independent bindings constructed from authenticated +GitHub state. Each binding names the repository, workflow revision, run, +attempt, exact head/base pair, numeric job, and that job's downloaded artifact +root; bindings must not be derived from candidate receipt fields. The compiler +defensively clones and freezes the receipts, requires every receipt and binding +to match exactly, and returns a runtime-branded canonical reporter record. + +The aggregate validator then resolves every receipt below its bound artifact +root, rejects missing, escaping, linked, conflicting, changing, or oversized +files, and hashes the actual bytes before comparing the claimed SHA-256 digest. +Only a separately branded verified-evidence object can reach final acceptance. +A syntactically complete receipt with invented provenance or artifact digests +therefore cannot qualify a runtime. + These are evidence requirements, not generated evidence. A later protected -collector must publish the receipts from real runners before activation can -consume them. +collector must publish the receipts from real runners, while its trusted +controller constructs the reporter from authenticated GitHub run/job state, +before activation can consume them. ## Activation boundary diff --git a/test/e2e/registry/activation-qualification.ts b/test/e2e/registry/activation-qualification.ts index 4a14116bba1..23d7bc93011 100644 --- a/test/e2e/registry/activation-qualification.ts +++ b/test/e2e/registry/activation-qualification.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + import { defineExecutionProfile, type ExecutionAcceleration, @@ -114,20 +118,22 @@ export interface QualificationArtifactReceipt { sha256: string; } +export interface NativeRuntimeQualificationProtectedRun { + repository: string; + workflow: string; + /** Exact trusted revision from which GitHub loaded the protected workflow. */ + workflowSha: string; + runId: number; + attempt: number; + jobId: number; + headSha: string; + baseSha: string; +} + export interface NativeRuntimeQualificationEvidence { schemaVersion: 1; caseId: string; - protectedRun: { - repository: string; - workflow: string; - /** Exact trusted revision from which GitHub loaded the protected workflow. */ - workflowSha: string; - runId: number; - attempt: number; - jobId: number; - headSha: string; - baseSha: string; - }; + protectedRun: NativeRuntimeQualificationProtectedRun; installer: { provider: ExecutionProviderId; architecture: ExecutionArchitecture; @@ -201,7 +207,29 @@ export interface NativeRuntimeQualificationEvidence { }; } +export interface NativeRuntimeQualificationProtectedRunBinding { + readonly protectedRun: NativeRuntimeQualificationProtectedRun; + readonly artifactRoot: string; +} + +export interface NativeRuntimeQualificationReporterRecord { + readonly qualificationId: string; + readonly caseIds: readonly string[]; +} + +export interface VerifiedNativeRuntimeQualificationEvidence { + readonly qualificationId: string; + readonly caseIds: readonly string[]; +} + const compiledQualifications = new WeakSet(); +interface CompiledReporterState { + readonly definition: CompiledNativeRuntimeQualification; + readonly evidence: readonly NativeRuntimeQualificationEvidence[]; + readonly artifactRootByCaseId: ReadonlyMap; +} +const compiledReporters = new WeakMap(); +const verifiedEvidence = new WeakMap(); const SHA_PATTERN = /^[a-f0-9]{40}$/u; const SHA256_PATTERN = /^[a-f0-9]{64}$/u; const IMAGE_REFERENCE_PATTERN = @@ -209,6 +237,7 @@ const IMAGE_REFERENCE_PATTERN = const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; const SAFE_HOST_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$/u; const SAFE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/=+-]{0,511}$/u; +const MAX_QUALIFICATION_ARTIFACT_BYTES = 64 * 1024 * 1024; const APPLICATION_BY_AGENT = { openclaw: "openclaw", @@ -224,6 +253,28 @@ function assertSingleLine(value: string, label: string): string { return normalized; } +function exactRecord( + value: unknown, + keys: readonly string[], + label: string, +): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + const record = value as Record; + if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { + throw new Error(`${label} schema is unsupported`); + } + return record; +} + +function deepFreeze(value: T, seen = new WeakSet()): T { + if (typeof value !== "object" || value === null || seen.has(value)) return value; + seen.add(value); + for (const child of Object.values(value)) deepFreeze(child, seen); + return Object.freeze(value); +} + function assertExactSet( actual: readonly T[], expected: readonly T[], @@ -257,16 +308,15 @@ export function qualificationCaseId(input: { acceleration: ExecutionAcceleration; inference: LocalInferenceProvider; }): string { + const acceleration = input.acceleration === "nvidia-gpu" ? "gpu" : input.acceleration; return [ input.provider, input.agent, "linux", input.architecture, - input.acceleration, + acceleration, input.inference, - ] - .join("-") - .replace("nvidia-gpu", "gpu"); + ].join("-"); } function coverageKey(input: { @@ -414,6 +464,252 @@ function assertArtifact(receipt: QualificationArtifactReceipt, label: string): v } } +function sameArtifactMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.mode === right.mode && + left.nlink === right.nlink && + left.uid === right.uid && + left.gid === right.gid && + left.size === right.size && + left.mtimeNs === right.mtimeNs && + left.ctimeNs === right.ctimeNs + ); +} + +function verifyArtifactContents( + artifactRoot: string, + receipt: QualificationArtifactReceipt, + claimedPaths: Map, +): void { + assertArtifact(receipt, "Qualification artifact"); + const candidate = path.join(artifactRoot, ...receipt.path.split(/[\\/]/u)); + let target: string; + try { + target = fs.realpathSync(candidate); + } catch { + throw new Error(`Qualification artifact '${receipt.path}' is missing`); + } + const relative = path.relative(artifactRoot, target); + if (relative === "" || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`Qualification artifact '${receipt.path}' escapes its trusted root`); + } + const candidateMetadata = fs.lstatSync(candidate); + if (!candidateMetadata.isFile() || candidateMetadata.isSymbolicLink()) { + throw new Error(`Qualification artifact '${receipt.path}' must be a real regular file`); + } + const previousDigest = claimedPaths.get(target); + if (previousDigest !== undefined) { + if (previousDigest !== receipt.sha256) { + throw new Error( + `Qualification evidence reuses artifact '${receipt.path}' with a different digest`, + ); + } + return; + } + claimedPaths.set(target, receipt.sha256); + const noFollow = fs.constants.O_NOFOLLOW; + if (typeof noFollow !== "number") { + throw new Error("Qualification artifact verification requires O_NOFOLLOW"); + } + let descriptor: number; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow); + } catch { + throw new Error(`Qualification artifact '${receipt.path}' is not a readable regular file`); + } + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if ( + !before.isFile() || + before.nlink !== 1n || + before.size < 1n || + before.size > BigInt(MAX_QUALIFICATION_ARTIFACT_BYTES) + ) { + throw new Error(`Qualification artifact '${receipt.path}' failed type, link, or size checks`); + } + const contents = Buffer.alloc(Number(before.size)); + let offset = 0; + while (offset < contents.length) { + const count = fs.readSync(descriptor, contents, offset, contents.length - offset, offset); + if (count === 0) break; + offset += count; + } + const overflow = Buffer.alloc(1); + const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (offset !== contents.length || overflowCount !== 0 || !sameArtifactMetadata(before, after)) { + throw new Error(`Qualification artifact '${receipt.path}' changed during verification`); + } + const actual = createHash("sha256").update(contents).digest("hex"); + if (actual !== receipt.sha256) { + throw new Error(`Qualification artifact '${receipt.path}' digest does not match its receipt`); + } + } finally { + fs.closeSync(descriptor); + } +} + +function qualificationArtifacts( + evidence: NativeRuntimeQualificationEvidence, +): readonly QualificationArtifactReceipt[] { + return [ + evidence.installer.invocation, + evidence.installer.script, + evidence.runtime.inferenceResult, + ...evidence.operations.map((operation) => operation.artifact), + evidence.recovery.artifact, + evidence.cleanup.artifact, + ...(evidence.nvidiaCdi ? [evidence.nvidiaCdi.artifact] : []), + ]; +} + +function exactReporterRun( + definition: CompiledNativeRuntimeQualification, + input: NativeRuntimeQualificationProtectedRun, +): Readonly { + if ( + input.repository !== definition.repository || + input.workflow !== definition.protectedWorkflow + ) { + throw new Error("Native runtime qualification reporter names the wrong protected workflow"); + } + if ( + !SHA_PATTERN.test(input.workflowSha) || + !SHA_PATTERN.test(input.headSha) || + !SHA_PATTERN.test(input.baseSha) || + input.headSha === input.baseSha + ) { + throw new Error( + "Native runtime qualification reporter must name exact workflow/head/base SHAs", + ); + } + assertPositiveInteger(input.runId, "Qualification reporter run id"); + assertPositiveInteger(input.attempt, "Qualification reporter run attempt"); + assertPositiveInteger(input.jobId, "Qualification reporter job id"); + return Object.freeze({ ...input }); +} + +function protectedRunKey(input: NativeRuntimeQualificationProtectedRun): string { + return JSON.stringify(input); +} + +/** + * Bind worker-authored evidence to run/job identities and artifact roots + * supplied independently by the trusted protected-E2E controller. The caller + * must construct bindings from authenticated control-plane state, never from + * candidate receipt fields. + */ +export function createNativeRuntimeQualificationReporterRecord( + definition: CompiledNativeRuntimeQualification, + evidence: readonly NativeRuntimeQualificationEvidence[], + bindings: readonly NativeRuntimeQualificationProtectedRunBinding[], +): NativeRuntimeQualificationReporterRecord { + if (!compiledQualifications.has(definition)) { + throw new Error("Native runtime qualification reporter requires a compiled definition"); + } + if (!Array.isArray(bindings) || bindings.length === 0) { + throw new Error("Native runtime qualification reporter requires trusted run bindings"); + } + const clonedEvidence = deepFreeze( + structuredClone(evidence) as NativeRuntimeQualificationEvidence[], + ); + validateNativeRuntimeQualificationEvidence(definition, clonedEvidence); + + const bindingByRun = new Map< + string, + { readonly artifactRoot: string; readonly protectedRun: NativeRuntimeQualificationProtectedRun } + >(); + for (const candidate of bindings) { + const binding = exactRecord( + candidate, + ["artifactRoot", "protectedRun"], + "Native runtime qualification protected-run binding", + ); + const runRecord = exactRecord( + binding.protectedRun, + ["attempt", "baseSha", "headSha", "jobId", "repository", "runId", "workflow", "workflowSha"], + "Native runtime qualification protected run", + ); + const protectedRun = exactReporterRun( + definition, + runRecord as unknown as NativeRuntimeQualificationProtectedRun, + ); + if (typeof binding.artifactRoot !== "string" || binding.artifactRoot === "") { + throw new Error("Native runtime qualification binding artifact root is invalid"); + } + const metadata = fs.lstatSync(binding.artifactRoot); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new Error("Native runtime qualification artifact root must be a real directory"); + } + const key = protectedRunKey(protectedRun); + if (bindingByRun.has(key)) { + throw new Error("Native runtime qualification reporter repeats a protected-run binding"); + } + bindingByRun.set(key, { + artifactRoot: fs.realpathSync(binding.artifactRoot), + protectedRun, + }); + } + + const usedBindings = new Set(); + const artifactRootByCaseId = new Map(); + for (const entry of clonedEvidence) { + const key = protectedRunKey(entry.protectedRun); + const binding = bindingByRun.get(key); + if (!binding) { + throw new Error( + `Qualification evidence '${entry.caseId}' has no trusted protected-run binding`, + ); + } + usedBindings.add(key); + artifactRootByCaseId.set(entry.caseId, binding.artifactRoot); + } + if (usedBindings.size !== bindingByRun.size) { + throw new Error("Native runtime qualification reporter has an unused protected-run binding"); + } + const reporter = Object.freeze({ + qualificationId: definition.id, + caseIds: Object.freeze(clonedEvidence.map((entry) => entry.caseId)), + }); + compiledReporters.set(reporter, { + definition, + evidence: clonedEvidence, + artifactRootByCaseId, + }); + return reporter; +} + +/** Re-hash every worker artifact before promoting a reporter to verified evidence. */ +export function verifyNativeRuntimeQualificationReporterArtifacts( + definition: CompiledNativeRuntimeQualification, + reporter: NativeRuntimeQualificationReporterRecord, +): VerifiedNativeRuntimeQualificationEvidence { + const state = compiledReporters.get(reporter); + if (state?.definition !== definition) { + throw new Error( + "Native runtime qualification artifacts require their canonical protected-workflow reporter", + ); + } + const claimedArtifactPaths = new Map(); + for (const entry of state.evidence) { + const artifactRoot = state.artifactRootByCaseId.get(entry.caseId); + if (!artifactRoot) { + throw new Error(`Qualification evidence '${entry.caseId}' lost its artifact binding`); + } + for (const artifact of qualificationArtifacts(entry)) { + verifyArtifactContents(artifactRoot, artifact, claimedArtifactPaths); + } + } + const verified = Object.freeze({ + qualificationId: definition.id, + caseIds: Object.freeze(state.evidence.map((entry) => entry.caseId)), + }); + verifiedEvidence.set(verified, state); + return verified; +} + function assertPositiveInteger(value: number, label: string): void { if (!Number.isSafeInteger(value) || value < 1) { throw new Error(`${label} must be a positive integer`); @@ -657,7 +953,7 @@ function assertCaseEvidence( } } -export function assertNativeRuntimeQualificationEvidence( +function validateNativeRuntimeQualificationEvidence( definition: CompiledNativeRuntimeQualification, evidence: readonly NativeRuntimeQualificationEvidence[], ): void { @@ -693,3 +989,17 @@ export function assertNativeRuntimeQualificationEvidence( ); } } + +/** Accept only evidence that crossed both trusted reporter and byte-verification boundaries. */ +export function assertNativeRuntimeQualificationEvidence( + definition: CompiledNativeRuntimeQualification, + evidence: VerifiedNativeRuntimeQualificationEvidence, +): void { + const state = verifiedEvidence.get(evidence); + if (state?.definition !== definition) { + throw new Error( + "Native runtime qualification evidence requires verified canonical reporter evidence", + ); + } + validateNativeRuntimeQualificationEvidence(definition, state.evidence); +} diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index c9d20cecc74..dcf2d4916a0 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -1,12 +1,22 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, it } from "vitest"; import { - assertNativeRuntimeQualificationEvidence, + assertNativeRuntimeQualificationEvidence as assertVerifiedNativeRuntimeQualificationEvidence, compileNativeRuntimeQualification, + createNativeRuntimeQualificationReporterRecord, type NativeRuntimeQualificationEvidence, + type NativeRuntimeQualificationProtectedRunBinding, + type QualificationArtifactReceipt, + type VerifiedNativeRuntimeQualificationEvidence, + verifyNativeRuntimeQualificationReporterArtifacts, } from "../registry/activation-qualification.ts"; import { hasRegisteredRuntimeProfile } from "../registry/registry.ts"; import { @@ -16,7 +26,8 @@ import { const HEAD_SHA = "1".repeat(40); const BASE_SHA = "2".repeat(40); -const ARTIFACT_SHA = "a".repeat(64); +const ARTIFACT_CONTENT = "verified native runtime qualification artifact\n"; +const ARTIFACT_SHA = createHash("sha256").update(ARTIFACT_CONTENT, "utf8").digest("hex"); const AUTHORITY_SHA = "b".repeat(64); const BINDING_SHA = "c".repeat(64); const SPEC_SHA = "d".repeat(64); @@ -132,6 +143,66 @@ function completeEvidence(): NativeRuntimeQualificationEvidence[] { }); } +function evidenceArtifacts( + evidence: readonly NativeRuntimeQualificationEvidence[], +): QualificationArtifactReceipt[] { + return evidence.flatMap((entry) => [ + entry.installer.invocation, + entry.installer.script, + entry.runtime.inferenceResult, + ...entry.operations.map((operation) => operation.artifact), + entry.recovery.artifact, + entry.cleanup.artifact, + ...(entry.nvidiaCdi ? [entry.nvidiaCdi.artifact] : []), + ]); +} + +function qualificationFixture(): { + artifactRoot: string; + bindings: NativeRuntimeQualificationProtectedRunBinding[]; + cleanup: () => void; +} { + const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-native-qualification-")); + return { + artifactRoot, + bindings: completeEvidence().map((entry) => ({ + protectedRun: structuredClone(entry.protectedRun), + artifactRoot, + })), + cleanup: () => fs.rmSync(artifactRoot, { force: true, recursive: true }), + }; +} + +function writeEvidenceArtifacts( + artifactRoot: string, + evidence: readonly NativeRuntimeQualificationEvidence[], +): void { + for (const receipt of evidenceArtifacts(evidence)) { + const target = path.join(artifactRoot, ...receipt.path.split(/[\\/]/u)); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, ARTIFACT_CONTENT, "utf8"); + } +} + +function assertNativeRuntimeQualificationEvidence( + definition: typeof PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence: readonly NativeRuntimeQualificationEvidence[], +): void { + const fixture = qualificationFixture(); + try { + const reporter = createNativeRuntimeQualificationReporterRecord( + definition, + evidence, + fixture.bindings, + ); + writeEvidenceArtifacts(fixture.artifactRoot, evidence); + const verified = verifyNativeRuntimeQualificationReporterArtifacts(definition, reporter); + assertVerifiedNativeRuntimeQualificationEvidence(definition, verified); + } finally { + fixture.cleanup(); + } +} + describe("native runtime activation qualification", () => { it("compiles every required all-agent, multiarch, CPU/GPU, and local-inference case", () => { const qualification = PODMAN_NATIVE_ACTIVATION_QUALIFICATION; @@ -162,18 +233,17 @@ describe("native runtime activation qualification", () => { }); expect(entry.profile.capabilities).toContain("transport.socket-free"); expect(entry.profile.capabilities).not.toContain("transport.docker-socket"); - expect(entry.obligations).toEqual( - expect.arrayContaining([ - "installer.install", - "runtime.docker-unavailable", - "agent.turn", - "sandbox.stop-start", - "sandbox.snapshot-restore", - "sandbox.rebuild", - "runtime.restart-reconcile", - "cleanup.exact", - ]), - ); + expect(entry.obligations).toEqual([ + "installer.install", + "runtime.docker-unavailable", + "agent.onboard", + "agent.turn", + "sandbox.stop-start", + "sandbox.snapshot-restore", + "sandbox.rebuild", + "runtime.restart-reconcile", + "cleanup.exact", + ]); expect(hasRegisteredRuntimeProfile(entry.profile.id)).toBe(false); } }); @@ -189,6 +259,18 @@ describe("native runtime activation qualification", () => { ); expect(mxc.engineName).toBe("test-mxc-native"); expect(mxc.cases.every((entry) => entry.id.startsWith("test-mxc-native-"))).toBe(true); + + const providerWithAccelerationText = compileNativeRuntimeQualification( + nativeRuntimeQualificationDefinition("nvidia-gpu-runtime"), + ); + expect( + providerWithAccelerationText.cases.every((entry) => + entry.id.startsWith("nvidia-gpu-runtime-"), + ), + ).toBe(true); + expect( + providerWithAccelerationText.cases.some((entry) => entry.id.includes("-amd64-gpu-ollama")), + ).toBe(true); }); it("fails closed when one required case is missing", () => { @@ -286,6 +368,115 @@ describe("native runtime activation qualification", () => { ).toThrow(/evidence is incomplete/u); }); + it("requires the canonical reporter and re-hashes every referenced artifact", () => { + expect(() => + assertVerifiedNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + completeEvidence() as unknown as VerifiedNativeRuntimeQualificationEvidence, + ), + ).toThrow(/verified canonical reporter evidence/u); + + const inventedDigest = completeEvidence(); + inventedDigest[0]!.installer.invocation.sha256 = "e".repeat(64); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + inventedDigest, + ), + ).toThrow(/digest does not match its receipt/u); + }); + + it("binds receipts to independent run metadata and freezes them before verification", () => { + const evidence = completeEvidence(); + const materialized = qualificationFixture(); + try { + const inventedRun = structuredClone(evidence); + inventedRun[0]!.protectedRun.runId = 999_999; + expect(() => + createNativeRuntimeQualificationReporterRecord( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + inventedRun, + materialized.bindings, + ), + ).toThrow(/no trusted protected-run binding/u); + + const reporter = createNativeRuntimeQualificationReporterRecord( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + materialized.bindings, + ); + evidence[0]!.installer.invocation.sha256 = "e".repeat(64); + writeEvidenceArtifacts(materialized.artifactRoot, completeEvidence()); + const verified = verifyNativeRuntimeQualificationReporterArtifacts( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + reporter, + ); + expect(() => + assertVerifiedNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + verified, + ), + ).not.toThrow(); + + const mxc = compileNativeRuntimeQualification(nativeRuntimeQualificationDefinition("mxc")); + expect(() => assertVerifiedNativeRuntimeQualificationEvidence(mxc, verified)).toThrow( + /verified canonical reporter evidence/u, + ); + } finally { + materialized.cleanup(); + } + }); + + it("rejects missing artifacts and symlink escapes from a bound job root", () => { + const missingEvidence = completeEvidence(); + const missing = qualificationFixture(); + try { + writeEvidenceArtifacts(missing.artifactRoot, missingEvidence); + const reporter = createNativeRuntimeQualificationReporterRecord( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + missingEvidence, + missing.bindings, + ); + const receipt = missingEvidence[0]!.installer.invocation; + fs.unlinkSync(path.join(missing.artifactRoot, ...receipt.path.split(/[\\/]/u))); + expect(() => + verifyNativeRuntimeQualificationReporterArtifacts( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + reporter, + ), + ).toThrow(/is missing/u); + } finally { + missing.cleanup(); + } + + const linkedEvidence = completeEvidence(); + const linked = qualificationFixture(); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-native-outside-")); + try { + writeEvidenceArtifacts(linked.artifactRoot, linkedEvidence); + const reporter = createNativeRuntimeQualificationReporterRecord( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + linkedEvidence, + linked.bindings, + ); + const receipt = linkedEvidence[0]!.installer.invocation; + const target = path.join(linked.artifactRoot, ...receipt.path.split(/[\\/]/u)); + const outsideFile = path.join(outside, "artifact.json"); + fs.writeFileSync(outsideFile, ARTIFACT_CONTENT, "utf8"); + fs.unlinkSync(target); + fs.symlinkSync(outsideFile, target); + expect(() => + verifyNativeRuntimeQualificationReporterArtifacts( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + reporter, + ), + ).toThrow(/escapes its trusted root/u); + } finally { + linked.cleanup(); + fs.rmSync(outside, { force: true, recursive: true }); + } + }); + it("rejects inexact source, image, operation, and CDI receipts", () => { const badSource = completeEvidence(); badSource[0]!.protectedRun.headSha = "main"; From b1b91105b871f099a7d1c21df07c4ce452c9b0fe Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 10:57:38 -0700 Subject: [PATCH 04/10] test(e2e): connect trusted qualification controller Signed-off-by: Aaron Erickson --- test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md | 15 ++-- test/e2e/registry/activation-qualification.ts | 11 ++- .../e2e-native-runtime-qualification.test.ts | 58 +++++++++++++- ...ative-runtime-qualification-controller.mts | 80 +++++++++++++++++++ 4 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 tools/e2e/native-runtime-qualification-controller.mts diff --git a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md index 5c3f947074f..a972ce5c95e 100644 --- a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md +++ b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md @@ -64,9 +64,13 @@ Raw worker receipts are never sufficient. The trusted protected-E2E controller must supply one or more independent bindings constructed from authenticated GitHub state. Each binding names the repository, workflow revision, run, attempt, exact head/base pair, numeric job, and that job's downloaded artifact -root; bindings must not be derived from candidate receipt fields. The compiler -defensively clones and freezes the receipts, requires every receipt and binding -to match exactly, and returns a runtime-branded canonical reporter record. +root; bindings must not be derived from candidate receipt fields. +`tools/e2e/native-runtime-qualification-controller.mts` is the named +controller-side acceptance adapter: callers provide authenticated run records +and jobs, and the adapter constructs bindings without accepting a worker-owned +binding argument. The compiler defensively clones and freezes the receipts, +requires every receipt and binding to match exactly, and returns a +runtime-branded canonical reporter record. The aggregate validator then resolves every receipt below its bound artifact root, rejects missing, escaping, linked, conflicting, changing, or oversized @@ -77,8 +81,9 @@ therefore cannot qualify a runtime. These are evidence requirements, not generated evidence. A later protected collector must publish the receipts from real runners, while its trusted -controller constructs the reporter from authenticated GitHub run/job state, -before activation can consume them. +controller supplies authenticated GitHub run/job state to the dormant +controller adapter before activation can consume them. No workflow invokes the +adapter in this slice. ## Activation boundary diff --git a/test/e2e/registry/activation-qualification.ts b/test/e2e/registry/activation-qualification.ts index 23d7bc93011..cd57c93c5e9 100644 --- a/test/e2e/registry/activation-qualification.ts +++ b/test/e2e/registry/activation-qualification.ts @@ -592,7 +592,16 @@ function exactReporterRun( } function protectedRunKey(input: NativeRuntimeQualificationProtectedRun): string { - return JSON.stringify(input); + return JSON.stringify([ + input.repository, + input.workflow, + input.workflowSha, + input.runId, + input.attempt, + input.jobId, + input.headSha, + input.baseSha, + ]); } /** diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index dcf2d4916a0..837ec1a27af 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -7,7 +7,10 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; - +import { + type AuthenticatedNativeRuntimeQualificationRun, + verifyNativeRuntimeQualificationFromTrustedController, +} from "../../../tools/e2e/native-runtime-qualification-controller.mts"; import { assertNativeRuntimeQualificationEvidence as assertVerifiedNativeRuntimeQualificationEvidence, compileNativeRuntimeQualification, @@ -173,6 +176,21 @@ function qualificationFixture(): { }; } +function authenticatedControllerRuns( + artifactRoot: string, +): AuthenticatedNativeRuntimeQualificationRun[] { + return PODMAN_NATIVE_ACTIVATION_QUALIFICATION.cases.map((_qualificationCase, index) => ({ + repository: "NVIDIA/NemoClaw", + workflow: "E2E / PR Gate", + workflowSha: BASE_SHA, + runId: 1000 + index, + attempt: 1, + headSha: HEAD_SHA, + baseSha: BASE_SHA, + jobs: [{ id: 2000 + index, artifactRoot }], + })); +} + function writeEvidenceArtifacts( artifactRoot: string, evidence: readonly NativeRuntimeQualificationEvidence[], @@ -386,6 +404,44 @@ describe("native runtime activation qualification", () => { ).toThrow(/digest does not match its receipt/u); }); + it("accepts only verified evidence bound from authenticated controller run jobs", () => { + const evidence = completeEvidence(); + const materialized = qualificationFixture(); + try { + writeEvidenceArtifacts(materialized.artifactRoot, evidence); + const authenticatedRuns = authenticatedControllerRuns(materialized.artifactRoot); + const verified = verifyNativeRuntimeQualificationFromTrustedController({ + definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + authenticatedRuns, + }); + expect(() => + assertVerifiedNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + verified, + ), + ).not.toThrow(); + expect(() => + assertVerifiedNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence as unknown as VerifiedNativeRuntimeQualificationEvidence, + ), + ).toThrow(/verified canonical reporter evidence/u); + + const workerInventedRun = structuredClone(evidence); + workerInventedRun[0]!.protectedRun.runId = 999_999; + expect(() => + verifyNativeRuntimeQualificationFromTrustedController({ + definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence: workerInventedRun, + authenticatedRuns, + }), + ).toThrow(/no trusted protected-run binding/u); + } finally { + materialized.cleanup(); + } + }); + it("binds receipts to independent run metadata and freezes them before verification", () => { const evidence = completeEvidence(); const materialized = qualificationFixture(); diff --git a/tools/e2e/native-runtime-qualification-controller.mts b/tools/e2e/native-runtime-qualification-controller.mts new file mode 100644 index 00000000000..970fe694bb0 --- /dev/null +++ b/tools/e2e/native-runtime-qualification-controller.mts @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + assertNativeRuntimeQualificationEvidence, + type CompiledNativeRuntimeQualification, + createNativeRuntimeQualificationReporterRecord, + type NativeRuntimeQualificationEvidence, + type NativeRuntimeQualificationProtectedRun, + type NativeRuntimeQualificationProtectedRunBinding, + type VerifiedNativeRuntimeQualificationEvidence, + verifyNativeRuntimeQualificationReporterArtifacts, +} from "../../test/e2e/registry/activation-qualification.ts"; + +export interface AuthenticatedNativeRuntimeQualificationJob { + /** GitHub Actions job ID read by the protected controller. */ + readonly id: number; + /** Controller-owned root of the downloaded artifact for this exact job. */ + readonly artifactRoot: string; +} + +export type AuthenticatedNativeRuntimeQualificationRun = Omit< + NativeRuntimeQualificationProtectedRun, + "jobId" +> & { + /** Jobs authenticated for this run through the GitHub control plane. */ + readonly jobs: readonly AuthenticatedNativeRuntimeQualificationJob[]; +}; + +export interface TrustedNativeRuntimeQualificationControllerInput { + readonly definition: CompiledNativeRuntimeQualification; + readonly evidence: readonly NativeRuntimeQualificationEvidence[]; + /** + * Control-plane records supplied by the protected controller after it has + * authenticated the workflow run and jobs. Never derive these from worker + * evidence or downloaded artifact contents. + */ + readonly authenticatedRuns: readonly AuthenticatedNativeRuntimeQualificationRun[]; +} + +function controllerBindings( + runs: readonly AuthenticatedNativeRuntimeQualificationRun[], +): NativeRuntimeQualificationProtectedRunBinding[] { + if (!Array.isArray(runs) || runs.length === 0) { + throw new Error("Native runtime qualification requires authenticated controller runs"); + } + return runs.flatMap(({ jobs, ...run }) => { + if (!Array.isArray(jobs) || jobs.length === 0) { + throw new Error("Native runtime qualification authenticated run has no jobs"); + } + return jobs.map((job) => ({ + artifactRoot: job.artifactRoot, + protectedRun: { ...run, jobId: job.id }, + })); + }); +} + +/** + * Trusted-controller acceptance boundary for native-runtime qualification. + * + * This adapter deliberately offers no caller-supplied binding argument. It + * constructs bindings only from authenticated run/job state, re-hashes every + * referenced artifact, and returns evidence only after the canonical + * acceptance assertion succeeds. The later activation workflow may call this + * adapter; defining it here does not activate a runtime provider. + */ +export function verifyNativeRuntimeQualificationFromTrustedController({ + definition, + evidence, + authenticatedRuns, +}: TrustedNativeRuntimeQualificationControllerInput): VerifiedNativeRuntimeQualificationEvidence { + const reporter = createNativeRuntimeQualificationReporterRecord( + definition, + evidence, + controllerBindings(authenticatedRuns), + ); + const verified = verifyNativeRuntimeQualificationReporterArtifacts(definition, reporter); + assertNativeRuntimeQualificationEvidence(definition, verified); + return verified; +} From a3812f5bf3be90456470ee89cdc377b7906d0c30 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 11:22:24 -0700 Subject: [PATCH 05/10] fix(e2e): stabilize qualification artifact trust Signed-off-by: Aaron Erickson --- test/e2e/registry/activation-qualification.ts | 184 +++++++++++++++--- .../e2e-native-runtime-qualification.test.ts | 110 ++++++++++- 2 files changed, 264 insertions(+), 30 deletions(-) diff --git a/test/e2e/registry/activation-qualification.ts b/test/e2e/registry/activation-qualification.ts index cd57c93c5e9..3ffe7b03300 100644 --- a/test/e2e/registry/activation-qualification.ts +++ b/test/e2e/registry/activation-qualification.ts @@ -238,6 +238,16 @@ const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; const SAFE_HOST_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$/u; const SAFE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/=+-]{0,511}$/u; const MAX_QUALIFICATION_ARTIFACT_BYTES = 64 * 1024 * 1024; +const PROTECTED_RUN_KEYS = [ + "attempt", + "baseSha", + "headSha", + "jobId", + "repository", + "runId", + "workflow", + "workflowSha", +] as const; const APPLICATION_BY_AGENT = { openclaw: "openclaw", @@ -452,10 +462,11 @@ export function compileNativeRuntimeQualification( function assertArtifact(receipt: QualificationArtifactReceipt, label: string): void { const artifactPath = assertSingleLine(receipt.path, `${label} path`); + const components = artifactPath.split(/[\\/]/u); if ( artifactPath.startsWith("/") || artifactPath.startsWith("\\") || - artifactPath.split(/[\\/]/u).some((part) => part === "..") + components.some((part) => part === "" || part === "." || part === "..") ) { throw new Error(`${label} path must be repository-relative and traversal-free`); } @@ -478,13 +489,19 @@ function sameArtifactMetadata(left: fs.BigIntStats, right: fs.BigIntStats): bool ); } -function verifyArtifactContents( +interface ArtifactPathSnapshot { + readonly candidate: string; + readonly components: readonly { + readonly path: string; + readonly metadata: fs.BigIntStats; + }[]; +} + +function resolvedArtifactTarget( artifactRoot: string, + candidate: string, receipt: QualificationArtifactReceipt, - claimedPaths: Map, -): void { - assertArtifact(receipt, "Qualification artifact"); - const candidate = path.join(artifactRoot, ...receipt.path.split(/[\\/]/u)); +): string { let target: string; try { target = fs.realpathSync(candidate); @@ -495,27 +512,93 @@ function verifyArtifactContents( if (relative === "" || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { throw new Error(`Qualification artifact '${receipt.path}' escapes its trusted root`); } - const candidateMetadata = fs.lstatSync(candidate); - if (!candidateMetadata.isFile() || candidateMetadata.isSymbolicLink()) { - throw new Error(`Qualification artifact '${receipt.path}' must be a real regular file`); + return target; +} + +function snapshotArtifactPath( + artifactRoot: string, + receipt: QualificationArtifactReceipt, +): ArtifactPathSnapshot { + const paths = [artifactRoot]; + for (const component of receipt.path.split(/[\\/]/u)) { + paths.push(path.join(paths.at(-1)!, component)); + } + let components: ArtifactPathSnapshot["components"]; + try { + components = paths.map((componentPath) => ({ + path: componentPath, + metadata: fs.lstatSync(componentPath, { bigint: true }), + })); + } catch { + throw new Error(`Qualification artifact '${receipt.path}' is missing`); } - const previousDigest = claimedPaths.get(target); - if (previousDigest !== undefined) { - if (previousDigest !== receipt.sha256) { + const candidate = paths.at(-1)!; + for (const [index, component] of components.entries()) { + const leaf = index === components.length - 1; + if (component.metadata.isSymbolicLink()) { + resolvedArtifactTarget(artifactRoot, candidate, receipt); + throw new Error( + `Qualification artifact '${receipt.path}' must use stable real path components`, + ); + } + if ((!leaf && !component.metadata.isDirectory()) || (leaf && !component.metadata.isFile())) { throw new Error( - `Qualification evidence reuses artifact '${receipt.path}' with a different digest`, + `Qualification artifact '${receipt.path}' must be below real directories in a regular file`, ); } - return; } - claimedPaths.set(target, receipt.sha256); + return { candidate, components }; +} + +function sameArtifactPathSnapshot( + left: ArtifactPathSnapshot, + right: ArtifactPathSnapshot, +): boolean { + return ( + left.candidate === right.candidate && + left.components.length === right.components.length && + left.components.every( + (component, index) => + component.path === right.components[index]?.path && + sameArtifactMetadata(component.metadata, right.components[index]!.metadata), + ) + ); +} + +function assertStableArtifactPath( + artifactRoot: string, + receipt: QualificationArtifactReceipt, + initial: ArtifactPathSnapshot, + opened: fs.BigIntStats, +): string { + const current = snapshotArtifactPath(artifactRoot, receipt); + resolvedArtifactTarget(artifactRoot, current.candidate, receipt); + const confirmed = snapshotArtifactPath(artifactRoot, receipt); + const confirmedLeaf = confirmed.components.at(-1)!.metadata; + if ( + !sameArtifactPathSnapshot(initial, current) || + !sameArtifactPathSnapshot(current, confirmed) || + !sameArtifactMetadata(opened, confirmedLeaf) + ) { + throw new Error(`Qualification artifact '${receipt.path}' changed during verification`); + } + return `${opened.dev}:${opened.ino}`; +} + +function verifyArtifactContents( + artifactRoot: string, + receipt: QualificationArtifactReceipt, + claimedPaths: Map, +): void { + assertArtifact(receipt, "Qualification artifact"); + const pathSnapshot = snapshotArtifactPath(artifactRoot, receipt); const noFollow = fs.constants.O_NOFOLLOW; if (typeof noFollow !== "number") { throw new Error("Qualification artifact verification requires O_NOFOLLOW"); } let descriptor: number; try { - descriptor = fs.openSync(target, fs.constants.O_RDONLY | noFollow); + descriptor = fs.openSync(pathSnapshot.candidate, fs.constants.O_RDONLY | noFollow); } catch { throw new Error(`Qualification artifact '${receipt.path}' is not a readable regular file`); } @@ -529,6 +612,17 @@ function verifyArtifactContents( ) { throw new Error(`Qualification artifact '${receipt.path}' failed type, link, or size checks`); } + const artifactIdentity = assertStableArtifactPath(artifactRoot, receipt, pathSnapshot, before); + const previousDigest = claimedPaths.get(artifactIdentity); + if (previousDigest !== undefined) { + if (previousDigest !== receipt.sha256) { + throw new Error( + `Qualification evidence reuses artifact '${receipt.path}' with a different digest`, + ); + } + return; + } + claimedPaths.set(artifactIdentity, receipt.sha256); const contents = Buffer.alloc(Number(before.size)); let offset = 0; while (offset < contents.length) { @@ -539,7 +633,12 @@ function verifyArtifactContents( const overflow = Buffer.alloc(1); const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); const after = fs.fstatSync(descriptor, { bigint: true }); - if (offset !== contents.length || overflowCount !== 0 || !sameArtifactMetadata(before, after)) { + if ( + offset !== contents.length || + overflowCount !== 0 || + !sameArtifactMetadata(before, after) || + assertStableArtifactPath(artifactRoot, receipt, pathSnapshot, after) !== artifactIdentity + ) { throw new Error(`Qualification artifact '${receipt.path}' changed during verification`); } const actual = createHash("sha256").update(contents).digest("hex"); @@ -638,7 +737,7 @@ export function createNativeRuntimeQualificationReporterRecord( ); const runRecord = exactRecord( binding.protectedRun, - ["attempt", "baseSha", "headSha", "jobId", "repository", "runId", "workflow", "workflowSha"], + PROTECTED_RUN_KEYS, "Native runtime qualification protected run", ); const protectedRun = exactReporterRun( @@ -665,7 +764,12 @@ export function createNativeRuntimeQualificationReporterRecord( const usedBindings = new Set(); const artifactRootByCaseId = new Map(); for (const entry of clonedEvidence) { - const key = protectedRunKey(entry.protectedRun); + const runRecord = exactRecord( + entry.protectedRun, + PROTECTED_RUN_KEYS, + `Qualification evidence '${entry.caseId}' protected run`, + ); + const key = protectedRunKey(runRecord as unknown as NativeRuntimeQualificationProtectedRun); const binding = bindingByRun.get(key); if (!binding) { throw new Error( @@ -869,14 +973,15 @@ function assertLifecycleEvidence( assertArtifact(evidence.cleanup.artifact, "Cleanup artifact"); } -function assertCaseEvidence( +function assertProtectedRunEvidence( definition: CompiledNativeRuntimeQualification, - qualificationCase: Readonly, evidence: NativeRuntimeQualificationEvidence, ): void { - if (evidence.schemaVersion !== 1) { - throw new Error(`Qualification evidence '${evidence.caseId}' has unsupported schemaVersion`); - } + exactRecord( + evidence.protectedRun, + PROTECTED_RUN_KEYS, + `Qualification evidence '${evidence.caseId}' protected run`, + ); if (evidence.protectedRun.repository !== definition.repository) { throw new Error(`Qualification evidence '${evidence.caseId}' belongs to the wrong repository`); } @@ -900,11 +1005,16 @@ function assertCaseEvidence( if (evidence.protectedRun.headSha === evidence.protectedRun.baseSha) { throw new Error(`Qualification evidence '${evidence.caseId}' head/base SHAs must differ`); } +} - const profile = qualificationCase.profile; +function assertInstallerReceipt( + definition: CompiledNativeRuntimeQualification, + qualificationCase: Readonly, + evidence: NativeRuntimeQualificationEvidence, +): void { if ( evidence.installer.provider !== definition.provider || - evidence.installer.architecture !== profile.architecture || + evidence.installer.architecture !== qualificationCase.profile.architecture || evidence.installer.dockerAvailability !== "unavailable" || evidence.installer.exitCode !== 0 ) { @@ -912,7 +1022,14 @@ function assertCaseEvidence( } assertArtifact(evidence.installer.invocation, "Installer invocation artifact"); assertArtifact(evidence.installer.script, "Installer script artifact"); +} +function assertRuntimeIdentity( + definition: CompiledNativeRuntimeQualification, + qualificationCase: Readonly, + evidence: NativeRuntimeQualificationEvidence, +): void { + const profile = qualificationCase.profile; if ( evidence.runtime.provider !== definition.provider || evidence.runtime.profileId !== profile.id || @@ -929,6 +1046,19 @@ function assertCaseEvidence( throw new Error(`Qualification evidence '${evidence.caseId}' names the wrong runtime engine`); } assertSingleLine(evidence.runtime.engineVersion, "Runtime engine version"); +} + +function assertCaseEvidence( + definition: CompiledNativeRuntimeQualification, + qualificationCase: Readonly, + evidence: NativeRuntimeQualificationEvidence, +): void { + if (evidence.schemaVersion !== 1) { + throw new Error(`Qualification evidence '${evidence.caseId}' has unsupported schemaVersion`); + } + assertProtectedRunEvidence(definition, evidence); + assertInstallerReceipt(definition, qualificationCase, evidence); + assertRuntimeIdentity(definition, qualificationCase, evidence); assertEngineAuthority(definition, evidence); const imageRoles: QualificationManagedImageRole[] = []; for (const image of evidence.runtime.managedImages) { @@ -949,7 +1079,7 @@ function assertCaseEvidence( assertArtifact(evidence.runtime.inferenceResult, "Inference result artifact"); assertLifecycleEvidence(qualificationCase, evidence, authoritySha256); - if (profile.acceleration === "nvidia-gpu") { + if (qualificationCase.profile.acceleration === "nvidia-gpu") { if ( evidence.nvidiaCdi?.devices.length !== 1 || evidence.nvidiaCdi.devices[0] !== "nvidia.com/gpu=all" diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index 837ec1a27af..4f7070bb1d0 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -6,7 +6,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { type AuthenticatedNativeRuntimeQualificationRun, verifyNativeRuntimeQualificationFromTrustedController, @@ -442,7 +442,51 @@ describe("native runtime activation qualification", () => { } }); - it("binds receipts to independent run metadata and freezes them before verification", () => { + it("binds one authenticated protected run to its exact matrix job set", () => { + const evidence = completeEvidence(); + const materialized = qualificationFixture(); + try { + const { jobId: _jobId, ...sharedRun } = structuredClone(evidence[0]!.protectedRun); + for (const entry of evidence) { + entry.protectedRun = { ...sharedRun, jobId: entry.protectedRun.jobId }; + } + const jobs = evidence.map((entry) => ({ + id: entry.protectedRun.jobId, + artifactRoot: materialized.artifactRoot, + })); + writeEvidenceArtifacts(materialized.artifactRoot, evidence); + expect(() => + verifyNativeRuntimeQualificationFromTrustedController({ + definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + authenticatedRuns: [{ ...sharedRun, jobs }], + }), + ).not.toThrow(); + expect(() => + verifyNativeRuntimeQualificationFromTrustedController({ + definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + authenticatedRuns: [{ ...sharedRun, jobs: jobs.slice(1) }], + }), + ).toThrow(/has no trusted protected-run binding/u); + expect(() => + verifyNativeRuntimeQualificationFromTrustedController({ + definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + authenticatedRuns: [ + { + ...sharedRun, + jobs: [...jobs, { id: 9999, artifactRoot: materialized.artifactRoot }], + }, + ], + }), + ).toThrow(/unused protected-run binding/u); + } finally { + materialized.cleanup(); + } + }); + + it("binds receipts to independent run metadata and snapshots them before verification", () => { const evidence = completeEvidence(); const materialized = qualificationFixture(); try { @@ -533,6 +577,56 @@ describe("native runtime activation qualification", () => { } }); + it("rejects an intermediate directory replaced after validation but before open", () => { + const evidence = completeEvidence(); + const materialized = qualificationFixture(); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-native-race-outside-")); + const receipt = evidence[0]!.installer.invocation; + const receiptParts = receipt.path.split(/[\\/]/u); + const canonicalArtifactRoot = fs.realpathSync(materialized.artifactRoot); + const intermediate = path.join(canonicalArtifactRoot, receiptParts[0]!); + const savedIntermediate = path.join(canonicalArtifactRoot, "qualification-before-race"); + const candidate = path.join(canonicalArtifactRoot, ...receiptParts); + const outsideTarget = path.join(outside, ...receiptParts.slice(1)); + let swapped = false; + const realOpen: typeof fs.openSync = fs.openSync.bind(fs); + let open: ReturnType | undefined; + try { + writeEvidenceArtifacts(materialized.artifactRoot, evidence); + fs.mkdirSync(path.dirname(outsideTarget), { recursive: true }); + fs.writeFileSync(outsideTarget, ARTIFACT_CONTENT, "utf8"); + const reporter = createNativeRuntimeQualificationReporterRecord( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + materialized.bindings, + ); + open = vi.spyOn(fs, "openSync").mockImplementation(((target, flags, mode) => { + if (!swapped && String(target) === candidate) { + fs.renameSync(intermediate, savedIntermediate); + fs.symlinkSync(outside, intermediate, "dir"); + swapped = true; + } + return realOpen(target, flags, mode); + }) as typeof fs.openSync); + + expect(() => + verifyNativeRuntimeQualificationReporterArtifacts( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + reporter, + ), + ).toThrow(/escapes its trusted root|changed during verification/u); + expect(swapped).toBe(true); + } finally { + open?.mockRestore(); + if (fs.lstatSync(intermediate, { throwIfNoEntry: false })?.isSymbolicLink()) { + fs.unlinkSync(intermediate); + } + if (fs.existsSync(savedIntermediate)) fs.renameSync(savedIntermediate, intermediate); + materialized.cleanup(); + fs.rmSync(outside, { force: true, recursive: true }); + } + }); + it("rejects inexact source, image, operation, and CDI receipts", () => { const badSource = completeEvidence(); badSource[0]!.protectedRun.headSha = "main"; @@ -558,6 +652,15 @@ describe("native runtime activation qualification", () => { ), ).toThrow(/protected workflow SHA/u); + const extendedProtectedRun = completeEvidence(); + Object.assign(extendedProtectedRun[0]!.protectedRun, { workerBinding: "untrusted" }); + expect(() => + assertNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + extendedProtectedRun, + ), + ).toThrow(/protected run schema is unsupported/u); + const mixedSourcePair = completeEvidence(); mixedSourcePair[0]!.protectedRun.headSha = "3".repeat(40); expect(() => @@ -604,7 +707,8 @@ describe("native runtime activation qualification", () => { it("binds application, engine, endpoint, and managed-runtime identity", () => { const wrongApplication = completeEvidence(); - wrongApplication[0]!.runtime.application = "hermes"; + const openclaw = wrongApplication.find((entry) => entry.runtime.agent === "openclaw")!; + openclaw.runtime.application = "hermes"; expect(() => assertNativeRuntimeQualificationEvidence( PODMAN_NATIVE_ACTIVATION_QUALIFICATION, From b3eeed4dcbc901d599ca72e29da4397676849e41 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 12:15:44 -0700 Subject: [PATCH 06/10] test(e2e): keep qualification race harness linear Signed-off-by: Aaron Erickson --- .../e2e-native-runtime-qualification.test.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index 4f7070bb1d0..a8f9489349c 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -600,14 +600,16 @@ describe("native runtime activation qualification", () => { evidence, materialized.bindings, ); - open = vi.spyOn(fs, "openSync").mockImplementation(((target, flags, mode) => { - if (!swapped && String(target) === candidate) { + open = vi + .spyOn(fs, "openSync") + .mockImplementationOnce(((target, flags, mode) => { + expect(String(target)).toBe(candidate); fs.renameSync(intermediate, savedIntermediate); fs.symlinkSync(outside, intermediate, "dir"); swapped = true; - } - return realOpen(target, flags, mode); - }) as typeof fs.openSync); + return realOpen(target, flags, mode); + }) as typeof fs.openSync) + .mockImplementation(realOpen); expect(() => verifyNativeRuntimeQualificationReporterArtifacts( @@ -618,10 +620,12 @@ describe("native runtime activation qualification", () => { expect(swapped).toBe(true); } finally { open?.mockRestore(); - if (fs.lstatSync(intermediate, { throwIfNoEntry: false })?.isSymbolicLink()) { + try { fs.unlinkSync(intermediate); + fs.renameSync(savedIntermediate, intermediate); + } catch { + // The one-shot swap did not complete; the original directory remains in place. } - if (fs.existsSync(savedIntermediate)) fs.renameSync(savedIntermediate, intermediate); materialized.cleanup(); fs.rmSync(outside, { force: true, recursive: true }); } From 2984428f5e7fc04a88fe1a9df4616b0fb37571fb Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 13:14:52 -0700 Subject: [PATCH 07/10] refactor: defer native qualification controller wrapper Signed-off-by: Aaron Erickson --- test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md | 17 ++- .../e2e-native-runtime-qualification.test.ts | 101 ------------------ ...ative-runtime-qualification-controller.mts | 80 -------------- 3 files changed, 7 insertions(+), 191 deletions(-) delete mode 100644 tools/e2e/native-runtime-qualification-controller.mts diff --git a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md index a972ce5c95e..3b77003fe66 100644 --- a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md +++ b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md @@ -65,12 +65,11 @@ must supply one or more independent bindings constructed from authenticated GitHub state. Each binding names the repository, workflow revision, run, attempt, exact head/base pair, numeric job, and that job's downloaded artifact root; bindings must not be derived from candidate receipt fields. -`tools/e2e/native-runtime-qualification-controller.mts` is the named -controller-side acceptance adapter: callers provide authenticated run records -and jobs, and the adapter constructs bindings without accepting a worker-owned -binding argument. The compiler defensively clones and freezes the receipts, -requires every receipt and binding to match exactly, and returns a -runtime-branded canonical reporter record. +The compiler defensively clones and freezes the receipts, requires every +receipt and binding to match exactly, and returns a runtime-branded canonical +reporter record. The later protected collector must own the authenticated +GitHub lookup and invoke this canonical reporter boundary directly; this +contract slice does not add an unconsumed controller wrapper. The aggregate validator then resolves every receipt below its bound artifact root, rejects missing, escaping, linked, conflicting, changing, or oversized @@ -80,10 +79,8 @@ A syntactically complete receipt with invented provenance or artifact digests therefore cannot qualify a runtime. These are evidence requirements, not generated evidence. A later protected -collector must publish the receipts from real runners, while its trusted -controller supplies authenticated GitHub run/job state to the dormant -controller adapter before activation can consume them. No workflow invokes the -adapter in this slice. +collector must publish the receipts from real runners and supply authenticated +GitHub run/job state before activation can consume them. ## Activation boundary diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index a8f9489349c..4c1cb240b2b 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -7,10 +7,6 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { - type AuthenticatedNativeRuntimeQualificationRun, - verifyNativeRuntimeQualificationFromTrustedController, -} from "../../../tools/e2e/native-runtime-qualification-controller.mts"; import { assertNativeRuntimeQualificationEvidence as assertVerifiedNativeRuntimeQualificationEvidence, compileNativeRuntimeQualification, @@ -176,21 +172,6 @@ function qualificationFixture(): { }; } -function authenticatedControllerRuns( - artifactRoot: string, -): AuthenticatedNativeRuntimeQualificationRun[] { - return PODMAN_NATIVE_ACTIVATION_QUALIFICATION.cases.map((_qualificationCase, index) => ({ - repository: "NVIDIA/NemoClaw", - workflow: "E2E / PR Gate", - workflowSha: BASE_SHA, - runId: 1000 + index, - attempt: 1, - headSha: HEAD_SHA, - baseSha: BASE_SHA, - jobs: [{ id: 2000 + index, artifactRoot }], - })); -} - function writeEvidenceArtifacts( artifactRoot: string, evidence: readonly NativeRuntimeQualificationEvidence[], @@ -404,88 +385,6 @@ describe("native runtime activation qualification", () => { ).toThrow(/digest does not match its receipt/u); }); - it("accepts only verified evidence bound from authenticated controller run jobs", () => { - const evidence = completeEvidence(); - const materialized = qualificationFixture(); - try { - writeEvidenceArtifacts(materialized.artifactRoot, evidence); - const authenticatedRuns = authenticatedControllerRuns(materialized.artifactRoot); - const verified = verifyNativeRuntimeQualificationFromTrustedController({ - definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - authenticatedRuns, - }); - expect(() => - assertVerifiedNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - verified, - ), - ).not.toThrow(); - expect(() => - assertVerifiedNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence as unknown as VerifiedNativeRuntimeQualificationEvidence, - ), - ).toThrow(/verified canonical reporter evidence/u); - - const workerInventedRun = structuredClone(evidence); - workerInventedRun[0]!.protectedRun.runId = 999_999; - expect(() => - verifyNativeRuntimeQualificationFromTrustedController({ - definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence: workerInventedRun, - authenticatedRuns, - }), - ).toThrow(/no trusted protected-run binding/u); - } finally { - materialized.cleanup(); - } - }); - - it("binds one authenticated protected run to its exact matrix job set", () => { - const evidence = completeEvidence(); - const materialized = qualificationFixture(); - try { - const { jobId: _jobId, ...sharedRun } = structuredClone(evidence[0]!.protectedRun); - for (const entry of evidence) { - entry.protectedRun = { ...sharedRun, jobId: entry.protectedRun.jobId }; - } - const jobs = evidence.map((entry) => ({ - id: entry.protectedRun.jobId, - artifactRoot: materialized.artifactRoot, - })); - writeEvidenceArtifacts(materialized.artifactRoot, evidence); - expect(() => - verifyNativeRuntimeQualificationFromTrustedController({ - definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - authenticatedRuns: [{ ...sharedRun, jobs }], - }), - ).not.toThrow(); - expect(() => - verifyNativeRuntimeQualificationFromTrustedController({ - definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - authenticatedRuns: [{ ...sharedRun, jobs: jobs.slice(1) }], - }), - ).toThrow(/has no trusted protected-run binding/u); - expect(() => - verifyNativeRuntimeQualificationFromTrustedController({ - definition: PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - authenticatedRuns: [ - { - ...sharedRun, - jobs: [...jobs, { id: 9999, artifactRoot: materialized.artifactRoot }], - }, - ], - }), - ).toThrow(/unused protected-run binding/u); - } finally { - materialized.cleanup(); - } - }); - it("binds receipts to independent run metadata and snapshots them before verification", () => { const evidence = completeEvidence(); const materialized = qualificationFixture(); diff --git a/tools/e2e/native-runtime-qualification-controller.mts b/tools/e2e/native-runtime-qualification-controller.mts deleted file mode 100644 index 970fe694bb0..00000000000 --- a/tools/e2e/native-runtime-qualification-controller.mts +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { - assertNativeRuntimeQualificationEvidence, - type CompiledNativeRuntimeQualification, - createNativeRuntimeQualificationReporterRecord, - type NativeRuntimeQualificationEvidence, - type NativeRuntimeQualificationProtectedRun, - type NativeRuntimeQualificationProtectedRunBinding, - type VerifiedNativeRuntimeQualificationEvidence, - verifyNativeRuntimeQualificationReporterArtifacts, -} from "../../test/e2e/registry/activation-qualification.ts"; - -export interface AuthenticatedNativeRuntimeQualificationJob { - /** GitHub Actions job ID read by the protected controller. */ - readonly id: number; - /** Controller-owned root of the downloaded artifact for this exact job. */ - readonly artifactRoot: string; -} - -export type AuthenticatedNativeRuntimeQualificationRun = Omit< - NativeRuntimeQualificationProtectedRun, - "jobId" -> & { - /** Jobs authenticated for this run through the GitHub control plane. */ - readonly jobs: readonly AuthenticatedNativeRuntimeQualificationJob[]; -}; - -export interface TrustedNativeRuntimeQualificationControllerInput { - readonly definition: CompiledNativeRuntimeQualification; - readonly evidence: readonly NativeRuntimeQualificationEvidence[]; - /** - * Control-plane records supplied by the protected controller after it has - * authenticated the workflow run and jobs. Never derive these from worker - * evidence or downloaded artifact contents. - */ - readonly authenticatedRuns: readonly AuthenticatedNativeRuntimeQualificationRun[]; -} - -function controllerBindings( - runs: readonly AuthenticatedNativeRuntimeQualificationRun[], -): NativeRuntimeQualificationProtectedRunBinding[] { - if (!Array.isArray(runs) || runs.length === 0) { - throw new Error("Native runtime qualification requires authenticated controller runs"); - } - return runs.flatMap(({ jobs, ...run }) => { - if (!Array.isArray(jobs) || jobs.length === 0) { - throw new Error("Native runtime qualification authenticated run has no jobs"); - } - return jobs.map((job) => ({ - artifactRoot: job.artifactRoot, - protectedRun: { ...run, jobId: job.id }, - })); - }); -} - -/** - * Trusted-controller acceptance boundary for native-runtime qualification. - * - * This adapter deliberately offers no caller-supplied binding argument. It - * constructs bindings only from authenticated run/job state, re-hashes every - * referenced artifact, and returns evidence only after the canonical - * acceptance assertion succeeds. The later activation workflow may call this - * adapter; defining it here does not activate a runtime provider. - */ -export function verifyNativeRuntimeQualificationFromTrustedController({ - definition, - evidence, - authenticatedRuns, -}: TrustedNativeRuntimeQualificationControllerInput): VerifiedNativeRuntimeQualificationEvidence { - const reporter = createNativeRuntimeQualificationReporterRecord( - definition, - evidence, - controllerBindings(authenticatedRuns), - ); - const verified = verifyNativeRuntimeQualificationReporterArtifacts(definition, reporter); - assertNativeRuntimeQualificationEvidence(definition, verified); - return verified; -} From c8436e2259a4b2cb82f95c6841b5826950f13578 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 13:31:05 -0700 Subject: [PATCH 08/10] test: cover single-job qualification evidence Signed-off-by: Aaron Erickson --- .../e2e-native-runtime-qualification.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index 4c1cb240b2b..03d16e1ace2 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -385,6 +385,51 @@ describe("native runtime activation qualification", () => { ).toThrow(/digest does not match its receipt/u); }); + it("accepts the complete matrix from one protected job and rejects an unused binding", () => { + const evidence = completeEvidence(); + const materialized = qualificationFixture(); + const sharedProtectedRun = structuredClone(evidence[0]!.protectedRun); + for (const entry of evidence) { + entry.protectedRun = structuredClone(sharedProtectedRun); + } + const sharedBinding = { + protectedRun: structuredClone(sharedProtectedRun), + artifactRoot: materialized.artifactRoot, + }; + const unusedBinding = { + protectedRun: { ...sharedProtectedRun, jobId: 9999 }, + artifactRoot: materialized.artifactRoot, + }; + try { + writeEvidenceArtifacts(materialized.artifactRoot, evidence); + const reporter = createNativeRuntimeQualificationReporterRecord( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + [sharedBinding], + ); + const verified = verifyNativeRuntimeQualificationReporterArtifacts( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + reporter, + ); + + expect(() => + assertVerifiedNativeRuntimeQualificationEvidence( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + verified, + ), + ).not.toThrow(); + expect(() => + createNativeRuntimeQualificationReporterRecord( + PODMAN_NATIVE_ACTIVATION_QUALIFICATION, + evidence, + [sharedBinding, unusedBinding], + ), + ).toThrow(/unused protected-run binding/u); + } finally { + materialized.cleanup(); + } + }); + it("binds receipts to independent run metadata and snapshots them before verification", () => { const evidence = completeEvidence(); const materialized = qualificationFixture(); From 8514315b8f60d285ff8113ac2a02ef006be5877f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 13:51:05 -0700 Subject: [PATCH 09/10] refactor: defer qualification evidence reporter Signed-off-by: Aaron Erickson --- test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md | 41 +- test/e2e/registry/activation-qualification.ts | 852 ------------------ .../e2e-native-runtime-qualification.test.ts | 611 +------------ 3 files changed, 24 insertions(+), 1480 deletions(-) diff --git a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md index 3b77003fe66..1c45299237f 100644 --- a/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md +++ b/test/e2e/docs/NATIVE_RUNTIME_QUALIFICATION.md @@ -28,12 +28,12 @@ Compilation requires 24 exact cases: Every case must declare installation, Docker-unavailable proof, onboarding, an agent turn, stop/start, snapshot/restore, rebuild, restart/reconciliation, and exact cleanup. Removing one case or obligation is a compile error, not a skip. -The agent identity also binds its application-facing name: OpenClaw, Hermes, -or `langchain-deepagents-code` for DCode. ## Exact evidence -Activation evidence is complete only when every compiled case has: +Every compiled case declares the evidence categories that the protected +collector must eventually produce. Those categories preserve the complete +activation target: - the exact protected workflow revision, run, job, attempt, head SHA, and base SHA; @@ -55,32 +55,15 @@ Activation evidence is complete only when every compiled case has: - exact cleanup proving external Ollama was retained or provider-owned NIM and vLLM were removed, with no provider-owned runtime IDs remaining. -Evidence paths must be relative and traversal-free. SHA and SHA-256 fields are -strict lowercase hexadecimal values. Missing, duplicate, unknown, or inexact -case evidence fails the aggregate qualification check. All cases must use the -configured protected workflow and one exact head/base/workflow source. - -Raw worker receipts are never sufficient. The trusted protected-E2E controller -must supply one or more independent bindings constructed from authenticated -GitHub state. Each binding names the repository, workflow revision, run, -attempt, exact head/base pair, numeric job, and that job's downloaded artifact -root; bindings must not be derived from candidate receipt fields. -The compiler defensively clones and freezes the receipts, requires every -receipt and binding to match exactly, and returns a runtime-branded canonical -reporter record. The later protected collector must own the authenticated -GitHub lookup and invoke this canonical reporter boundary directly; this -contract slice does not add an unconsumed controller wrapper. - -The aggregate validator then resolves every receipt below its bound artifact -root, rejects missing, escaping, linked, conflicting, changing, or oversized -files, and hashes the actual bytes before comparing the claimed SHA-256 digest. -Only a separately branded verified-evidence object can reach final acceptance. -A syntactically complete receipt with invented provenance or artifact digests -therefore cannot qualify a runtime. - -These are evidence requirements, not generated evidence. A later protected -collector must publish the receipts from real runners and supply authenticated -GitHub run/job state before activation can consume them. +These are compile-time requirements, not a receipt schema or generated +evidence. This slice intentionally does not export a reporter, artifact +verifier, or process-local evidence brand before a protected workflow consumes +that API. The protected-collector slice must add those pieces together: obtain +authenticated GitHub run/job state independently of worker receipts, verify +every artifact below its downloaded job root, and fail closed on incomplete, +inexact, linked, escaping, conflicting, changing, or oversized evidence. All +cases must bind to one exact protected workflow/head/base source before the +runtime can activate. ## Activation boundary diff --git a/test/e2e/registry/activation-qualification.ts b/test/e2e/registry/activation-qualification.ts index 3ffe7b03300..757749c8a5c 100644 --- a/test/e2e/registry/activation-qualification.ts +++ b/test/e2e/registry/activation-qualification.ts @@ -1,10 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; - import { defineExecutionProfile, type ExecutionAcceleration, @@ -24,8 +20,6 @@ export const NATIVE_RUNTIME_QUALIFICATION_INFERENCE = { } as const satisfies Readonly>; export type LocalInferenceProvider = "ollama" | "nim" | "vllm"; -export type QualificationApplication = "openclaw" | "hermes" | "langchain-deepagents-code"; -export type QualificationManagedImageRole = "agent" | "inference" | "probe"; export type QualificationObligation = | "installer.install" | "runtime.docker-unavailable" @@ -113,147 +107,7 @@ export interface CompiledNativeRuntimeQualification { cases: readonly Readonly[]; } -export interface QualificationArtifactReceipt { - path: string; - sha256: string; -} - -export interface NativeRuntimeQualificationProtectedRun { - repository: string; - workflow: string; - /** Exact trusted revision from which GitHub loaded the protected workflow. */ - workflowSha: string; - runId: number; - attempt: number; - jobId: number; - headSha: string; - baseSha: string; -} - -export interface NativeRuntimeQualificationEvidence { - schemaVersion: 1; - caseId: string; - protectedRun: NativeRuntimeQualificationProtectedRun; - installer: { - provider: ExecutionProviderId; - architecture: ExecutionArchitecture; - dockerAvailability: "unavailable"; - exitCode: 0; - invocation: QualificationArtifactReceipt; - script: QualificationArtifactReceipt; - }; - runtime: { - provider: ExecutionProviderId; - profileId: string; - agent: RuntimeAgent; - application: QualificationApplication; - inference: LocalInferenceProvider; - architecture: ExecutionArchitecture; - acceleration: ExecutionAcceleration; - rootMode: "rootless"; - engineName: string; - engineVersion: string; - engineAuthority: { - schemaVersion: 1; - providerId: ExecutionProviderId; - operation: "host-local-inference"; - engineId: string; - authorityId: string; - bindingSha256: string; - }; - managedImages: readonly { - role: QualificationManagedImageRole; - imageRef: string; - }[]; - route: { - service: LocalInferenceProvider; - endpoint: { - host: string; - port: number; - networkName: string; - gatewayProviderBaseUrl: string; - applicationBaseUrl: "https://inference.local/v1"; - }; - authority: { - receiptSha256: string; - kind: "host" | "container"; - runtimeId: string | null; - containerName: string | null; - specSha256: string | null; - }; - }; - modelId: string; - inferenceResult: QualificationArtifactReceipt; - }; - operations: readonly { - id: QualificationObligation; - authoritySha256: string; - artifact: QualificationArtifactReceipt; - }[]; - recovery: { - status: "reconciled"; - authoritySha256: string; - artifact: QualificationArtifactReceipt; - }; - cleanup: { - status: "retained-external" | "removed-owned"; - authoritySha256: string; - providerOwnedRuntimeIds: readonly string[]; - artifact: QualificationArtifactReceipt; - }; - nvidiaCdi?: { - devices: readonly ["nvidia.com/gpu=all"]; - artifact: QualificationArtifactReceipt; - }; -} - -export interface NativeRuntimeQualificationProtectedRunBinding { - readonly protectedRun: NativeRuntimeQualificationProtectedRun; - readonly artifactRoot: string; -} - -export interface NativeRuntimeQualificationReporterRecord { - readonly qualificationId: string; - readonly caseIds: readonly string[]; -} - -export interface VerifiedNativeRuntimeQualificationEvidence { - readonly qualificationId: string; - readonly caseIds: readonly string[]; -} - -const compiledQualifications = new WeakSet(); -interface CompiledReporterState { - readonly definition: CompiledNativeRuntimeQualification; - readonly evidence: readonly NativeRuntimeQualificationEvidence[]; - readonly artifactRootByCaseId: ReadonlyMap; -} -const compiledReporters = new WeakMap(); -const verifiedEvidence = new WeakMap(); -const SHA_PATTERN = /^[a-f0-9]{40}$/u; -const SHA256_PATTERN = /^[a-f0-9]{64}$/u; -const IMAGE_REFERENCE_PATTERN = - /^(?:[A-Za-z0-9._-]+(?::[0-9]+)?\/)*(?:[A-Za-z0-9._-]+)@sha256:[a-f0-9]{64}$/u; const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; -const SAFE_HOST_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?$/u; -const SAFE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/=+-]{0,511}$/u; -const MAX_QUALIFICATION_ARTIFACT_BYTES = 64 * 1024 * 1024; -const PROTECTED_RUN_KEYS = [ - "attempt", - "baseSha", - "headSha", - "jobId", - "repository", - "runId", - "workflow", - "workflowSha", -] as const; - -const APPLICATION_BY_AGENT = { - openclaw: "openclaw", - hermes: "hermes", - dcode: "langchain-deepagents-code", -} as const satisfies Readonly>; function assertSingleLine(value: string, label: string): string { const normalized = value.trim(); @@ -263,28 +117,6 @@ function assertSingleLine(value: string, label: string): string { return normalized; } -function exactRecord( - value: unknown, - keys: readonly string[], - label: string, -): Record { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - throw new Error(`${label} must be an object`); - } - const record = value as Record; - if (Object.keys(record).sort().join(",") !== [...keys].sort().join(",")) { - throw new Error(`${label} schema is unsupported`); - } - return record; -} - -function deepFreeze(value: T, seen = new WeakSet()): T { - if (typeof value !== "object" || value === null || seen.has(value)) return value; - seen.add(value); - for (const child of Object.values(value)) deepFreeze(child, seen); - return Object.freeze(value); -} - function assertExactSet( actual: readonly T[], expected: readonly T[], @@ -456,689 +288,5 @@ export function compileNativeRuntimeQualification( engineName, cases: Object.freeze([...cases].sort((left, right) => compareCodeUnits(left.id, right.id))), }); - compiledQualifications.add(compiled); return compiled; } - -function assertArtifact(receipt: QualificationArtifactReceipt, label: string): void { - const artifactPath = assertSingleLine(receipt.path, `${label} path`); - const components = artifactPath.split(/[\\/]/u); - if ( - artifactPath.startsWith("/") || - artifactPath.startsWith("\\") || - components.some((part) => part === "" || part === "." || part === "..") - ) { - throw new Error(`${label} path must be repository-relative and traversal-free`); - } - if (!SHA256_PATTERN.test(receipt.sha256)) { - throw new Error(`${label} sha256 must be an exact lowercase SHA-256 digest`); - } -} - -function sameArtifactMetadata(left: fs.BigIntStats, right: fs.BigIntStats): boolean { - return ( - left.dev === right.dev && - left.ino === right.ino && - left.mode === right.mode && - left.nlink === right.nlink && - left.uid === right.uid && - left.gid === right.gid && - left.size === right.size && - left.mtimeNs === right.mtimeNs && - left.ctimeNs === right.ctimeNs - ); -} - -interface ArtifactPathSnapshot { - readonly candidate: string; - readonly components: readonly { - readonly path: string; - readonly metadata: fs.BigIntStats; - }[]; -} - -function resolvedArtifactTarget( - artifactRoot: string, - candidate: string, - receipt: QualificationArtifactReceipt, -): string { - let target: string; - try { - target = fs.realpathSync(candidate); - } catch { - throw new Error(`Qualification artifact '${receipt.path}' is missing`); - } - const relative = path.relative(artifactRoot, target); - if (relative === "" || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { - throw new Error(`Qualification artifact '${receipt.path}' escapes its trusted root`); - } - return target; -} - -function snapshotArtifactPath( - artifactRoot: string, - receipt: QualificationArtifactReceipt, -): ArtifactPathSnapshot { - const paths = [artifactRoot]; - for (const component of receipt.path.split(/[\\/]/u)) { - paths.push(path.join(paths.at(-1)!, component)); - } - let components: ArtifactPathSnapshot["components"]; - try { - components = paths.map((componentPath) => ({ - path: componentPath, - metadata: fs.lstatSync(componentPath, { bigint: true }), - })); - } catch { - throw new Error(`Qualification artifact '${receipt.path}' is missing`); - } - const candidate = paths.at(-1)!; - for (const [index, component] of components.entries()) { - const leaf = index === components.length - 1; - if (component.metadata.isSymbolicLink()) { - resolvedArtifactTarget(artifactRoot, candidate, receipt); - throw new Error( - `Qualification artifact '${receipt.path}' must use stable real path components`, - ); - } - if ((!leaf && !component.metadata.isDirectory()) || (leaf && !component.metadata.isFile())) { - throw new Error( - `Qualification artifact '${receipt.path}' must be below real directories in a regular file`, - ); - } - } - return { candidate, components }; -} - -function sameArtifactPathSnapshot( - left: ArtifactPathSnapshot, - right: ArtifactPathSnapshot, -): boolean { - return ( - left.candidate === right.candidate && - left.components.length === right.components.length && - left.components.every( - (component, index) => - component.path === right.components[index]?.path && - sameArtifactMetadata(component.metadata, right.components[index]!.metadata), - ) - ); -} - -function assertStableArtifactPath( - artifactRoot: string, - receipt: QualificationArtifactReceipt, - initial: ArtifactPathSnapshot, - opened: fs.BigIntStats, -): string { - const current = snapshotArtifactPath(artifactRoot, receipt); - resolvedArtifactTarget(artifactRoot, current.candidate, receipt); - const confirmed = snapshotArtifactPath(artifactRoot, receipt); - const confirmedLeaf = confirmed.components.at(-1)!.metadata; - if ( - !sameArtifactPathSnapshot(initial, current) || - !sameArtifactPathSnapshot(current, confirmed) || - !sameArtifactMetadata(opened, confirmedLeaf) - ) { - throw new Error(`Qualification artifact '${receipt.path}' changed during verification`); - } - return `${opened.dev}:${opened.ino}`; -} - -function verifyArtifactContents( - artifactRoot: string, - receipt: QualificationArtifactReceipt, - claimedPaths: Map, -): void { - assertArtifact(receipt, "Qualification artifact"); - const pathSnapshot = snapshotArtifactPath(artifactRoot, receipt); - const noFollow = fs.constants.O_NOFOLLOW; - if (typeof noFollow !== "number") { - throw new Error("Qualification artifact verification requires O_NOFOLLOW"); - } - let descriptor: number; - try { - descriptor = fs.openSync(pathSnapshot.candidate, fs.constants.O_RDONLY | noFollow); - } catch { - throw new Error(`Qualification artifact '${receipt.path}' is not a readable regular file`); - } - try { - const before = fs.fstatSync(descriptor, { bigint: true }); - if ( - !before.isFile() || - before.nlink !== 1n || - before.size < 1n || - before.size > BigInt(MAX_QUALIFICATION_ARTIFACT_BYTES) - ) { - throw new Error(`Qualification artifact '${receipt.path}' failed type, link, or size checks`); - } - const artifactIdentity = assertStableArtifactPath(artifactRoot, receipt, pathSnapshot, before); - const previousDigest = claimedPaths.get(artifactIdentity); - if (previousDigest !== undefined) { - if (previousDigest !== receipt.sha256) { - throw new Error( - `Qualification evidence reuses artifact '${receipt.path}' with a different digest`, - ); - } - return; - } - claimedPaths.set(artifactIdentity, receipt.sha256); - const contents = Buffer.alloc(Number(before.size)); - let offset = 0; - while (offset < contents.length) { - const count = fs.readSync(descriptor, contents, offset, contents.length - offset, offset); - if (count === 0) break; - offset += count; - } - const overflow = Buffer.alloc(1); - const overflowCount = fs.readSync(descriptor, overflow, 0, 1, offset); - const after = fs.fstatSync(descriptor, { bigint: true }); - if ( - offset !== contents.length || - overflowCount !== 0 || - !sameArtifactMetadata(before, after) || - assertStableArtifactPath(artifactRoot, receipt, pathSnapshot, after) !== artifactIdentity - ) { - throw new Error(`Qualification artifact '${receipt.path}' changed during verification`); - } - const actual = createHash("sha256").update(contents).digest("hex"); - if (actual !== receipt.sha256) { - throw new Error(`Qualification artifact '${receipt.path}' digest does not match its receipt`); - } - } finally { - fs.closeSync(descriptor); - } -} - -function qualificationArtifacts( - evidence: NativeRuntimeQualificationEvidence, -): readonly QualificationArtifactReceipt[] { - return [ - evidence.installer.invocation, - evidence.installer.script, - evidence.runtime.inferenceResult, - ...evidence.operations.map((operation) => operation.artifact), - evidence.recovery.artifact, - evidence.cleanup.artifact, - ...(evidence.nvidiaCdi ? [evidence.nvidiaCdi.artifact] : []), - ]; -} - -function exactReporterRun( - definition: CompiledNativeRuntimeQualification, - input: NativeRuntimeQualificationProtectedRun, -): Readonly { - if ( - input.repository !== definition.repository || - input.workflow !== definition.protectedWorkflow - ) { - throw new Error("Native runtime qualification reporter names the wrong protected workflow"); - } - if ( - !SHA_PATTERN.test(input.workflowSha) || - !SHA_PATTERN.test(input.headSha) || - !SHA_PATTERN.test(input.baseSha) || - input.headSha === input.baseSha - ) { - throw new Error( - "Native runtime qualification reporter must name exact workflow/head/base SHAs", - ); - } - assertPositiveInteger(input.runId, "Qualification reporter run id"); - assertPositiveInteger(input.attempt, "Qualification reporter run attempt"); - assertPositiveInteger(input.jobId, "Qualification reporter job id"); - return Object.freeze({ ...input }); -} - -function protectedRunKey(input: NativeRuntimeQualificationProtectedRun): string { - return JSON.stringify([ - input.repository, - input.workflow, - input.workflowSha, - input.runId, - input.attempt, - input.jobId, - input.headSha, - input.baseSha, - ]); -} - -/** - * Bind worker-authored evidence to run/job identities and artifact roots - * supplied independently by the trusted protected-E2E controller. The caller - * must construct bindings from authenticated control-plane state, never from - * candidate receipt fields. - */ -export function createNativeRuntimeQualificationReporterRecord( - definition: CompiledNativeRuntimeQualification, - evidence: readonly NativeRuntimeQualificationEvidence[], - bindings: readonly NativeRuntimeQualificationProtectedRunBinding[], -): NativeRuntimeQualificationReporterRecord { - if (!compiledQualifications.has(definition)) { - throw new Error("Native runtime qualification reporter requires a compiled definition"); - } - if (!Array.isArray(bindings) || bindings.length === 0) { - throw new Error("Native runtime qualification reporter requires trusted run bindings"); - } - const clonedEvidence = deepFreeze( - structuredClone(evidence) as NativeRuntimeQualificationEvidence[], - ); - validateNativeRuntimeQualificationEvidence(definition, clonedEvidence); - - const bindingByRun = new Map< - string, - { readonly artifactRoot: string; readonly protectedRun: NativeRuntimeQualificationProtectedRun } - >(); - for (const candidate of bindings) { - const binding = exactRecord( - candidate, - ["artifactRoot", "protectedRun"], - "Native runtime qualification protected-run binding", - ); - const runRecord = exactRecord( - binding.protectedRun, - PROTECTED_RUN_KEYS, - "Native runtime qualification protected run", - ); - const protectedRun = exactReporterRun( - definition, - runRecord as unknown as NativeRuntimeQualificationProtectedRun, - ); - if (typeof binding.artifactRoot !== "string" || binding.artifactRoot === "") { - throw new Error("Native runtime qualification binding artifact root is invalid"); - } - const metadata = fs.lstatSync(binding.artifactRoot); - if (!metadata.isDirectory() || metadata.isSymbolicLink()) { - throw new Error("Native runtime qualification artifact root must be a real directory"); - } - const key = protectedRunKey(protectedRun); - if (bindingByRun.has(key)) { - throw new Error("Native runtime qualification reporter repeats a protected-run binding"); - } - bindingByRun.set(key, { - artifactRoot: fs.realpathSync(binding.artifactRoot), - protectedRun, - }); - } - - const usedBindings = new Set(); - const artifactRootByCaseId = new Map(); - for (const entry of clonedEvidence) { - const runRecord = exactRecord( - entry.protectedRun, - PROTECTED_RUN_KEYS, - `Qualification evidence '${entry.caseId}' protected run`, - ); - const key = protectedRunKey(runRecord as unknown as NativeRuntimeQualificationProtectedRun); - const binding = bindingByRun.get(key); - if (!binding) { - throw new Error( - `Qualification evidence '${entry.caseId}' has no trusted protected-run binding`, - ); - } - usedBindings.add(key); - artifactRootByCaseId.set(entry.caseId, binding.artifactRoot); - } - if (usedBindings.size !== bindingByRun.size) { - throw new Error("Native runtime qualification reporter has an unused protected-run binding"); - } - const reporter = Object.freeze({ - qualificationId: definition.id, - caseIds: Object.freeze(clonedEvidence.map((entry) => entry.caseId)), - }); - compiledReporters.set(reporter, { - definition, - evidence: clonedEvidence, - artifactRootByCaseId, - }); - return reporter; -} - -/** Re-hash every worker artifact before promoting a reporter to verified evidence. */ -export function verifyNativeRuntimeQualificationReporterArtifacts( - definition: CompiledNativeRuntimeQualification, - reporter: NativeRuntimeQualificationReporterRecord, -): VerifiedNativeRuntimeQualificationEvidence { - const state = compiledReporters.get(reporter); - if (state?.definition !== definition) { - throw new Error( - "Native runtime qualification artifacts require their canonical protected-workflow reporter", - ); - } - const claimedArtifactPaths = new Map(); - for (const entry of state.evidence) { - const artifactRoot = state.artifactRootByCaseId.get(entry.caseId); - if (!artifactRoot) { - throw new Error(`Qualification evidence '${entry.caseId}' lost its artifact binding`); - } - for (const artifact of qualificationArtifacts(entry)) { - verifyArtifactContents(artifactRoot, artifact, claimedArtifactPaths); - } - } - const verified = Object.freeze({ - qualificationId: definition.id, - caseIds: Object.freeze(state.evidence.map((entry) => entry.caseId)), - }); - verifiedEvidence.set(verified, state); - return verified; -} - -function assertPositiveInteger(value: number, label: string): void { - if (!Number.isSafeInteger(value) || value < 1) { - throw new Error(`${label} must be a positive integer`); - } -} - -function assertSha256(value: string, label: string): void { - if (!SHA256_PATTERN.test(value)) { - throw new Error(`${label} must be an exact lowercase SHA-256 digest`); - } -} - -function assertSafeId(value: string, label: string): void { - if (!SAFE_ID_PATTERN.test(value)) { - throw new Error(`${label} must be an exact non-empty runtime identifier`); - } -} - -function requiredManagedImageRoles( - inference: LocalInferenceProvider, -): readonly QualificationManagedImageRole[] { - return inference === "ollama" ? ["agent", "probe"] : ["agent", "inference", "probe"]; -} - -function assertExactUrl(value: string, label: string): URL { - const text = assertSingleLine(value, label); - let parsed: URL; - try { - parsed = new URL(text); - } catch { - throw new Error(`${label} must be an exact absolute URL`); - } - if (parsed.username || parsed.password || parsed.search || parsed.hash || parsed.href !== text) { - throw new Error(`${label} must not contain credentials, query parameters, or fragments`); - } - return parsed; -} - -function assertEngineAuthority( - definition: CompiledNativeRuntimeQualification, - evidence: NativeRuntimeQualificationEvidence, -): void { - const authority = evidence.runtime.engineAuthority; - if ( - authority.schemaVersion !== 1 || - authority.providerId !== definition.provider || - authority.operation !== "host-local-inference" - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' has invalid engine authority`); - } - assertSafeId(authority.engineId, "Runtime engine id"); - assertSafeId(authority.authorityId, "Runtime authority id"); - assertSha256(authority.bindingSha256, "Runtime authority binding"); -} - -function assertInferenceRoute( - qualificationCase: Readonly, - evidence: NativeRuntimeQualificationEvidence, -): string { - const route = evidence.runtime.route; - if (route.service !== qualificationCase.inference) { - throw new Error(`Qualification evidence '${evidence.caseId}' has a different route service`); - } - const endpoint = route.endpoint; - if (!SAFE_HOST_PATTERN.test(endpoint.host)) { - throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid endpoint host`); - } - assertPositiveInteger(endpoint.port, "Inference endpoint port"); - if (endpoint.port > 65_535) { - throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid endpoint port`); - } - assertSafeId(endpoint.networkName, "Inference endpoint network"); - const gateway = assertExactUrl(endpoint.gatewayProviderBaseUrl, "Gateway provider base URL"); - if (gateway.protocol !== "http:" || Number(gateway.port) !== endpoint.port) { - throw new Error( - `Qualification evidence '${evidence.caseId}' has a mismatched gateway endpoint`, - ); - } - if (endpoint.applicationBaseUrl !== "https://inference.local/v1") { - throw new Error( - `Qualification evidence '${evidence.caseId}' has a noncanonical application route`, - ); - } - - const authority = route.authority; - assertSha256(authority.receiptSha256, "Inference authority receipt"); - if (qualificationCase.inference === "ollama") { - if ( - authority.kind !== "host" || - authority.runtimeId !== null || - authority.containerName !== null || - authority.specSha256 !== null - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' must retain external Ollama`); - } - } else { - if ( - authority.kind !== "container" || - authority.runtimeId === null || - authority.containerName === null || - authority.specSha256 === null - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' must name managed inference`); - } - assertSafeId(authority.runtimeId, "Managed inference runtime id"); - assertSafeId(authority.containerName, "Managed inference container name"); - assertSha256(authority.specSha256, "Managed inference specification"); - } - return authority.receiptSha256; -} - -function assertLifecycleEvidence( - qualificationCase: Readonly, - evidence: NativeRuntimeQualificationEvidence, - authoritySha256: string, -): void { - const operationIds = evidence.operations.map((operation) => operation.id); - assertExactSet( - operationIds, - qualificationCase.obligations, - `Qualification evidence '${evidence.caseId}' operations`, - ); - for (const operation of evidence.operations) { - if (operation.authoritySha256 !== authoritySha256) { - throw new Error(`Operation '${operation.id}' is bound to different runtime authority`); - } - assertArtifact(operation.artifact, `Operation '${operation.id}' artifact`); - } - if ( - evidence.recovery.status !== "reconciled" || - evidence.recovery.authoritySha256 !== authoritySha256 - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' has invalid recovery evidence`); - } - assertArtifact(evidence.recovery.artifact, "Recovery artifact"); - - const cleanupStatus = - qualificationCase.inference === "ollama" ? "retained-external" : "removed-owned"; - if ( - evidence.cleanup.status !== cleanupStatus || - evidence.cleanup.authoritySha256 !== authoritySha256 || - evidence.cleanup.providerOwnedRuntimeIds.length !== 0 - ) { - throw new Error( - `Qualification evidence '${evidence.caseId}' has invalid exact cleanup evidence`, - ); - } - assertArtifact(evidence.cleanup.artifact, "Cleanup artifact"); -} - -function assertProtectedRunEvidence( - definition: CompiledNativeRuntimeQualification, - evidence: NativeRuntimeQualificationEvidence, -): void { - exactRecord( - evidence.protectedRun, - PROTECTED_RUN_KEYS, - `Qualification evidence '${evidence.caseId}' protected run`, - ); - if (evidence.protectedRun.repository !== definition.repository) { - throw new Error(`Qualification evidence '${evidence.caseId}' belongs to the wrong repository`); - } - if (evidence.protectedRun.workflow !== definition.protectedWorkflow) { - throw new Error(`Qualification evidence '${evidence.caseId}' belongs to the wrong workflow`); - } - if (!SHA_PATTERN.test(evidence.protectedRun.workflowSha)) { - throw new Error( - `Qualification evidence '${evidence.caseId}' must name the exact protected workflow SHA`, - ); - } - assertPositiveInteger(evidence.protectedRun.runId, "Protected run id"); - assertPositiveInteger(evidence.protectedRun.attempt, "Protected run attempt"); - assertPositiveInteger(evidence.protectedRun.jobId, "Protected run job id"); - if ( - !SHA_PATTERN.test(evidence.protectedRun.headSha) || - !SHA_PATTERN.test(evidence.protectedRun.baseSha) - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' must name exact head/base SHAs`); - } - if (evidence.protectedRun.headSha === evidence.protectedRun.baseSha) { - throw new Error(`Qualification evidence '${evidence.caseId}' head/base SHAs must differ`); - } -} - -function assertInstallerReceipt( - definition: CompiledNativeRuntimeQualification, - qualificationCase: Readonly, - evidence: NativeRuntimeQualificationEvidence, -): void { - if ( - evidence.installer.provider !== definition.provider || - evidence.installer.architecture !== qualificationCase.profile.architecture || - evidence.installer.dockerAvailability !== "unavailable" || - evidence.installer.exitCode !== 0 - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid installer receipt`); - } - assertArtifact(evidence.installer.invocation, "Installer invocation artifact"); - assertArtifact(evidence.installer.script, "Installer script artifact"); -} - -function assertRuntimeIdentity( - definition: CompiledNativeRuntimeQualification, - qualificationCase: Readonly, - evidence: NativeRuntimeQualificationEvidence, -): void { - const profile = qualificationCase.profile; - if ( - evidence.runtime.provider !== definition.provider || - evidence.runtime.profileId !== profile.id || - evidence.runtime.agent !== qualificationCase.agent || - evidence.runtime.application !== APPLICATION_BY_AGENT[qualificationCase.agent] || - evidence.runtime.inference !== qualificationCase.inference || - evidence.runtime.architecture !== profile.architecture || - evidence.runtime.acceleration !== profile.acceleration || - evidence.runtime.rootMode !== "rootless" - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' has an invalid runtime identity`); - } - if (evidence.runtime.engineName !== definition.engineName) { - throw new Error(`Qualification evidence '${evidence.caseId}' names the wrong runtime engine`); - } - assertSingleLine(evidence.runtime.engineVersion, "Runtime engine version"); -} - -function assertCaseEvidence( - definition: CompiledNativeRuntimeQualification, - qualificationCase: Readonly, - evidence: NativeRuntimeQualificationEvidence, -): void { - if (evidence.schemaVersion !== 1) { - throw new Error(`Qualification evidence '${evidence.caseId}' has unsupported schemaVersion`); - } - assertProtectedRunEvidence(definition, evidence); - assertInstallerReceipt(definition, qualificationCase, evidence); - assertRuntimeIdentity(definition, qualificationCase, evidence); - assertEngineAuthority(definition, evidence); - const imageRoles: QualificationManagedImageRole[] = []; - for (const image of evidence.runtime.managedImages) { - imageRoles.push(image.role); - if (!IMAGE_REFERENCE_PATTERN.test(image.imageRef)) { - throw new Error( - `Qualification evidence '${evidence.caseId}' must use exact image references`, - ); - } - } - assertExactSet( - imageRoles, - requiredManagedImageRoles(qualificationCase.inference), - `Qualification evidence '${evidence.caseId}' managed image roles`, - ); - const authoritySha256 = assertInferenceRoute(qualificationCase, evidence); - assertSingleLine(evidence.runtime.modelId, "Inference model id"); - assertArtifact(evidence.runtime.inferenceResult, "Inference result artifact"); - assertLifecycleEvidence(qualificationCase, evidence, authoritySha256); - - if (qualificationCase.profile.acceleration === "nvidia-gpu") { - if ( - evidence.nvidiaCdi?.devices.length !== 1 || - evidence.nvidiaCdi.devices[0] !== "nvidia.com/gpu=all" - ) { - throw new Error(`Qualification evidence '${evidence.caseId}' must prove NVIDIA CDI access`); - } - assertArtifact(evidence.nvidiaCdi.artifact, "NVIDIA CDI artifact"); - } else if (evidence.nvidiaCdi !== undefined) { - throw new Error(`CPU qualification evidence '${evidence.caseId}' must not claim NVIDIA CDI`); - } -} - -function validateNativeRuntimeQualificationEvidence( - definition: CompiledNativeRuntimeQualification, - evidence: readonly NativeRuntimeQualificationEvidence[], -): void { - if (!compiledQualifications.has(definition)) { - throw new Error("Native runtime qualification evidence requires a compiled definition"); - } - const casesById = new Map(definition.cases.map((entry) => [entry.id, entry])); - const evidenceById = new Map(); - const sourcePairs = new Set(); - for (const entry of evidence) { - if (evidenceById.has(entry.caseId)) { - throw new Error(`Native runtime qualification evidence repeats case '${entry.caseId}'`); - } - const qualificationCase = casesById.get(entry.caseId); - if (!qualificationCase) { - throw new Error(`Native runtime qualification evidence names unknown case '${entry.caseId}'`); - } - assertCaseEvidence(definition, qualificationCase, entry); - sourcePairs.add( - `${entry.protectedRun.headSha}:${entry.protectedRun.baseSha}:${entry.protectedRun.workflowSha}`, - ); - evidenceById.set(entry.caseId, entry); - } - const missing = definition.cases.filter((entry) => !evidenceById.has(entry.id)); - if (missing.length > 0) { - throw new Error( - `Native runtime qualification evidence is incomplete: ${missing.map((entry) => entry.id).join(", ")}`, - ); - } - if (sourcePairs.size !== 1) { - throw new Error( - "Native runtime qualification evidence must use one exact head/base/workflow source", - ); - } -} - -/** Accept only evidence that crossed both trusted reporter and byte-verification boundaries. */ -export function assertNativeRuntimeQualificationEvidence( - definition: CompiledNativeRuntimeQualification, - evidence: VerifiedNativeRuntimeQualificationEvidence, -): void { - const state = verifiedEvidence.get(evidence); - if (state?.definition !== definition) { - throw new Error( - "Native runtime qualification evidence requires verified canonical reporter evidence", - ); - } - validateNativeRuntimeQualificationEvidence(definition, state.evidence); -} diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index 03d16e1ace2..7541f9c42bb 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -1,207 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { describe, expect, it, vi } from "vitest"; -import { - assertNativeRuntimeQualificationEvidence as assertVerifiedNativeRuntimeQualificationEvidence, - compileNativeRuntimeQualification, - createNativeRuntimeQualificationReporterRecord, - type NativeRuntimeQualificationEvidence, - type NativeRuntimeQualificationProtectedRunBinding, - type QualificationArtifactReceipt, - type VerifiedNativeRuntimeQualificationEvidence, - verifyNativeRuntimeQualificationReporterArtifacts, -} from "../registry/activation-qualification.ts"; +import { describe, expect, it } from "vitest"; +import { compileNativeRuntimeQualification } from "../registry/activation-qualification.ts"; import { hasRegisteredRuntimeProfile } from "../registry/registry.ts"; import { nativeRuntimeQualificationDefinition, PODMAN_NATIVE_ACTIVATION_QUALIFICATION, } from "./native-runtime-qualification-fixtures.ts"; -const HEAD_SHA = "1".repeat(40); -const BASE_SHA = "2".repeat(40); -const ARTIFACT_CONTENT = "verified native runtime qualification artifact\n"; -const ARTIFACT_SHA = createHash("sha256").update(ARTIFACT_CONTENT, "utf8").digest("hex"); -const AUTHORITY_SHA = "b".repeat(64); -const BINDING_SHA = "c".repeat(64); -const SPEC_SHA = "d".repeat(64); -const IMAGE_REFS = { - agent: `nvcr.io/nvidia/nemoclaw-agent@sha256:${"1".repeat(64)}`, - inference: `nvcr.io/nvidia/nemoclaw-inference@sha256:${"2".repeat(64)}`, - probe: `quay.io/curl/curl@sha256:${"3".repeat(64)}`, -} as const; - -const APPLICATIONS = { - openclaw: "openclaw", - hermes: "hermes", - dcode: "langchain-deepagents-code", -} as const; - -function artifact(label: string) { - return { path: `qualification/${label}.json`, sha256: ARTIFACT_SHA }; -} - -function completeEvidence(): NativeRuntimeQualificationEvidence[] { - return PODMAN_NATIVE_ACTIVATION_QUALIFICATION.cases.map((qualificationCase, index) => { - const managed = qualificationCase.inference !== "ollama"; - return { - schemaVersion: 1, - caseId: qualificationCase.id, - protectedRun: { - repository: "NVIDIA/NemoClaw", - workflow: "E2E / PR Gate", - workflowSha: BASE_SHA, - runId: 1000 + index, - attempt: 1, - jobId: 2000 + index, - headSha: HEAD_SHA, - baseSha: BASE_SHA, - }, - installer: { - provider: PODMAN_NATIVE_ACTIVATION_QUALIFICATION.provider, - architecture: qualificationCase.profile.architecture, - dockerAvailability: "unavailable", - exitCode: 0, - invocation: artifact(`${qualificationCase.id}-installer-invocation`), - script: artifact(`${qualificationCase.id}-installer-script`), - }, - runtime: { - provider: PODMAN_NATIVE_ACTIVATION_QUALIFICATION.provider, - profileId: qualificationCase.profile.id, - agent: qualificationCase.agent, - application: APPLICATIONS[qualificationCase.agent], - inference: qualificationCase.inference, - architecture: qualificationCase.profile.architecture, - acceleration: qualificationCase.profile.acceleration, - rootMode: "rootless", - engineName: "podman", - engineVersion: "5.6.2", - engineAuthority: { - schemaVersion: 1, - providerId: PODMAN_NATIVE_ACTIVATION_QUALIFICATION.provider, - operation: "host-local-inference", - engineId: "podman-rootless", - authorityId: "podman:host-local-inference", - bindingSha256: BINDING_SHA, - }, - managedImages: [ - { role: "agent" as const, imageRef: IMAGE_REFS.agent }, - { role: "probe" as const, imageRef: IMAGE_REFS.probe }, - ...(managed ? [{ role: "inference" as const, imageRef: IMAGE_REFS.inference }] : []), - ], - route: { - service: qualificationCase.inference, - endpoint: { - host: "podman.internal", - port: 8000, - networkName: "podman-inference", - gatewayProviderBaseUrl: "http://host.openshell.internal:8000/v1", - applicationBaseUrl: "https://inference.local/v1", - }, - authority: { - receiptSha256: AUTHORITY_SHA, - kind: managed ? ("container" as const) : ("host" as const), - runtimeId: managed ? `podman-${qualificationCase.inference}` : null, - containerName: managed ? `nemoclaw-${qualificationCase.inference}` : null, - specSha256: managed ? SPEC_SHA : null, - }, - }, - modelId: `${qualificationCase.inference}-qualification-model`, - inferenceResult: artifact(`${qualificationCase.id}-inference-result`), - }, - operations: qualificationCase.obligations.map((id) => ({ - id, - authoritySha256: AUTHORITY_SHA, - artifact: artifact(`${qualificationCase.id}-${id}`), - })), - recovery: { - status: "reconciled" as const, - authoritySha256: AUTHORITY_SHA, - artifact: artifact(`${qualificationCase.id}-recovery`), - }, - cleanup: { - status: managed ? ("removed-owned" as const) : ("retained-external" as const), - authoritySha256: AUTHORITY_SHA, - providerOwnedRuntimeIds: [], - artifact: artifact(`${qualificationCase.id}-cleanup`), - }, - ...(qualificationCase.profile.acceleration === "nvidia-gpu" - ? { - nvidiaCdi: { - devices: ["nvidia.com/gpu=all"] as const, - artifact: artifact(`${qualificationCase.id}-nvidia-cdi`), - }, - } - : {}), - }; - }); -} - -function evidenceArtifacts( - evidence: readonly NativeRuntimeQualificationEvidence[], -): QualificationArtifactReceipt[] { - return evidence.flatMap((entry) => [ - entry.installer.invocation, - entry.installer.script, - entry.runtime.inferenceResult, - ...entry.operations.map((operation) => operation.artifact), - entry.recovery.artifact, - entry.cleanup.artifact, - ...(entry.nvidiaCdi ? [entry.nvidiaCdi.artifact] : []), - ]); -} - -function qualificationFixture(): { - artifactRoot: string; - bindings: NativeRuntimeQualificationProtectedRunBinding[]; - cleanup: () => void; -} { - const artifactRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-native-qualification-")); - return { - artifactRoot, - bindings: completeEvidence().map((entry) => ({ - protectedRun: structuredClone(entry.protectedRun), - artifactRoot, - })), - cleanup: () => fs.rmSync(artifactRoot, { force: true, recursive: true }), - }; -} - -function writeEvidenceArtifacts( - artifactRoot: string, - evidence: readonly NativeRuntimeQualificationEvidence[], -): void { - for (const receipt of evidenceArtifacts(evidence)) { - const target = path.join(artifactRoot, ...receipt.path.split(/[\\/]/u)); - fs.mkdirSync(path.dirname(target), { recursive: true }); - fs.writeFileSync(target, ARTIFACT_CONTENT, "utf8"); - } -} - -function assertNativeRuntimeQualificationEvidence( - definition: typeof PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence: readonly NativeRuntimeQualificationEvidence[], -): void { - const fixture = qualificationFixture(); - try { - const reporter = createNativeRuntimeQualificationReporterRecord( - definition, - evidence, - fixture.bindings, - ); - writeEvidenceArtifacts(fixture.artifactRoot, evidence); - const verified = verifyNativeRuntimeQualificationReporterArtifacts(definition, reporter); - assertVerifiedNativeRuntimeQualificationEvidence(definition, verified); - } finally { - fixture.cleanup(); - } -} - describe("native runtime activation qualification", () => { it("compiles every required all-agent, multiarch, CPU/GPU, and local-inference case", () => { const qualification = PODMAN_NATIVE_ACTIVATION_QUALIFICATION; @@ -219,6 +26,16 @@ describe("native runtime activation qualification", () => { expect(new Set(qualification.cases.map((entry) => entry.inference))).toEqual( new Set(["ollama", "nim", "vllm"]), ); + expect( + qualification.cases + .filter((entry) => entry.profile.acceleration === "nvidia-gpu") + .every((entry) => entry.evidenceKinds.includes("nvidia-cdi")), + ).toBe(true); + expect( + qualification.cases + .filter((entry) => entry.profile.acceleration === "cpu") + .every((entry) => !entry.evidenceKinds.includes("nvidia-cdi")), + ).toBe(true); for (const entry of qualification.cases) { expect(entry).toMatchObject({ gate: "protected-e2e", @@ -350,408 +167,4 @@ describe("native runtime activation qualification", () => { }), ).toThrow(/must be socket-free/u); }); - - it("accepts only a complete exact evidence set for the compiled candidate", () => { - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - completeEvidence(), - ), - ).not.toThrow(); - - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - completeEvidence().slice(1), - ), - ).toThrow(/evidence is incomplete/u); - }); - - it("requires the canonical reporter and re-hashes every referenced artifact", () => { - expect(() => - assertVerifiedNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - completeEvidence() as unknown as VerifiedNativeRuntimeQualificationEvidence, - ), - ).toThrow(/verified canonical reporter evidence/u); - - const inventedDigest = completeEvidence(); - inventedDigest[0]!.installer.invocation.sha256 = "e".repeat(64); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - inventedDigest, - ), - ).toThrow(/digest does not match its receipt/u); - }); - - it("accepts the complete matrix from one protected job and rejects an unused binding", () => { - const evidence = completeEvidence(); - const materialized = qualificationFixture(); - const sharedProtectedRun = structuredClone(evidence[0]!.protectedRun); - for (const entry of evidence) { - entry.protectedRun = structuredClone(sharedProtectedRun); - } - const sharedBinding = { - protectedRun: structuredClone(sharedProtectedRun), - artifactRoot: materialized.artifactRoot, - }; - const unusedBinding = { - protectedRun: { ...sharedProtectedRun, jobId: 9999 }, - artifactRoot: materialized.artifactRoot, - }; - try { - writeEvidenceArtifacts(materialized.artifactRoot, evidence); - const reporter = createNativeRuntimeQualificationReporterRecord( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - [sharedBinding], - ); - const verified = verifyNativeRuntimeQualificationReporterArtifacts( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - reporter, - ); - - expect(() => - assertVerifiedNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - verified, - ), - ).not.toThrow(); - expect(() => - createNativeRuntimeQualificationReporterRecord( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - [sharedBinding, unusedBinding], - ), - ).toThrow(/unused protected-run binding/u); - } finally { - materialized.cleanup(); - } - }); - - it("binds receipts to independent run metadata and snapshots them before verification", () => { - const evidence = completeEvidence(); - const materialized = qualificationFixture(); - try { - const inventedRun = structuredClone(evidence); - inventedRun[0]!.protectedRun.runId = 999_999; - expect(() => - createNativeRuntimeQualificationReporterRecord( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - inventedRun, - materialized.bindings, - ), - ).toThrow(/no trusted protected-run binding/u); - - const reporter = createNativeRuntimeQualificationReporterRecord( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - materialized.bindings, - ); - evidence[0]!.installer.invocation.sha256 = "e".repeat(64); - writeEvidenceArtifacts(materialized.artifactRoot, completeEvidence()); - const verified = verifyNativeRuntimeQualificationReporterArtifacts( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - reporter, - ); - expect(() => - assertVerifiedNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - verified, - ), - ).not.toThrow(); - - const mxc = compileNativeRuntimeQualification(nativeRuntimeQualificationDefinition("mxc")); - expect(() => assertVerifiedNativeRuntimeQualificationEvidence(mxc, verified)).toThrow( - /verified canonical reporter evidence/u, - ); - } finally { - materialized.cleanup(); - } - }); - - it("rejects missing artifacts and symlink escapes from a bound job root", () => { - const missingEvidence = completeEvidence(); - const missing = qualificationFixture(); - try { - writeEvidenceArtifacts(missing.artifactRoot, missingEvidence); - const reporter = createNativeRuntimeQualificationReporterRecord( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - missingEvidence, - missing.bindings, - ); - const receipt = missingEvidence[0]!.installer.invocation; - fs.unlinkSync(path.join(missing.artifactRoot, ...receipt.path.split(/[\\/]/u))); - expect(() => - verifyNativeRuntimeQualificationReporterArtifacts( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - reporter, - ), - ).toThrow(/is missing/u); - } finally { - missing.cleanup(); - } - - const linkedEvidence = completeEvidence(); - const linked = qualificationFixture(); - const outside = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-native-outside-")); - try { - writeEvidenceArtifacts(linked.artifactRoot, linkedEvidence); - const reporter = createNativeRuntimeQualificationReporterRecord( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - linkedEvidence, - linked.bindings, - ); - const receipt = linkedEvidence[0]!.installer.invocation; - const target = path.join(linked.artifactRoot, ...receipt.path.split(/[\\/]/u)); - const outsideFile = path.join(outside, "artifact.json"); - fs.writeFileSync(outsideFile, ARTIFACT_CONTENT, "utf8"); - fs.unlinkSync(target); - fs.symlinkSync(outsideFile, target); - expect(() => - verifyNativeRuntimeQualificationReporterArtifacts( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - reporter, - ), - ).toThrow(/escapes its trusted root/u); - } finally { - linked.cleanup(); - fs.rmSync(outside, { force: true, recursive: true }); - } - }); - - it("rejects an intermediate directory replaced after validation but before open", () => { - const evidence = completeEvidence(); - const materialized = qualificationFixture(); - const outside = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-native-race-outside-")); - const receipt = evidence[0]!.installer.invocation; - const receiptParts = receipt.path.split(/[\\/]/u); - const canonicalArtifactRoot = fs.realpathSync(materialized.artifactRoot); - const intermediate = path.join(canonicalArtifactRoot, receiptParts[0]!); - const savedIntermediate = path.join(canonicalArtifactRoot, "qualification-before-race"); - const candidate = path.join(canonicalArtifactRoot, ...receiptParts); - const outsideTarget = path.join(outside, ...receiptParts.slice(1)); - let swapped = false; - const realOpen: typeof fs.openSync = fs.openSync.bind(fs); - let open: ReturnType | undefined; - try { - writeEvidenceArtifacts(materialized.artifactRoot, evidence); - fs.mkdirSync(path.dirname(outsideTarget), { recursive: true }); - fs.writeFileSync(outsideTarget, ARTIFACT_CONTENT, "utf8"); - const reporter = createNativeRuntimeQualificationReporterRecord( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - evidence, - materialized.bindings, - ); - open = vi - .spyOn(fs, "openSync") - .mockImplementationOnce(((target, flags, mode) => { - expect(String(target)).toBe(candidate); - fs.renameSync(intermediate, savedIntermediate); - fs.symlinkSync(outside, intermediate, "dir"); - swapped = true; - return realOpen(target, flags, mode); - }) as typeof fs.openSync) - .mockImplementation(realOpen); - - expect(() => - verifyNativeRuntimeQualificationReporterArtifacts( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - reporter, - ), - ).toThrow(/escapes its trusted root|changed during verification/u); - expect(swapped).toBe(true); - } finally { - open?.mockRestore(); - try { - fs.unlinkSync(intermediate); - fs.renameSync(savedIntermediate, intermediate); - } catch { - // The one-shot swap did not complete; the original directory remains in place. - } - materialized.cleanup(); - fs.rmSync(outside, { force: true, recursive: true }); - } - }); - - it("rejects inexact source, image, operation, and CDI receipts", () => { - const badSource = completeEvidence(); - badSource[0]!.protectedRun.headSha = "main"; - expect(() => - assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, badSource), - ).toThrow(/exact head\/base SHAs/u); - - const wrongWorkflow = completeEvidence(); - wrongWorkflow[0]!.protectedRun.workflow = "Unprotected runtime test"; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - wrongWorkflow, - ), - ).toThrow(/wrong workflow/u); - - const wrongWorkflowSource = structuredClone(completeEvidence()); - wrongWorkflowSource[0]!.protectedRun.workflowSha = "main"; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - wrongWorkflowSource, - ), - ).toThrow(/protected workflow SHA/u); - - const extendedProtectedRun = completeEvidence(); - Object.assign(extendedProtectedRun[0]!.protectedRun, { workerBinding: "untrusted" }); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - extendedProtectedRun, - ), - ).toThrow(/protected run schema is unsupported/u); - - const mixedSourcePair = completeEvidence(); - mixedSourcePair[0]!.protectedRun.headSha = "3".repeat(40); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - mixedSourcePair, - ), - ).toThrow(/one exact head\/base\/workflow source/u); - - const mixedWorkflowSource = structuredClone(completeEvidence()); - mixedWorkflowSource[0]!.protectedRun.workflowSha = "4".repeat(40); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - mixedWorkflowSource, - ), - ).toThrow(/one exact head\/base\/workflow source/u); - - const badImage = completeEvidence(); - badImage[0]!.runtime.managedImages = [ - { role: "agent", imageRef: "nvcr.io/nvidia/nemoclaw-agent:latest" }, - { role: "probe", imageRef: IMAGE_REFS.probe }, - ]; - expect(() => - assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, badImage), - ).toThrow(/exact image references/u); - - const badOperations = completeEvidence(); - badOperations[0]!.operations = badOperations[0]!.operations.slice(1); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - badOperations, - ), - ).toThrow(/operations is incomplete/u); - - const gpuEvidence = completeEvidence(); - const gpu = gpuEvidence.find((entry) => entry.runtime.acceleration === "nvidia-gpu")!; - gpu.nvidiaCdi = undefined; - expect(() => - assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, gpuEvidence), - ).toThrow(/must prove NVIDIA CDI access/u); - }); - - it("binds application, engine, endpoint, and managed-runtime identity", () => { - const wrongApplication = completeEvidence(); - const openclaw = wrongApplication.find((entry) => entry.runtime.agent === "openclaw")!; - openclaw.runtime.application = "hermes"; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - wrongApplication, - ), - ).toThrow(/invalid runtime identity/u); - - const wrongEngine = completeEvidence(); - wrongEngine[0]!.runtime.engineName = "docker"; - expect(() => - assertNativeRuntimeQualificationEvidence(PODMAN_NATIVE_ACTIVATION_QUALIFICATION, wrongEngine), - ).toThrow(/wrong runtime engine/u); - - const missingAuthority = completeEvidence(); - missingAuthority[0]!.runtime.engineAuthority.authorityId = ""; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - missingAuthority, - ), - ).toThrow(/exact non-empty runtime identifier/u); - - const wrongEndpoint = completeEvidence(); - wrongEndpoint[0]!.runtime.route.endpoint.gatewayProviderBaseUrl = - "http://host.openshell.internal:9000/v1"; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - wrongEndpoint, - ), - ).toThrow(/mismatched gateway endpoint/u); - - const missingManagedImage = completeEvidence(); - const managedImageCase = missingManagedImage.find( - (entry) => entry.runtime.inference === "nim", - )!; - managedImageCase.runtime.managedImages = managedImageCase.runtime.managedImages.filter( - (image) => image.role !== "inference", - ); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - missingManagedImage, - ), - ).toThrow(/managed image roles is incomplete/u); - - const missingRuntimeId = completeEvidence(); - const managedRuntimeCase = missingRuntimeId.find( - (entry) => entry.runtime.inference === "vllm", - )!; - managedRuntimeCase.runtime.route.authority.runtimeId = null; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - missingRuntimeId, - ), - ).toThrow(/must name managed inference/u); - }); - - it("binds lifecycle, recovery, and exact cleanup to one durable authority", () => { - const wrongOperation = completeEvidence(); - wrongOperation[0]!.operations[0]!.authoritySha256 = "e".repeat(64); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - wrongOperation, - ), - ).toThrow(/different runtime authority/u); - - const wrongRecovery = completeEvidence(); - wrongRecovery[0]!.recovery.authoritySha256 = "e".repeat(64); - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - wrongRecovery, - ), - ).toThrow(/invalid recovery evidence/u); - - const wrongCleanup = completeEvidence(); - const managedCleanupCase = wrongCleanup.find((entry) => entry.runtime.inference === "nim")!; - managedCleanupCase.cleanup.status = "retained-external"; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - wrongCleanup, - ), - ).toThrow(/invalid exact cleanup evidence/u); - - const residualRuntime = completeEvidence(); - residualRuntime[0]!.cleanup.providerOwnedRuntimeIds = ["podman-stale-container"]; - expect(() => - assertNativeRuntimeQualificationEvidence( - PODMAN_NATIVE_ACTIVATION_QUALIFICATION, - residualRuntime, - ), - ).toThrow(/invalid exact cleanup evidence/u); - }); }); From 326edb502fc6f835b151b0fcf4e1681f1e8bf486 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 13:59:53 -0700 Subject: [PATCH 10/10] test: reject malformed qualification evidence kinds Signed-off-by: Aaron Erickson --- .../e2e-native-runtime-qualification.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/e2e/support/e2e-native-runtime-qualification.test.ts b/test/e2e/support/e2e-native-runtime-qualification.test.ts index 7541f9c42bb..785870ff03c 100644 --- a/test/e2e/support/e2e-native-runtime-qualification.test.ts +++ b/test/e2e/support/e2e-native-runtime-qualification.test.ts @@ -136,6 +136,41 @@ describe("native runtime activation qualification", () => { ).toThrow(/evidence kinds is incomplete/u); }); + it("rejects duplicate and unknown exact-evidence declarations", () => { + const duplicateEvidence = nativeRuntimeQualificationDefinition("duplicate-evidence-runtime"); + const firstDuplicate = duplicateEvidence.cases[0]!; + expect(() => + compileNativeRuntimeQualification({ + ...duplicateEvidence, + cases: [ + { + ...firstDuplicate, + evidenceKinds: [...firstDuplicate.evidenceKinds, firstDuplicate.evidenceKinds[0]!], + }, + ...duplicateEvidence.cases.slice(1), + ], + }), + ).toThrow(/contains duplicate values/u); + + const unknownEvidence = nativeRuntimeQualificationDefinition("unknown-evidence-runtime"); + const firstUnknown = unknownEvidence.cases[0]!; + expect(() => + compileNativeRuntimeQualification({ + ...unknownEvidence, + cases: [ + { + ...firstUnknown, + evidenceKinds: [ + ...firstUnknown.evidenceKinds, + "worker-self-attested", + ] as unknown as typeof firstUnknown.evidenceKinds, + }, + ...unknownEvidence.cases.slice(1), + ], + }), + ).toThrow(/unknown: worker-self-attested/u); + }); + it("rejects Docker availability and Docker-socket substitutions", () => { const dockerPresent = nativeRuntimeQualificationDefinition("docker-present-runtime"); const first = dockerPresent.cases[0]!;