diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8998624a35d..8c62ddd195c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -306,7 +306,7 @@ repos: entry: npm run source-shape:check language: system pass_filenames: false - files: ^(test/|scripts/find-source-shape-tests\.ts$|ci/source-shape-test-budget\.json$) + files: ^(.+\.(test|spec)\.(js|ts|mjs|mts|cjs|cts)|scripts/find-source-shape-tests\.ts|ci/source-shape-test-budget\.json)$ priority: 20 - id: test-file-size-budget diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4c5e115869..cb4c1d4b571 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -226,6 +226,29 @@ npx vitest run --project e2e-support This project is fast and does not run live targets. Live E2E remains opt-in through `npm run test:live-e2e` or the applicable GitHub Actions workflow. +### Test Declarative Behavior + +Do not read a shipped YAML, JSON, manifest, workflow, or E2E runtime file only to assert its keys, +lists, or literal text. Schema tests should use small synthetic fixtures. Behavior tests should pass +the configuration through its consumer or validator and mutate important inputs to prove both the +accepted and rejected outcomes. + +A direct read may remain only when it protects a security or compatibility trust boundary that +cannot be observed at a more stable boundary. Put this annotation immediately above that one test +and give the concrete reason: + +```ts +// source-shape-contract: security -- Cross-field digest equality protects the shipped trust anchor +it("keeps both immutable image digests aligned", () => { + // ... +}); +``` + +`npm run source-shape:check` rejects unsupported categories, short or misplaced reasons, and any +exception whose file, test title, and category are not in the exact reviewed allowlist. It also +rejects unused allowlist entries, so one exception cannot silently replace another. Its output and +metrics list every accepted exception so these contracts remain visible during review. + ### Focused Vitest Feedback Use `npm run test:changed` for the staged, unstaged, and untracked changes in the current checkout, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 2835d6e2d93..3f10d934c8d 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -1,3 +1,455 @@ { - "maxSourceShapeCases": 0 + "maxSourceShapeCases": 0, + "sourceShapeContractExceptions": [ + { + "file": "src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts", + "test": "rejects host subprocess control and allowlist names as MCP credentials", + "category": "compatibility" + }, + { + "file": "src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts", + "test": "rejects OpenShell child-environment compatibility keys as MCP credentials", + "category": "compatibility" + }, + { + "file": "src/lib/agent/base-image-hermes.test.ts", + "test": "accepts only the tracked published Hermes base digest", + "category": "security" + }, + { + "file": "src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts", + "test": "matches the bundled local-inference host-gateway ports (#5744)", + "category": "compatibility" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "does not expose stale published-launchable controls", + "category": "security" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "fails closed on unsupported reusable test-suite values before checkout", + "category": "security" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "grants the reusable workflow permission ceiling so GitHub can start the run", + "category": "security" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "keeps instance deletion inside the workflow ownership boundary", + "category": "security" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "keeps manual dispatch inputs out of the Brev credential boundary", + "category": "security" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "keeps write permissions out of the secret-bearing target-branch job", + "category": "security" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "passes only declared inputs and secrets to branch validation", + "category": "compatibility" + }, + { + "file": "test/brev-nightly-workflow.test.ts", + "test": "verifies the pinned Brev CLI digest before extracting it", + "category": "security" + }, + { + "file": "test/cloudflared-update-check-workflow.test.ts", + "test": "keeps automatic and on-demand update checks reachable and credential-free", + "category": "security" + }, + { + "file": "test/code-scanning-workflow.test.ts", + "test": "groups CodeQL action updates so Dependabot keeps the shared revision synchronized", + "category": "security" + }, + { + "file": "test/code-scanning-workflow.test.ts", + "test": "keeps every CodeQL action on one immutable revision", + "category": "security" + }, + { + "file": "test/dcode-base-image-workflow.test.ts", + "test": "accepts every discovered publisher and rejects supply-chain mutations", + "category": "security" + }, + { + "file": "test/dcode-start-keepalive.test.ts", + "test": "execs an explicitly supplied command instead of idling", + "category": "compatibility" + }, + { + "file": "test/dcode-start-keepalive.test.ts", + "test": "refuses to launch when effective rlimits fail verification (#6545)", + "category": "security" + }, + { + "file": "test/dcode-start-keepalive.test.ts", + "test": "refuses to launch when the required rlimit helper is missing (#6545)", + "category": "security" + }, + { + "file": "test/dcode-start-keepalive.test.ts", + "test": "stays alive as a long-running process when invoked with no command", + "category": "compatibility" + }, + { + "file": "test/e2e-advisor.test.ts", + "test": "strips untrusted symlinks before secret-bearing advisor steps", + "category": "security" + }, + { + "file": "test/e2e-fixture-dependency-review.test.ts", + "test": "keeps installed fixture dependencies on exact versions", + "category": "security" + }, + { + "file": "test/e2e-release-gate-workflow.test.ts", + "test": "replaces legacy target_ref dispatches with the validated checkout contract", + "category": "security" + }, + { + "file": "test/e2e/live/hermes-e2e.test.ts", + "test": "hermes-e2e: install.sh onboards Hermes and proves health plus live inference", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "compiles absence probes for every preflight failure contract", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "compiles fail-closed absence probes for targets that forbid runtime side effects", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "compiles host-preservation probes for every post-reboot recovery target", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "indexes every registered state by its unique id", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "keeps policy-selection failures limited to the installed CLI", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "omits host gateway probes for targets whose loaded agent runtime is terminal", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "rejects an unknown state with an actionable inventory", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-expected-state.test.ts", + "test": "resolves every state id consumed by the typed target registry", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-live-project-config.test.ts", + "test": "keeps the drift-preflight bypass out of live projects (#6692)", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-live-registry-discovery.test.ts", + "test": "classifies every shipped target as supported or with a concrete reason", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-live-skip-name-contract.test.ts", + "test": "matches the workflow's exact `-t \"^${TARGET_ID}$\"` regex for every target", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-live-skip-name-contract.test.ts", + "test": "registers every target under a name equal to its id", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-manifests.test.ts", + "test": "resolves every typed target manifest path to a validated manifest", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-matrix.test.ts", + "test": "builds the default live matrix from every fixture-supported target", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-registry.test.ts", + "test": "CLI should emit multiple selected live matrix entries", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-registry.test.ts", + "test": "should return actionable unknown target error", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-workflow.test.ts", + "test": "derives test selectors from code and workflow jobs from workflow metadata", + "category": "compatibility" + }, + { + "file": "test/e2e/support/e2e-workflow.test.ts", + "test": "rejects channels stop/start workflow-boundary drift for secret and artifact handling", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-workflow.test.ts", + "test": "rejects credential-backed provider smokes in the PR-safe inference-routing job", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-workflow.test.ts", + "test": "rejects diagnostics workflow-boundary drift for secret and Docker auth handling", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-workflow.test.ts", + "test": "rejects duplicate unguarded Docker Hub auth in messaging-compatible-endpoint", + "category": "security" + }, + { + "file": "test/e2e/support/e2e-workflow.test.ts", + "test": "rejects free-standing E2E artifact uploads from raw temp paths", + "category": "security" + }, + { + "file": "test/fetch-guard-patch-regression.test.ts", + "test": "requires classifier review and integrity evidence when the OpenClaw build pin changes", + "category": "security" + }, + { + "file": "test/inference-options-docs.test.ts", + "test": "keeps a per-model task-fit comparison table for curated onboarding models", + "category": "compatibility" + }, + { + "file": "test/label-merged-pr-release-target-workflow.test.ts", + "test": "keeps fork-safe labeling inside the trusted metadata boundary", + "category": "security" + }, + { + "file": "test/macos-e2e-workflow-boundary.test.ts", + "test": "keeps secret-bearing live E2E on trusted main-branch code", + "category": "security" + }, + { + "file": "test/macos-e2e-workflow-boundary.test.ts", + "test": "pins live macOS artifact publishers to an immutable action", + "category": "security" + }, + { + "file": "test/macos-e2e-workflow-boundary.test.ts", + "test": "runs final-destroy against a pinned Docker setup on trusted Intel macOS", + "category": "compatibility" + }, + { + "file": "test/mcp-openshell-workflow.test.ts", + "test": "keeps the setup docs aligned with the stable default", + "category": "compatibility" + }, + { + "file": "test/openclaw-dependency-review.test.ts", + "test": "runs and gates the real patched-distribution harness only from trusted main code", + "category": "security" + }, + { + "file": "test/openclaw-lifecycle-policy.test.ts", + "test": "cross-checks the allowlist against every production archive install boundary", + "category": "security" + }, + { + "file": "test/plugin-vitest-project.test.ts", + "test": "defines one canonical plugin project for root and standalone runs", + "category": "compatibility" + }, + { + "file": "test/plugin-vitest-project.test.ts", + "test": "pilots assertion presence only in expect-based plugin tests (#6692)", + "category": "compatibility" + }, + { + "file": "test/pr-e2e-gate-workflow.test.ts", + "test": "limits triggers and job permissions", + "category": "security" + }, + { + "file": "test/pr-e2e-gate-workflow.test.ts", + "test": "pins both controller checkouts and installs without lifecycle scripts or caches", + "category": "security" + }, + { + "file": "test/pr-review-advisor-workflow-boundary.test.ts", + "test": "requires one advisor lane to publish the PR comment", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "does not persist checkout credentials in PR or main jobs", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "does not run npm lifecycle scripts during CI dependency installs", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "keeps the installer verifier inside the trusted composite action", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "keeps the trusted test-size guard closed around budget policy changes", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "pins downloaded CI tooling to reviewed integrity", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "publishes coverage only from same-repository code (#6692)", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "reuses the same shared CI actions in PR and main workflows", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "routes only code-changing PRs through the code-check path", + "category": "compatibility" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "runs pull request installer verification from immutable trusted code", + "category": "security" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "runs repository checks for every operational dependency-pin authority and consumer", + "category": "compatibility" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "runs the source-shape guard for root and co-located tests", + "category": "compatibility" + }, + { + "file": "test/pr-workflow-contract.test.ts", + "test": "scopes pre-push typechecks to project and transitive inputs", + "category": "compatibility" + }, + { + "file": "test/regression-e2e-workflow.test.ts", + "test": "collects the gateway drift regression from its integration project (#6692)", + "category": "compatibility" + }, + { + "file": "test/regression-e2e-workflow.test.ts", + "test": "prepares every discovered non-hermetic Vitest job before execution (#6692)", + "category": "security" + }, + { + "file": "test/regression-e2e-workflow.test.ts", + "test": "runs the OpenClaw custom-plugin lifecycle and EXDEV guard in a secret-free lane", + "category": "security" + }, + { + "file": "test/regression-e2e-workflow.test.ts", + "test": "runs WhatsApp compact QR through Vitest instead of the retired shell script", + "category": "compatibility" + }, + { + "file": "test/regression-e2e-workflow.test.ts", + "test": "stages the public NVIDIA key for the Model Router's NVIDIA credential", + "category": "security" + }, + { + "file": "test/repro-4538-raw-doctor-perms.test.ts", + "test": "emitted openclaw() guard restores the contract AND preserves a nonzero exit", + "category": "security" + }, + { + "file": "test/repro-4538-raw-doctor-perms.test.ts", + "test": "emitted openclaw() guard restores the contract even under an inherited `set -e`", + "category": "security" + }, + { + "file": "test/repro-4538-raw-doctor-perms.test.ts", + "test": "restore helper re-asserts 2770/660 after the tree is tightened to 700/600", + "category": "security" + }, + { + "file": "test/repro-5978-policy-denial-hint.test.ts", + "test": "prints only once when the file is sourced twice in one login shell", + "category": "compatibility" + }, + { + "file": "test/runner.test.ts", + "test": "walkthrough.sh does not embed NVIDIA_INFERENCE_API_KEY in tmux or sandbox commands", + "category": "security" + }, + { + "file": "test/validate-blueprint.test.ts", + "test": "pins the sandbox image by digest instead of a mutable tag (#1438)", + "category": "security" + }, + { + "file": "test/validate-blueprint.test.ts", + "test": "populates the top-level digest field with the image digest (#1438)", + "category": "security" + }, + { + "file": "test/vitest-coverage-thresholds.test.ts", + "test": "enforces the exact per-file security floors for full and merged coverage (#6692)", + "category": "security" + }, + { + "file": "test/vitest-developer-feedback.test.ts", + "test": "lets Vitest select reporters and preserves failed-test logs in CI (#6692)", + "category": "compatibility" + }, + { + "file": "test/vitest-state-isolation.test.ts", + "test": "keeps root and live projects free of unvalidated automatic cleanup", + "category": "security" + }, + { + "file": "test/vitest-state-isolation.test.ts", + "test": "keeps standalone plugin runs aligned without enabling mockReset", + "category": "compatibility" + }, + { + "file": "test/vitest-temp-root.test.ts", + "test": "wires cleanup into root and standalone plugin test runs", + "category": "compatibility" + }, + { + "file": "test/vitest-watch-triggers.test.ts", + "test": "registers the focused mappings at the root configuration boundary (#6692)", + "category": "compatibility" + } + ] } diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 126a65a54df..0fd77977b5e 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -6,7 +6,7 @@ "src/lib/inference/nim.test.ts": 2068, "src/lib/onboard/preflight.test.ts": 1904, "test/generate-openclaw-config.test.ts": 1941, - "test/install-preflight.test.ts": 3934, + "test/install-preflight.test.ts": 3921, "test/nemoclaw-start.test.ts": 4826, "test/onboard-messaging.test.ts": 2049, "test/onboard-selection.test.ts": 4769, diff --git a/scripts/checks/vitest-project-overlap.ts b/scripts/checks/vitest-project-overlap.ts index bd88c7e6b25..83dfdf033e0 100644 --- a/scripts/checks/vitest-project-overlap.ts +++ b/scripts/checks/vitest-project-overlap.ts @@ -41,6 +41,7 @@ const TEST_FILE_PATTERN = /\.(?:test|spec)\.(?:[cm]?[jt]sx?)$/; const SKIP_DIRECTORIES = new Set([".git", "node_modules"]); const INSTALLER_INTEGRATION_TESTS = new Set([ "test/install-build-dependency-preflight.test.ts", + "test/install-clone-ref.test.ts", "test/install-express-prompt.test.ts", "test/install-openshell-version-check.test.ts", "test/install-preflight-docker-bootstrap.test.ts", diff --git a/scripts/find-source-shape-tests.ts b/scripts/find-source-shape-tests.ts index d547bb1573b..54521b41402 100755 --- a/scripts/find-source-shape-tests.ts +++ b/scripts/find-source-shape-tests.ts @@ -2,11 +2,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // -// Finds tests that read production source text and assert on its shape. These -// tests tend to couple coverage to implementation strings instead of behavior. +// Finds tests that read production source or shipped declarative configuration +// and assert on its raw shape. These tests tend to couple coverage to +// implementation details instead of behavior. import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; -import { basename, dirname, join, relative } from "node:path"; +import { basename, dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import ts from "typescript"; @@ -34,16 +35,35 @@ type SourceShapeCase = { readonly sourceReads: readonly SourceRead[]; }; +type ContractCategory = "compatibility" | "security"; + +type SourceShapeContractException = SourceShapeCase & + Readonly<{ category: ContractCategory; reason: string }>; + +type ContractExceptionAllowance = Readonly<{ + file: string; + test: string; + category: ContractCategory; +}>; + +type InvalidContractException = Readonly<{ file: string; line: number; reason: string }>; + type Report = { readonly summary: { readonly source_shape_cases: number; readonly source_shape_assertions: number; readonly source_shape_files: number; readonly source_shape_max_cases_per_file: number; + readonly source_shape_contract_exceptions: number; + readonly source_shape_invalid_contract_exceptions: number; }; readonly cases: readonly SourceShapeCase[]; + readonly contractExceptions: readonly SourceShapeContractException[]; + readonly invalidContractExceptions: readonly InvalidContractException[]; }; +type FileReport = Pick; + type VariableDecl = { readonly name: string; readonly initializer: ts.Expression; @@ -99,12 +119,53 @@ function isTestFile(absPath: string): boolean { return TEST_NAME_PATTERN.test(basename(rel)); } -function stripStringLiterals(text: string): string { - return text.replace(/(['"`])(?:\\.|(?!\1)[\s\S])*\1/g, ""); +const TEXT_IDENTIFIER_CACHE = new Map>(); + +function isReferenceIdentifier(node: ts.Identifier): boolean { + const parent = node.parent; + return !( + (ts.isPropertyAccessExpression(parent) && parent.name === node) || + (ts.isPropertyAssignment(parent) && parent.name === node) || + (ts.isBindingElement(parent) && parent.propertyName === node) + ); +} + +function identifiersInText(text: string): ReadonlySet { + const cached = TEXT_IDENTIFIER_CACHE.get(text); + if (cached) return cached; + const identifiers = new Set(); + const sourceFile = ts.createSourceFile( + "source-shape-fragment.ts", + text, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS, + ); + function visit(node: ts.Node): void { + if (ts.isIdentifier(node) && isReferenceIdentifier(node)) identifiers.add(node.text); + ts.forEachChild(node, visit); + } + visit(sourceFile); + TEXT_IDENTIFIER_CACHE.set(text, identifiers); + return identifiers; } function textContainsIdentifier(text: string, identifier: string): boolean { - return new RegExp(`\\b${escapeRegExp(identifier)}\\b`).test(stripStringLiterals(text)); + return identifiersInText(text).has(identifier); +} + +function nodeReferencesAnyIdentifier(root: ts.Node, identifiers: ReadonlySet): boolean { + let found = false; + function visit(node: ts.Node): void { + if (found) return; + if (ts.isIdentifier(node) && identifiers.has(node.text) && isReferenceIdentifier(node)) { + found = true; + return; + } + ts.forEachChild(node, visit); + } + visit(root); + return found; } function escapeRegExp(value: string): string { @@ -117,27 +178,9 @@ function looksLikeTestFixturePath(text: string): boolean { /Dockerfile\.sandbox/.test(normalized) || /(?:^|\/)fixtures?\//.test(normalized) || /\.agents\/skills/.test(normalized) || - // The isolated E2E suite under test/e2e/ is entirely test - // assets and a self-contained runtime: target manifests, validation - // suite YAMLs, expected-state YAMLs, and the suite's own runner. Reads of - // these paths are not source-shape coupling to product code. - /(?:^|["'`/])test\/e2e(?:["'`/]|$)/.test(normalized) - ); -} - -function looksLikeDeclarativeConfigPath(text: string): boolean { - const normalized = normalizePathText(text); - // Declarative configs below have dedicated schema/resolver validation. Keep - // this scanner focused on source-code shape assertions rather than treating - // every schema-backed config invariant as source-text coupling. - return ( - /nemoclaw-blueprint\/blueprint\.yaml/.test(normalized) || - /nemoclaw-blueprint\/policies\//.test(normalized) || - /src\/lib\/messaging\/channels\/[^/]+\/policy\/[^/]+\.yaml/.test(normalized) || - /nemoclaw-blueprint\/provider-profiles\//.test(normalized) || - /nemoclaw-blueprint\/router\/pool-config\.yaml/.test(normalized) || - /nemoclaw-blueprint\/model-specific-setup\//.test(normalized) || - /agents\/[^/]+\/policy-(?:additions|permissive)\.yaml/.test(normalized) + (/(?:^|["'`/])test\/e2e\//.test(normalized) && + /\.(?:[cm]?[jt]s|sh|py|md)(?:["'`),]|\s)/.test(normalized) && + !/vitest(?:\.[\w-]+)?\.config\.[cm]?[jt]s/.test(normalized)) ); } @@ -146,7 +189,7 @@ function isProductionPathExpression( productionPathVars: ReadonlySet, ): boolean { const normalized = normalizePathText(text); - if (looksLikeTestFixturePath(normalized) || looksLikeDeclarativeConfigPath(normalized)) { + if (looksLikeTestFixturePath(normalized)) { return false; } if ([...productionPathVars].some((name) => textContainsIdentifier(normalized, name))) return true; @@ -156,16 +199,16 @@ function isProductionPathExpression( function hasDirectProductionPathHint(text: string): boolean { return ( - /["'`](?:\.\.\/)?(?:\.github|agents|bin|dist|nemoclaw|nemoclaw-blueprint|scripts|src|test\/e2e)\//.test( + /["'`](?:\.\.\/)*(?:\.github|agents|bin|dist|nemoclaw|nemoclaw-blueprint|scripts|src|test\/e2e)\//.test( text, ) || - /["'`](?:package\.json|install\.sh|\.pre-commit-config\.yaml)["'`]/.test(text) || - /["'`]\.\.\/Dockerfile(?:\.base)?["'`]/.test(text) || - /["'`]\.\.\/bin\//.test(text) || - /["'`]\.\.\/agents\//.test(text) || - /["'`]\.\.\/scripts\//.test(text) || - /["'`]\.\.\/src\//.test(text) || - /["'`]\.\.\/dist\//.test(text) || + /["'`](?:\.\.\/)*(?:package\.json|install\.sh|\.pre-commit-config\.yaml)["'`]/.test(text) || + /["'`](?:\.\.\/)+Dockerfile(?:\.base)?["'`]/.test(text) || + /["'`](?:\.\.\/)+bin\//.test(text) || + /["'`](?:\.\.\/)+agents\//.test(text) || + /["'`](?:\.\.\/)+scripts\//.test(text) || + /["'`](?:\.\.\/)+src\//.test(text) || + /["'`](?:\.\.\/)+dist\//.test(text) || /["'`]\.\.\/["'`]\s*,\s*["'`](?:\.github|agents|bin|dist|nemoclaw|nemoclaw-blueprint|scripts|src|Dockerfile(?:\.base)?|install\.sh|package\.json)["'`]/.test( text, ) || @@ -211,9 +254,121 @@ function isReadFileCall(node: ts.CallExpression): boolean { return false; } -function isExecutionResultDerivation(initText: string): boolean { - return /\b(?:spawnSync|execFileSync|execSync|run(?:Logged|Docker|Bash|WithLib|Embedded|Patch|Hermes|Openclaw|Daemon|Fetch|Command)\w*)\b/.test( - initText, +function isCommandExecutionText(text: string): boolean { + return /\b(?:spawn|spawnSync|exec|execFile|execFileSync|execSync|run(?:Logged|Docker|Bash|WithLib|Embedded|Patch|Hermes|Openclaw|Daemon|Fetch|Command)\w*)\b/.test( + text, + ); +} + +type DynamicFunctionBindings = { + readonly constructors: ReadonlySet; + readonly functions: ReadonlySet; +}; + +const DYNAMIC_FUNCTION_BINDING_CACHE = new WeakMap(); + +function unwrapTransparentExpression(expression: ts.Expression): ts.Expression { + let current = expression; + while ( + ts.isAwaitExpression(current) || + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + } + return current; +} + +function isDynamicFunctionConstructorFactory(expression: ts.Expression): boolean { + const current = unwrapTransparentExpression(expression); + if (!ts.isPropertyAccessExpression(current) || current.name.text !== "constructor") return false; + const prototypeCall = unwrapTransparentExpression(current.expression); + if ( + !ts.isCallExpression(prototypeCall) || + !ts.isPropertyAccessExpression(prototypeCall.expression) || + !ts.isIdentifier(prototypeCall.expression.expression) || + prototypeCall.expression.expression.text !== "Object" || + prototypeCall.expression.name.text !== "getPrototypeOf" + ) { + return false; + } + const candidate = prototypeCall.arguments[0]; + return Boolean( + candidate && + (ts.isArrowFunction(candidate) || ts.isFunctionExpression(candidate)) && + candidate.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword), + ); +} + +function dynamicFunctionBindings(sourceFile: ts.SourceFile): DynamicFunctionBindings { + const cached = DYNAMIC_FUNCTION_BINDING_CACHE.get(sourceFile); + if (cached) return cached; + const variables = collectVariableDecls(sourceFile); + const constructors = new Set(); + for (const variable of variables) { + if (isDynamicFunctionConstructorFactory(variable.initializer)) constructors.add(variable.name); + } + let changed = true; + while (changed) { + changed = false; + for (const variable of variables) { + if (constructors.has(variable.name)) continue; + const initializer = unwrapTransparentExpression(variable.initializer); + if (ts.isIdentifier(initializer) && constructors.has(initializer.text)) { + constructors.add(variable.name); + changed = true; + } + } + } + + const functions = new Set(); + for (const variable of variables) { + const initializer = unwrapTransparentExpression(variable.initializer); + if ( + ts.isNewExpression(initializer) && + ts.isIdentifier(initializer.expression) && + constructors.has(initializer.expression.text) + ) { + functions.add(variable.name); + } + } + const result = { constructors, functions }; + DYNAMIC_FUNCTION_BINDING_CACHE.set(sourceFile, result); + return result; +} + +function containsDynamicFunctionExecution(sourceFile: ts.SourceFile, root: ts.Node): boolean { + const { constructors, functions } = dynamicFunctionBindings(sourceFile); + let found = false; + function isConstructorCall(expression: ts.Expression): boolean { + const current = unwrapTransparentExpression(expression); + return ( + (ts.isNewExpression(current) || ts.isCallExpression(current)) && + ts.isIdentifier(current.expression) && + constructors.has(current.expression.text) + ); + } + function visit(node: ts.Node): void { + if (found) return; + if (ts.isCallExpression(node)) { + const callee = unwrapTransparentExpression(node.expression); + if ((ts.isIdentifier(callee) && functions.has(callee.text)) || isConstructorCall(callee)) { + found = true; + return; + } + } + ts.forEachChild(node, visit); + } + visit(root); + return found; +} + +function isExecutionResultDerivation(sourceFile: ts.SourceFile, root: ts.Node): boolean { + return ( + isCommandExecutionText(root.getText(sourceFile)) || + containsDynamicFunctionExecution(sourceFile, root) ); } @@ -225,12 +380,20 @@ function looksLikeSourceTreeEnumeration(text: string): boolean { return /\breaddirSync\s*\(/.test(text) && looksLikeSourceFileExtensionFilter(text); } +function bindingNames(name: ts.BindingName): string[] { + if (ts.isIdentifier(name)) return [name.text]; + return name.elements.flatMap((element) => + ts.isOmittedExpression(element) ? [] : bindingNames(element.name), + ); +} + function collectVariableDecls(sourceFile: ts.SourceFile): VariableDecl[] { const variables: VariableDecl[] = []; function visit(node: ts.Node): void { - if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) { - variables.push({ name: node.name.text, initializer: node.initializer }); + if (ts.isVariableDeclaration(node) && node.initializer) { + for (const name of bindingNames(node.name)) + variables.push({ name, initializer: node.initializer }); } ts.forEachChild(node, visit); } @@ -239,6 +402,218 @@ function collectVariableDecls(sourceFile: ts.SourceFile): VariableDecl[] { return variables; } +function isDeclarativeImportPath(value: string): boolean { + return ( + /\.(?:json|ya?ml|toml|lock)$/.test(value) || + /vitest(?:\.[\w-]+)?\.config(?:\.[cm]?[jt]s)?$/.test(value) + ); +} + +function moduleLoadSpecifier(expression: ts.Expression): string | null { + let current = expression; + while ( + ts.isAwaitExpression(current) || + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) || + ts.isPropertyAccessExpression(current) || + ts.isElementAccessExpression(current) + ) { + current = current.expression; + } + if (!ts.isCallExpression(current) || current.arguments.length === 0) return null; + const isRequire = ts.isIdentifier(current.expression) && current.expression.text === "require"; + const isDynamicImport = current.expression.kind === ts.SyntaxKind.ImportKeyword; + const specifier = current.arguments[0]; + return (isRequire || isDynamicImport) && ts.isStringLiteralLike(specifier) + ? specifier.text + : null; +} + +function containsShippedDeclarativeLoad(sourceFile: ts.SourceFile, root: ts.Node): boolean { + let found = false; + const sourcePath = resolve(REPO_ROOT, sourceFile.fileName); + function visit(node: ts.Node): void { + if (found) return; + if (ts.isExpression(node)) { + const specifier = moduleLoadSpecifier(node); + if (specifier?.startsWith(".")) { + const importedPath = normalizePathText( + relative(REPO_ROOT, resolve(dirname(sourcePath), specifier)), + ); + if (isDeclarativeImportPath(importedPath) && !looksLikeTestFixturePath(importedPath)) { + found = true; + return; + } + } + } + ts.forEachChild(node, visit); + } + visit(root); + return found; +} + +type ImportBinding = { + readonly local: string; + readonly imported: string; + readonly path: string; + readonly sourceRead: SourceRead; + readonly node: ts.Node; +}; + +function collectImportBindings(sourceFile: ts.SourceFile, relPath: string): ImportBinding[] { + const result: ImportBinding[] = []; + const add = ( + local: string, + imported: string, + specifier: string, + node: ts.Node, + expression: string, + ): void => { + if (!specifier.startsWith(".")) return; + const path = normalizePathText( + relative(REPO_ROOT, resolve(REPO_ROOT, dirname(relPath), specifier)), + ); + const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart()); + result.push({ + local, + imported, + path, + node, + sourceRead: { + line: line + 1, + column: character + 1, + variable: local, + expression, + }, + }); + }; + + function visit(node: ts.Node): void { + if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) { + const clause = node.importClause; + if (clause && !clause.isTypeOnly) { + const specifier = node.moduleSpecifier.text; + const expression = node.getText(sourceFile); + if (clause.name) add(clause.name.text, "default", specifier, node, expression); + const bindings = clause.namedBindings; + if (bindings && ts.isNamespaceImport(bindings)) { + add(bindings.name.text, "*", specifier, node, expression); + } + if (bindings && ts.isNamedImports(bindings)) { + for (const element of bindings.elements) { + if (!element.isTypeOnly) { + add( + element.name.text, + element.propertyName?.text ?? element.name.text, + specifier, + node, + expression, + ); + } + } + } + } + } + if (ts.isVariableDeclaration(node) && node.initializer) { + const specifier = moduleLoadSpecifier(node.initializer); + if (specifier) { + for (const local of bindingNames(node.name)) { + add(local, "*", specifier, node, node.initializer.getText(sourceFile)); + } + } + } + ts.forEachChild(node, visit); + } + visit(sourceFile); + return result; +} + +function scopedImportBindings( + sourceFile: ts.SourceFile, + imports: readonly ImportBinding[], + testCall: ts.CallExpression, + body: ts.Node, +): ImportBinding[] { + return imports.filter((binding) => { + if (isAncestor(body, binding.node)) return true; + const scope = nearestLexicalScope(binding.node); + return scope === sourceFile || isAncestor(scope, testCall); + }); +} + +function collectProductionConsumerNames(imports: readonly ImportBinding[]): Set { + return new Set( + imports + .filter( + ({ path }) => + !isDeclarativeImportPath(path) && + !TEST_NAME_PATTERN.test(path) && + /^(?:agents|bin|nemoclaw\/src|scripts|src|tools)\//.test(path), + ) + .map(({ local }) => local), + ); +} + +function isProductionBehaviorDerivation( + value: ts.Expression, + productionConsumerNames: ReadonlySet, +): boolean { + let expression = value; + while ( + ts.isCallExpression(expression) || + ts.isAwaitExpression(expression) || + ts.isParenthesizedExpression(expression) || + ts.isAsExpression(expression) || + ts.isNonNullExpression(expression) || + ts.isPropertyAccessExpression(expression) || + ts.isElementAccessExpression(expression) + ) { + expression = expression.expression; + } + return ts.isIdentifier(expression) && productionConsumerNames.has(expression.text); +} + +function collectDeclarativeImports(imports: readonly ImportBinding[]): Map { + return new Map( + imports + .filter(({ path }) => isDeclarativeImportPath(path) && !looksLikeTestFixturePath(path)) + .map(({ local, sourceRead }) => [local, sourceRead]), + ); +} + +const RAW_CONFIG_ACCESSORS: Readonly> = { + "test/helpers/e2e-workflow-contract": ["readWorkflow", "readYaml"], + "test/e2e/registry/registry": ["getTarget", "listTargets", "requireTargets"], + "test/e2e/registry/expected-states": [ + "getExpectedState", + "listExpectedStates", + "requireExpectedState", + ], +}; + +function collectRawConfigAccessors(imports: readonly ImportBinding[]): Map { + const accessors = new Map(); + for (const binding of imports) { + const module = binding.path.replace(/\.[cm]?[jt]s$/, ""); + const exportedNames = RAW_CONFIG_ACCESSORS[module]; + if (!exportedNames) continue; + const names = binding.imported === "*" ? exportedNames : [binding.imported]; + for (const imported of names) { + if (!exportedNames.includes(imported)) continue; + const name = binding.imported === "*" ? `${binding.local}.${imported}` : binding.local; + accessors.set(name, { + name, + parameterNames: [], + parameterizedPathRead: imported === "readYaml", + sourceRead: binding.sourceRead, + }); + } + } + return accessors; +} + function isAncestor(ancestor: ts.Node, node: ts.Node): boolean { let current: ts.Node | undefined = node; while (current) { @@ -270,6 +645,100 @@ function scopedVariableDecls( }); } +function collectSetupHookAssignments( + sourceFile: ts.SourceFile, + testCall: ts.CallExpression, +): VariableDecl[] { + const setupHookNames = new Set(["beforeAll", "beforeEach"]); + for (const statement of sourceFile.statements) { + if ( + !ts.isImportDeclaration(statement) || + !ts.isStringLiteral(statement.moduleSpecifier) || + statement.moduleSpecifier.text !== "vitest" || + !statement.importClause?.namedBindings || + !ts.isNamedImports(statement.importClause.namedBindings) + ) { + continue; + } + for (const element of statement.importClause.namedBindings.elements) { + const imported = element.propertyName?.text ?? element.name.text; + if (imported === "beforeAll" || imported === "beforeEach") { + setupHookNames.add(element.name.text); + } + } + } + const functionLikes = collectFunctionLikes(sourceFile); + const functionAliases = collectVariableDecls(sourceFile).filter((variable) => + ts.isIdentifier(variable.initializer), + ); + const visibleBindings = new Set(); + function collectVisibleBindings(node: ts.Node): void { + if (ts.isVariableDeclaration(node)) { + const scope = nearestLexicalScope(node); + if (scope === sourceFile || isAncestor(scope, testCall)) { + for (const name of bindingNames(node.name)) visibleBindings.add(name); + } + } + ts.forEachChild(node, collectVisibleBindings); + } + collectVisibleBindings(sourceFile); + + const assignments: VariableDecl[] = []; + function visit(node: ts.Node): void { + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + setupHookNames.has(node.expression.text) + ) { + const scope = nearestLexicalScope(node); + const appliesToTest = scope === sourceFile || isAncestor(scope, testCall); + const inlineCallback = node.arguments.find( + (argument): argument is ts.ArrowFunction | ts.FunctionExpression => + ts.isArrowFunction(argument) || ts.isFunctionExpression(argument), + ); + const callbackReference = node.arguments.find(ts.isIdentifier); + let callbackName = callbackReference?.text; + const seenAliases = new Set(); + while (callbackName && !seenAliases.has(callbackName)) { + seenAliases.add(callbackName); + const alias = functionAliases.find((candidate) => { + if (candidate.name !== callbackName) return false; + const aliasScope = nearestLexicalScope(candidate.initializer); + return aliasScope === sourceFile || isAncestor(aliasScope, node); + }); + if (!alias || !ts.isIdentifier(alias.initializer)) break; + callbackName = alias.initializer.text; + } + const referencedCallback = callbackName + ? functionLikes.find((candidate) => { + if (candidate.name !== callbackName) return false; + const callbackScope = nearestLexicalScope(candidate.node); + return callbackScope === sourceFile || isAncestor(callbackScope, node); + }) + : undefined; + const callbackBody = inlineCallback?.body ?? referencedCallback?.body; + if (appliesToTest && callbackBody) { + function collectAssignments(child: ts.Node): void { + if (child !== callbackBody && isNestedFunctionLike(child)) return; + if ( + ts.isBinaryExpression(child) && + child.operatorToken.kind === ts.SyntaxKind.EqualsToken && + ts.isIdentifier(child.left) && + visibleBindings.has(child.left.text) + ) { + assignments.push({ name: child.left.text, initializer: child.right }); + } + ts.forEachChild(child, collectAssignments); + } + collectAssignments(callbackBody); + } + } + ts.forEachChild(node, visit); + } + visit(sourceFile); + return assignments; +} + function collectProductionPathVars( sourceFile: ts.SourceFile, variables: readonly VariableDecl[], @@ -291,7 +760,6 @@ function collectProductionPathVars( [...pathVars].some((name) => textContainsIdentifier(initText, name)); if ( !looksLikeTestFixturePath(initText) && - !looksLikeDeclarativeConfigPath(initText) && (isRepositoryRoot || directlyNamesProductionPath || derivesNamedProductionPath) ) { pathVars.add(variable.name); @@ -305,6 +773,9 @@ function collectProductionPathVars( function callTargetName(expression: ts.Expression): string | null { if (ts.isIdentifier(expression)) return expression.text; + if (ts.isPropertyAccessExpression(expression) && ts.isIdentifier(expression.expression)) { + return `${expression.expression.text}.${expression.name.text}`; + } return null; } @@ -312,26 +783,38 @@ function nestedSourceReadInNode( sourceFile: ts.SourceFile, root: ts.Node, productionPathVars: ReadonlySet, + sourceFunctions: ReadonlyMap, ): SourceRead | null { let sourceRead: SourceRead | null = null; function visit(node: ts.Node): void { if (sourceRead) return; if (isNestedFunctionLike(node)) return; - if ( - ts.isCallExpression(node) && - isReadFileCall(node) && - node.arguments.length > 0 && - isProductionPathExpression(node.arguments[0].getText(sourceFile), productionPathVars) - ) { - const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart()); - sourceRead = { - line: line + 1, - column: character + 1, - variable: "", - expression: node.getText(sourceFile), - }; - return; + if (ts.isCallExpression(node)) { + const target = callTargetName(node.expression); + const functionRead = target ? sourceFunctions.get(target) : undefined; + const readsProductionFile = + isReadFileCall(node) && + node.arguments.length > 0 && + isProductionPathExpression(node.arguments[0].getText(sourceFile), productionPathVars); + const readsThroughFunction = + functionRead && + (!functionRead.parameterizedPathRead || + node.arguments.some((argument) => + isProductionPathExpression(argument.getText(sourceFile), productionPathVars), + )); + if (readsProductionFile || readsThroughFunction) { + const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart()); + sourceRead = { + line: line + 1, + column: character + 1, + variable: "", + expression: functionRead + ? `${node.getText(sourceFile)} -> ${functionRead.sourceRead.expression}` + : node.getText(sourceFile), + }; + return; + } } ts.forEachChild(node, visit); } @@ -345,56 +828,26 @@ function sourceReadFromInitializer( variable: VariableDecl, productionPathVars: ReadonlySet, sourceFunctions: ReadonlyMap, + productionConsumerNames: ReadonlySet, ): SourceRead | null { const init = variable.initializer; - const nestedRead = nestedSourceReadInNode(sourceFile, init, productionPathVars); - if (nestedRead) { - return { ...nestedRead, variable: variable.name }; - } - if (!ts.isCallExpression(init)) { + if ( + isProductionBehaviorDerivation(init, productionConsumerNames) || + isExecutionResultDerivation(sourceFile, init) + ) { return null; } - - if (isReadFileCall(init) && init.arguments.length > 0) { - const targetText = init.arguments[0].getText(sourceFile); - if (!isProductionPathExpression(targetText, productionPathVars)) { - return null; - } - - const { line, character } = sourceFile.getLineAndCharacterOfPosition( - variable.initializer.getStart(), - ); + if (containsShippedDeclarativeLoad(sourceFile, init)) { + const { line, character } = sourceFile.getLineAndCharacterOfPosition(init.getStart()); return { line: line + 1, column: character + 1, variable: variable.name, - expression: variable.initializer.getText(sourceFile), + expression: init.getText(sourceFile), }; } - - const targetName = callTargetName(init.expression); - const functionSourceRead = targetName ? sourceFunctions.get(targetName) : undefined; - if (!functionSourceRead) { - return null; - } - if ( - functionSourceRead.parameterizedPathRead && - !init.arguments.some((argument) => - isProductionPathExpression(argument.getText(sourceFile), productionPathVars), - ) - ) { - return null; - } - - const { line, character } = sourceFile.getLineAndCharacterOfPosition( - variable.initializer.getStart(), - ); - return { - line: line + 1, - column: character + 1, - variable: variable.name, - expression: `${variable.initializer.getText(sourceFile)} -> ${functionSourceRead.sourceRead.expression}`, - }; + const nestedRead = nestedSourceReadInNode(sourceFile, init, productionPathVars, sourceFunctions); + return nestedRead ? { ...nestedRead, variable: variable.name } : null; } function isNestedFunctionLike(node: ts.Node): boolean { @@ -427,6 +880,19 @@ function functionLikeNameAndBody(node: ts.Node): { return null; } +function collectFunctionLikes( + sourceFile: ts.SourceFile, +): NonNullable>[] { + const result: NonNullable>[] = []; + function visit(node: ts.Node): void { + const candidate = functionLikeNameAndBody(node); + if (candidate) result.push(candidate); + ts.forEachChild(node, visit); + } + visit(sourceFile); + return result; +} + function collectSourceTreeFunctionNames(sourceFile: ts.SourceFile): Set { const names = new Set(); @@ -490,8 +956,7 @@ function collectSourceFunctions( name: string, node: ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction, ): void { - const functionText = node.getText(sourceFile); - if (isExecutionResultDerivation(functionText)) return; + if (isExecutionResultDerivation(sourceFile, node)) return; let sourceRead: SourceRead | null = null; let parameterizedPathRead = false; @@ -540,6 +1005,43 @@ function collectSourceFunctions( return sourceFunctions; } +function collectRawProjectionFunctionNames( + sourceFile: ts.SourceFile, + candidates: readonly NonNullable>[], + testCall: ts.CallExpression, + testBodyNode: ts.Node, + sourceVars: ReadonlySet, + rawHelperNames: ReadonlySet, + productionConsumerNames: ReadonlySet, +): Set { + const projections = new Set(); + const scopedCandidates = candidates.filter((candidate) => { + const scope = nearestLexicalScope(candidate.node); + return ( + isAncestor(testBodyNode, candidate.node) || + scope === sourceFile || + isAncestor(scope, testCall) + ); + }); + for (let hop = 0; hop < 2; hop += 1) { + for (const candidate of scopedCandidates) { + if (projections.has(candidate.name)) continue; + const returned = ts.isBlock(candidate.body) + ? candidate.body.statements.find(ts.isReturnStatement)?.expression + : candidate.body; + if ( + !(returned && isProductionBehaviorDerivation(returned, productionConsumerNames)) && + !isExecutionResultDerivation(sourceFile, candidate.body) && + (nodeReferencesAnyIdentifier(candidate.body, new Set([...sourceVars, ...projections])) || + callsRawShapeHelper(candidate.body, rawHelperNames)) + ) { + projections.add(candidate.name); + } + } + } + return projections; +} + function collectSourceTreeShapeVars( sourceFile: ts.SourceFile, body: ts.Node, @@ -576,8 +1078,7 @@ function collectSourceTreeShapeVars( changed = false; for (const variable of variables) { if (sourceVars.has(variable.name)) continue; - const initText = variable.initializer.getText(sourceFile); - if ([...sourceVars].some((name) => textContainsIdentifier(initText, name))) { + if (nodeReferencesAnyIdentifier(variable.initializer, sourceVars)) { sourceVars.add(variable.name); changed = true; } @@ -586,10 +1087,7 @@ function collectSourceTreeShapeVars( function visit(node: ts.Node): void { if (ts.isForOfStatement(node)) { - const expressionText = node.expression.getText(sourceFile); - const iteratesSourceTree = [...sourceVars].some((name) => - textContainsIdentifier(expressionText, name), - ); + const iteratesSourceTree = nodeReferencesAnyIdentifier(node.expression, sourceVars); if (iteratesSourceTree) { const initializer = node.initializer; if (ts.isVariableDeclarationList(initializer)) { @@ -608,15 +1106,80 @@ function collectSourceTreeShapeVars( return { sourceVars, pathVars, sourceTreeFunctions }; } +function rootIdentifier(expression: ts.Expression): string | null { + let current = expression; + while ( + ts.isCallExpression(current) || + ts.isPropertyAccessExpression(current) || + ts.isElementAccessExpression(current) || + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isNonNullExpression(current) + ) { + current = current.expression; + } + return ts.isIdentifier(current) ? current.text : null; +} + +function propagateAccumulatorSourceVars( + sourceFile: ts.SourceFile, + root: ts.Node, + sourceVars: Set, + sourceFunctions: ReadonlyMap, +): boolean { + let changed = false; + const functionNames = new Set(sourceFunctions.keys()); + const referencesSource = (node: ts.Node): boolean => + nodeReferencesAnyIdentifier(node, sourceVars) || callsRawShapeHelper(node, functionNames); + const add = (name: string | null): void => { + if (name && !sourceVars.has(name)) { + sourceVars.add(name); + changed = true; + } + }; + function visit(node: ts.Node): void { + if (node !== root && isNestedFunctionLike(node)) return; + if (ts.isForOfStatement(node) && referencesSource(node.expression)) { + if (ts.isVariableDeclarationList(node.initializer)) { + for (const declaration of node.initializer.declarations) { + for (const name of bindingNames(declaration.name)) add(name); + } + } + } + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + /^(?:add|push|set)$/.test(node.expression.name.text) && + node.arguments.some(referencesSource) + ) { + add(rootIdentifier(node.expression.expression)); + } + if ( + ts.isBinaryExpression(node) && + node.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + node.operatorToken.kind <= ts.SyntaxKind.LastAssignment && + referencesSource(node.right) + ) { + add(rootIdentifier(node.left)); + } + ts.forEachChild(node, visit); + } + visit(root); + return changed; +} + function collectSourceVars( sourceFile: ts.SourceFile, + root: ts.Node, variables: readonly VariableDecl[], productionPathVars: ReadonlySet, sourceFunctions: ReadonlyMap, + productionConsumerNames: ReadonlySet, initialSourceVars: ReadonlySet = new Set(), + initialSourceReads: readonly SourceRead[] = [], ): { sourceVars: Set; sourceReads: SourceRead[] } { const sourceVars = new Set(initialSourceVars); - const sourceReads: SourceRead[] = []; + const sourceReads: SourceRead[] = [...initialSourceReads]; for (const variable of variables) { const sourceRead = sourceReadFromInitializer( @@ -624,6 +1187,7 @@ function collectSourceVars( variable, productionPathVars, sourceFunctions, + productionConsumerNames, ); if (sourceRead) { sourceVars.add(variable.name); @@ -637,25 +1201,61 @@ function collectSourceVars( for (const variable of variables) { if (sourceVars.has(variable.name)) continue; const initText = variable.initializer.getText(sourceFile); - const referencesSource = [...sourceVars].some((name) => - textContainsIdentifier(initText, name), - ); + const referencesSource = nodeReferencesAnyIdentifier(variable.initializer, sourceVars); const readsProductionFileCollection = isReadFileExpressionText(initText) && [...productionPathVars].some((name) => textContainsIdentifier(initText, name)); if ( (referencesSource || readsProductionFileCollection) && - !isExecutionResultDerivation(initText) + !isExecutionResultDerivation(sourceFile, variable.initializer) && + !isProductionBehaviorDerivation(variable.initializer, productionConsumerNames) ) { sourceVars.add(variable.name); changed = true; } } + if (propagateAccumulatorSourceVars(sourceFile, root, sourceVars, sourceFunctions)) + changed = true; } return { sourceVars, sourceReads }; } +function collectExecutionResultVars( + sourceFile: ts.SourceFile, + candidates: readonly NonNullable>[], + variables: readonly VariableDecl[], +): Set { + const executionFunctionNames = new Set(); + for (let hop = 0; hop < 2; hop += 1) { + for (const candidate of candidates) { + if ( + isExecutionResultDerivation(sourceFile, candidate.body) || + nodeReferencesAnyIdentifier(candidate.body, executionFunctionNames) + ) { + executionFunctionNames.add(candidate.name); + } + } + } + const executionVars = new Set(); + let changed = true; + while (changed) { + changed = false; + for (const variable of variables) { + if (executionVars.has(variable.name)) continue; + const root = rootIdentifier(variable.initializer); + if ( + isExecutionResultDerivation(sourceFile, variable.initializer) || + (root && (executionVars.has(root) || executionFunctionNames.has(root))) + ) { + executionVars.add(variable.name); + changed = true; + } + } + } + return executionVars; +} + function getExpectBase(expression: ts.Expression): ts.CallExpression | null { if (ts.isCallExpression(expression)) { if (ts.isIdentifier(expression.expression) && expression.expression.text === "expect") { @@ -679,6 +1279,16 @@ function matcherName(expression: ts.Expression): string { return expression.getText(); } +function callsRawShapeHelper(root: ts.Node, helperNames: ReadonlySet): boolean { + const target = ts.isCallExpression(root) ? callTargetName(root.expression) : null; + if (target && helperNames.has(target)) return true; + let found = false; + ts.forEachChild(root, (child) => { + found ||= callsRawShapeHelper(child, helperNames); + }); + return found; +} + function assertionFromSubject( sourceFile: ts.SourceFile, node: ts.CallExpression, @@ -686,18 +1296,27 @@ function assertionFromSubject( matcher: string, sourceVars: ReadonlySet, productionPathVars: ReadonlySet, + sourceTreeFunctions: ReadonlySet, + productionConsumerNames: ReadonlySet, ): Assertion | null { + if (isProductionBehaviorDerivation(subjectExpr, productionConsumerNames)) return null; const subject = subjectExpr.getText(sourceFile); if (/\bfs\.statSync\(/.test(subject)) { return null; } - const referencesSource = [...sourceVars].some((name) => textContainsIdentifier(subject, name)); + const referencesSource = nodeReferencesAnyIdentifier(subjectExpr, sourceVars); const directSourceRead = ts.isCallExpression(subjectExpr) && isReadFileCall(subjectExpr) && subjectExpr.arguments.length > 0 && isProductionPathExpression(subjectExpr.arguments[0].getText(sourceFile), productionPathVars); - if (!referencesSource && !directSourceRead) { + const directDeclarativeLoad = containsShippedDeclarativeLoad(sourceFile, subjectExpr); + if ( + !referencesSource && + !directSourceRead && + !directDeclarativeLoad && + !callsRawShapeHelper(subjectExpr, sourceTreeFunctions) + ) { return null; } @@ -712,6 +1331,8 @@ function assertionFromSubject( } const ASSERT_MATCHERS = new Set([ + "deepEqual", + "deepStrictEqual", "doesNotMatch", "doesNotReject", "doesNotThrow", @@ -724,6 +1345,114 @@ const ASSERT_MATCHERS = new Set([ "notEqual", "notStrictEqual", "ok", + "partialDeepStrictEqual", + "rejects", + "strictEqual", + "throws", +]); + +type AssertBindings = { + readonly namespaces: ReadonlySet; + readonly methods: ReadonlyMap; +}; + +const ASSERT_BINDING_CACHE = new WeakMap(); +const NODE_ASSERT_MODULES = new Set([ + "assert", + "assert/strict", + "node:assert", + "node:assert/strict", +]); + +function nodeAssertBindings(sourceFile: ts.SourceFile): AssertBindings { + const cached = ASSERT_BINDING_CACHE.get(sourceFile); + if (cached) return cached; + const namespaces = new Set(["assert"]); + const methods = new Map(); + const addNamedBinding = (local: string, imported: string): void => { + if (imported === "strict") namespaces.add(local); + else methods.set(local, imported); + }; + + function visit(node: ts.Node): void { + if ( + ts.isImportDeclaration(node) && + ts.isStringLiteral(node.moduleSpecifier) && + NODE_ASSERT_MODULES.has(node.moduleSpecifier.text) + ) { + const clause = node.importClause; + if (clause && !clause.isTypeOnly) { + if (clause.name) namespaces.add(clause.name.text); + const bindings = clause.namedBindings; + if (bindings && ts.isNamespaceImport(bindings)) namespaces.add(bindings.name.text); + if (bindings && ts.isNamedImports(bindings)) { + for (const element of bindings.elements) { + if (!element.isTypeOnly) { + addNamedBinding(element.name.text, element.propertyName?.text ?? element.name.text); + } + } + } + } + } + if (ts.isVariableDeclaration(node) && node.initializer) { + const initializer = unwrapTransparentExpression(node.initializer); + const specifier = moduleLoadSpecifier(node.initializer); + if (specifier && NODE_ASSERT_MODULES.has(specifier)) { + if (ts.isIdentifier(node.name)) { + const imported = ts.isPropertyAccessExpression(initializer) + ? initializer.name.text + : "strict"; + addNamedBinding(node.name.text, imported); + } + if (ts.isObjectBindingPattern(node.name)) { + for (const element of node.name.elements) { + if (!ts.isIdentifier(element.name)) continue; + const imported = + element.propertyName && ts.isIdentifier(element.propertyName) + ? element.propertyName.text + : element.name.text; + addNamedBinding(element.name.text, imported); + } + } + } else if (ts.isObjectBindingPattern(node.name)) { + const root = rootIdentifier(initializer); + if (root && namespaces.has(root)) { + for (const element of node.name.elements) { + if (!ts.isIdentifier(element.name)) continue; + const imported = element.propertyName?.getText(sourceFile) ?? element.name.text; + addNamedBinding(element.name.text, imported); + } + } + } else if (ts.isIdentifier(node.name)) { + if (ts.isIdentifier(initializer) && methods.has(initializer.text)) { + methods.set(node.name.text, methods.get(initializer.text) ?? ""); + } else if (ts.isPropertyAccessExpression(initializer)) { + const root = rootIdentifier(initializer); + if (root && namespaces.has(root)) { + addNamedBinding(node.name.text, initializer.name.text); + } + } + } + } + ts.forEachChild(node, visit); + } + visit(sourceFile); + const result = { namespaces, methods }; + ASSERT_BINDING_CACHE.set(sourceFile, result); + return result; +} + +const ASSERT_TWO_ARGUMENT_MATCHERS = new Set([ + "deepEqual", + "deepStrictEqual", + "doesNotMatch", + "equal", + "match", + "notDeepEqual", + "notDeepStrictEqual", + "notEqual", + "notStrictEqual", + "partialDeepStrictEqual", "rejects", "strictEqual", "throws", @@ -734,27 +1463,49 @@ function assertionFromAssertCall( node: ts.CallExpression, sourceVars: ReadonlySet, productionPathVars: ReadonlySet, + sourceTreeFunctions: ReadonlySet, + productionConsumerNames: ReadonlySet, + executionResultVars: ReadonlySet, ): Assertion | null { const expression = node.expression; - if (!ts.isPropertyAccessExpression(expression)) { - return null; - } - if (!ts.isIdentifier(expression.expression) || expression.expression.text !== "assert") { - return null; + const bindings = nodeAssertBindings(sourceFile); + let method: string | null = null; + if (ts.isIdentifier(expression)) { + if (bindings.namespaces.has(expression.text)) method = "ok"; + else method = bindings.methods.get(expression.text) ?? null; + } else if (ts.isPropertyAccessExpression(expression)) { + const root = rootIdentifier(expression); + if (root && bindings.namespaces.has(root)) { + method = expression.name.text === "strict" ? "ok" : expression.name.text; + } } - const method = expression.name.text; - if (!ASSERT_MATCHERS.has(method) || node.arguments.length === 0) { + if (!method || !ASSERT_MATCHERS.has(method) || node.arguments.length === 0) { return null; } - return assertionFromSubject( - sourceFile, - node, - node.arguments[0], - `assert.${method}`, - sourceVars, - productionPathVars, + const firstArgument = node.arguments[0]; + const firstRoot = firstArgument ? rootIdentifier(firstArgument) : null; + const firstIsBehavior = Boolean( + firstArgument && + (isProductionBehaviorDerivation(firstArgument, productionConsumerNames) || + isExecutionResultDerivation(sourceFile, firstArgument) || + (firstRoot && executionResultVars.has(firstRoot))), ); + const argumentCount = ASSERT_TWO_ARGUMENT_MATCHERS.has(method) && !firstIsBehavior ? 2 : 1; + for (const argument of node.arguments.slice(0, argumentCount)) { + const assertion = assertionFromSubject( + sourceFile, + node, + argument, + `assert.${method}`, + sourceVars, + productionPathVars, + sourceTreeFunctions, + productionConsumerNames, + ); + if (assertion) return assertion; + } + return null; } function expressionReferencesSource( @@ -762,19 +1513,14 @@ function expressionReferencesSource( sourceVars: ReadonlySet, productionPathVars: ReadonlySet, sourceTreeFunctions: ReadonlySet, + productionConsumerNames: ReadonlySet, ): boolean { - const text = expression.getText(); - const callsSourceTreeHelper = - ts.isCallExpression(expression) && - ts.isIdentifier(expression.expression) && - sourceTreeFunctions.has(expression.expression.text) && - (expression.arguments.length === 0 || - expression.arguments.some((argument) => - isProductionPathExpression(argument.getText(), productionPathVars), - )); + if (isProductionBehaviorDerivation(expression, productionConsumerNames)) return false; + const callsSourceTreeHelper = callsRawShapeHelper(expression, sourceTreeFunctions); return ( - [...sourceVars].some((name) => textContainsIdentifier(text, name)) || + nodeReferencesAnyIdentifier(expression, sourceVars) || callsSourceTreeHelper || + containsShippedDeclarativeLoad(expression.getSourceFile(), expression) || (ts.isCallExpression(expression) && isReadFileCall(expression) && expression.arguments.length > 0 && @@ -788,6 +1534,8 @@ function assertionFromExpectCall( sourceVars: ReadonlySet, productionPathVars: ReadonlySet, sourceTreeFunctions: ReadonlySet, + productionConsumerNames: ReadonlySet, + executionResultVars: ReadonlySet, ): Assertion | null { if ( sourceVars.size > 0 && @@ -810,7 +1558,8 @@ function assertionFromExpectCall( if (!expectBase || expectBase.arguments.length === 0) { return null; } - + const subjectRoot = rootIdentifier(expectBase.arguments[0]); + if (subjectRoot && executionResultVars.has(subjectRoot)) return null; const subjectAssertion = assertionFromSubject( sourceFile, node, @@ -818,12 +1567,20 @@ function assertionFromExpectCall( matcherName(node.expression), sourceVars, productionPathVars, + sourceTreeFunctions, + productionConsumerNames, ); if (subjectAssertion) return subjectAssertion; if ( node.arguments.some((argument) => - expressionReferencesSource(argument, sourceVars, productionPathVars, sourceTreeFunctions), + expressionReferencesSource( + argument, + sourceVars, + productionPathVars, + sourceTreeFunctions, + productionConsumerNames, + ), ) ) { const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart()); @@ -845,6 +1602,8 @@ function assertionFromCall( sourceVars: ReadonlySet, productionPathVars: ReadonlySet, sourceTreeFunctions: ReadonlySet, + productionConsumerNames: ReadonlySet, + executionResultVars: ReadonlySet, ): Assertion | null { return ( assertionFromExpectCall( @@ -853,7 +1612,18 @@ function assertionFromCall( sourceVars, productionPathVars, sourceTreeFunctions, - ) || assertionFromAssertCall(sourceFile, node, sourceVars, productionPathVars) + productionConsumerNames, + executionResultVars, + ) || + assertionFromAssertCall( + sourceFile, + node, + sourceVars, + productionPathVars, + sourceTreeFunctions, + productionConsumerNames, + executionResultVars, + ) ); } @@ -902,6 +1672,8 @@ function collectAssertionsInNode( sourceVars: ReadonlySet, productionPathVars: ReadonlySet, sourceTreeFunctions: ReadonlySet = new Set(), + productionConsumerNames: ReadonlySet = new Set(), + executionResultVars: ReadonlySet = new Set(), ): Assertion[] { const assertions: Assertion[] = []; @@ -913,6 +1685,8 @@ function collectAssertionsInNode( sourceVars, productionPathVars, sourceTreeFunctions, + productionConsumerNames, + executionResultVars, ); if (assertion) assertions.push(assertion); } @@ -960,7 +1734,15 @@ function fallbackLineScan(sourceFile: ts.SourceFile, root: ts.Node): Assertion[] function visit(node: ts.Node): void { if (ts.isCallExpression(node)) { - const assertion = assertionFromCall(sourceFile, node, sourceVars, new Set(), new Set()); + const assertion = assertionFromCall( + sourceFile, + node, + sourceVars, + new Set(), + new Set(), + new Set(), + new Set(), + ); if (assertion) assertions.push(assertion); } ts.forEachChild(node, visit); @@ -969,16 +1751,83 @@ function fallbackLineScan(sourceFile: ts.SourceFile, root: ts.Node): Assertion[] return assertions; } -function scanSourceText(fileName: string, relPath: string, text: string): SourceShapeCase[] { +type ContractAnnotation = { + readonly line: number; + readonly category?: SourceShapeContractException["category"]; + readonly reason?: string; + readonly error?: string; +}; + +const CONTRACT_CATEGORIES = new Set(["compatibility", "security"]); + +function collectContractAnnotations(sourceFile: ts.SourceFile, text: string): ContractAnnotation[] { + const annotations: ContractAnnotation[] = []; + const commentRanges = new Map(); + function collectRanges(node: ts.Node): void { + for (const range of ts.getLeadingCommentRanges(text, node.getFullStart()) ?? []) { + commentRanges.set(range.pos, range); + } + for (const range of ts.getTrailingCommentRanges(text, node.getEnd()) ?? []) { + commentRanges.set(range.pos, range); + } + ts.forEachChild(node, collectRanges); + } + collectRanges(sourceFile); + + for (const range of commentRanges.values()) { + if (range.kind !== ts.SyntaxKind.SingleLineCommentTrivia) continue; + const comment = text.slice(range.pos, range.end); + if (!comment.includes("source-shape-contract:")) continue; + const { line } = sourceFile.getLineAndCharacterOfPosition(range.pos); + const match = comment.match(/^\/\/\s*source-shape-contract:\s*([a-z-]+)\s+--\s+(.+?)\s*$/); + const category = match?.[1]; + const reason = match?.[2]; + const error = !match + ? "invalid annotation syntax" + : !category || !CONTRACT_CATEGORIES.has(category) + ? "unsupported category" + : !reason || reason.length < 24 || reason.trim().split(/\s+/).length < 4 + ? "reason is too short" + : undefined; + annotations.push({ + line: line + 1, + ...(error + ? { error } + : { category: category as SourceShapeContractException["category"], reason }), + }); + } + return annotations; +} + +function scanSourceTextReport(fileName: string, relPath: string, text: string): FileReport { const sourceFile = ts.createSourceFile(fileName, text, ts.ScriptTarget.Latest, true); + const functionLikes = collectFunctionLikes(sourceFile); const allVariables = collectVariableDecls(sourceFile); + const imports = collectImportBindings(sourceFile, relPath); + const annotations = collectContractAnnotations(sourceFile, text); + const annotationsByLine = new Map(annotations.map((annotation) => [annotation.line, annotation])); + const usedAnnotationLines = new Set(); const cases: SourceShapeCase[] = []; + const contractExceptions: SourceShapeContractException[] = []; + const invalidContractExceptions: InvalidContractException[] = []; function visit(node: ts.Node): void { if (ts.isCallExpression(node) && isTestCall(node)) { const body = testBody(node); if (body) { - const variables = scopedVariableDecls(sourceFile, allVariables, node, body); + const scopedImports = scopedImportBindings(sourceFile, imports, node, body); + const productionConsumerNames = collectProductionConsumerNames(scopedImports); + const declarativeImports = collectDeclarativeImports(scopedImports); + const rawConfigAccessors = collectRawConfigAccessors(scopedImports); + const variables = [ + ...scopedVariableDecls(sourceFile, allVariables, node, body), + ...collectSetupHookAssignments(sourceFile, node), + ]; + const executionResultVars = collectExecutionResultVars( + sourceFile, + functionLikes, + variables, + ); const productionPathVars = collectProductionPathVars(sourceFile, variables); const sourceTreeShapeVars = collectSourceTreeShapeVars( sourceFile, @@ -988,33 +1837,78 @@ function scanSourceText(fileName: string, relPath: string, text: string): Source ); const sourcePathVars = new Set([...productionPathVars, ...sourceTreeShapeVars.pathVars]); const sourceFunctions = collectSourceFunctions(sourceFile, sourcePathVars); - const { sourceVars, sourceReads } = collectSourceVars( + for (const [name, accessor] of rawConfigAccessors) sourceFunctions.set(name, accessor); + const initialSourceVars = new Set([ + ...sourceTreeShapeVars.sourceVars, + ...declarativeImports.keys(), + ]); + const initialSourceReads = [...declarativeImports.values()]; + const sourceCollection = collectSourceVars( sourceFile, + body, variables, sourcePathVars, sourceFunctions, - sourceTreeShapeVars.sourceVars, + productionConsumerNames, + initialSourceVars, + initialSourceReads, + ); + const rawProjectionFunctions = collectRawProjectionFunctionNames( + sourceFile, + functionLikes, + node, + body, + sourceCollection.sourceVars, + new Set(sourceFunctions.keys()), + productionConsumerNames, ); + const { sourceVars, sourceReads } = sourceCollection; const assertions = dedupeAssertions([ ...collectAssertionsInNode( sourceFile, body, sourceVars, sourcePathVars, - sourceTreeShapeVars.sourceTreeFunctions, + new Set([ + ...sourceTreeShapeVars.sourceTreeFunctions, + ...rawProjectionFunctions, + ...rawConfigAccessors.keys(), + ]), + productionConsumerNames, + executionResultVars, ), ...fallbackLineScan(sourceFile, body), ]); if (assertions.length > 0) { const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart()); - cases.push({ + const entry: SourceShapeCase = { file: relPath, line: line + 1, column: character + 1, name: testCaseName(sourceFile, node), assertions, sourceReads, - }); + }; + const annotation = annotationsByLine.get(line); + if (annotation) { + usedAnnotationLines.add(annotation.line); + if (annotation.category && annotation.reason) { + contractExceptions.push({ + ...entry, + category: annotation.category, + reason: annotation.reason, + }); + } else { + invalidContractExceptions.push({ + file: relPath, + line: annotation.line, + reason: annotation.error ?? "invalid annotation", + }); + cases.push(entry); + } + } else { + cases.push(entry); + } } } } @@ -1022,30 +1916,54 @@ function scanSourceText(fileName: string, relPath: string, text: string): Source } visit(sourceFile); - return cases; + for (const annotation of annotations) { + if (usedAnnotationLines.has(annotation.line)) continue; + invalidContractExceptions.push({ + file: relPath, + line: annotation.line, + reason: annotation.error ?? "annotation must be immediately above a detected test", + }); + } + return { cases, contractExceptions, invalidContractExceptions }; } -function scanFile(absPath: string): SourceShapeCase[] { +function scanSourceText(fileName: string, relPath: string, text: string): SourceShapeCase[] { + return [...scanSourceTextReport(fileName, relPath, text).cases]; +} + +function scanFile(absPath: string): FileReport { const relPath = normalizePathText(relative(REPO_ROOT, absPath)); const text = readFileSync(absPath, "utf-8"); - return scanSourceText(absPath, relPath, text); + return scanSourceTextReport(absPath, relPath, text); } -function scan(): Report { - const cases = [...walkFiles(REPO_ROOT)].filter(isTestFile).flatMap(scanFile); +export function sourceShapeSummary(report: FileReport): Report["summary"] { const casesPerFile = new Map(); - for (const entry of cases) { + for (const entry of report.cases) { casesPerFile.set(entry.file, (casesPerFile.get(entry.file) ?? 0) + 1); } + return { + source_shape_cases: report.cases.length, + source_shape_assertions: report.cases.reduce((sum, entry) => sum + entry.assertions.length, 0), + source_shape_files: casesPerFile.size, + source_shape_max_cases_per_file: Math.max(0, ...casesPerFile.values()), + source_shape_contract_exceptions: report.contractExceptions.length, + source_shape_invalid_contract_exceptions: report.invalidContractExceptions.length, + }; +} +function scan(): Report { + const fileReports = [...walkFiles(REPO_ROOT)].filter(isTestFile).map(scanFile); + const cases = fileReports.flatMap((report) => report.cases); + const contractExceptions = fileReports.flatMap((report) => report.contractExceptions); + const invalidContractExceptions = fileReports.flatMap( + (report) => report.invalidContractExceptions, + ); return { - summary: { - source_shape_cases: cases.length, - source_shape_assertions: cases.reduce((sum, entry) => sum + entry.assertions.length, 0), - source_shape_files: casesPerFile.size, - source_shape_max_cases_per_file: Math.max(0, ...casesPerFile.values()), - }, + summary: sourceShapeSummary({ cases, contractExceptions, invalidContractExceptions }), cases, + contractExceptions, + invalidContractExceptions, }; } @@ -1058,29 +1976,87 @@ function printMetrics(report: Report): void { function printHuman(report: Report): void { if (report.cases.length === 0) { console.log("No source-shape tests detected."); - printMetrics(report); - return; - } - - console.log(`Detected ${report.summary.source_shape_cases} source-shape test cases:`); - for (const testCase of report.cases) { - console.log(`- ${testCase.file}:${testCase.line}:${testCase.column} ${testCase.name}`); - for (const assertion of testCase.assertions) { - console.log( - ` - ${assertion.line}:${assertion.column} ${assertion.matcher} on ${assertion.subject}`, - ); + } else { + console.log(`Detected ${report.summary.source_shape_cases} source-shape test cases:`); + for (const testCase of report.cases) { + console.log(`- ${testCase.file}:${testCase.line}:${testCase.column} ${testCase.name}`); + for (const assertion of testCase.assertions) { + console.log( + ` - ${assertion.line}:${assertion.column} ${assertion.matcher} on ${assertion.subject}`, + ); + } } } + for (const exception of report.contractExceptions) { + console.log( + `EXCEPTION ${exception.file}:${exception.line} ${exception.category} -- ${exception.reason}`, + ); + } + for (const invalid of report.invalidContractExceptions) { + console.log(`INVALID ${invalid.file}:${invalid.line} ${invalid.reason}`); + } printMetrics(report); } +function contractExceptionKey(file: string, test: string, category: ContractCategory): string { + return `${file}\0${test}\0${category}`; +} + +export function contractExceptionAllowlistErrors( + actual: readonly Pick[], + allowed: readonly ContractExceptionAllowance[], +): string[] { + const actualKeyList = actual.map(({ file, name, category }) => + contractExceptionKey(file, name, category), + ); + const actualKeys = new Set(actualKeyList); + const allowedKeys = new Set( + allowed.map(({ file, test, category }) => contractExceptionKey(file, test, category)), + ); + return [ + ...[...actualKeys] + .filter((key) => actualKeyList.filter((candidate) => candidate === key).length > 1) + .map((key) => `duplicate source-shape exception identity: ${key.replaceAll("\0", " :: ")}`), + ...[...actualKeys] + .filter((key) => !allowedKeys.has(key)) + .map((key) => `unapproved source-shape exception: ${key.replaceAll("\0", " :: ")}`), + ...[...allowedKeys] + .filter((key) => !actualKeys.has(key)) + .map((key) => `unused source-shape exception allowance: ${key.replaceAll("\0", " :: ")}`), + ]; +} + function checkBudget(report: Report): void { const budgetPath = join(REPO_ROOT, "ci", "source-shape-test-budget.json"); const budget = JSON.parse(readFileSync(budgetPath, "utf-8")) as { readonly maxSourceShapeCases?: unknown; + readonly sourceShapeContractExceptions?: unknown; }; - if (typeof budget.maxSourceShapeCases !== "number") { - throw new Error(`${budgetPath} must define numeric maxSourceShapeCases`); + if ( + typeof budget.maxSourceShapeCases !== "number" || + !Array.isArray(budget.sourceShapeContractExceptions) + ) { + throw new Error( + `${budgetPath} must define numeric maxSourceShapeCases and sourceShapeContractExceptions[]`, + ); + } + const allowed = budget.sourceShapeContractExceptions as ContractExceptionAllowance[]; + for (const entry of allowed) { + const keys = entry && typeof entry === "object" ? Object.keys(entry).sort() : []; + if ( + keys.join(",") !== "category,file,test" || + typeof entry.file !== "string" || + typeof entry.test !== "string" || + !CONTRACT_CATEGORIES.has(entry.category) + ) { + throw new Error(`${budgetPath} has an invalid sourceShapeContractExceptions entry`); + } + } + if ( + new Set(allowed.map(({ file, test, category }) => contractExceptionKey(file, test, category))) + .size !== allowed.length + ) { + throw new Error(`${budgetPath} has duplicate sourceShapeContractExceptions entries`); } const actual = report.summary.source_shape_cases; @@ -1088,7 +2064,18 @@ function checkBudget(report: Report): void { console.error( `Source-shape test budget exceeded: ${actual} cases > ${budget.maxSourceShapeCases}.`, ); - console.error("Replace source-text assertions with behavior tests, then ratchet the budget."); + console.error( + "Replace raw source/config assertions with behavior tests, then ratchet the budget.", + ); + process.exitCode = 1; + } + const allowlistErrors = contractExceptionAllowlistErrors(report.contractExceptions, allowed); + if (allowlistErrors.length > 0) { + for (const error of allowlistErrors) console.error(error); + process.exitCode = 1; + } + if (report.invalidContractExceptions.length > 0) { + console.error(`Invalid source-shape exceptions: ${report.invalidContractExceptions.length}.`); process.exitCode = 1; } } @@ -1114,6 +2101,10 @@ export function scanTextForTest(relPath: string, text: string): SourceShapeCase[ return scanSourceText(relPath, normalizePathText(relPath), text); } +export function scanTextForTestReport(relPath: string, text: string): FileReport { + return scanSourceTextReport(relPath, normalizePathText(relPath), text); +} + function isDirectInvocation(): boolean { const invoked = process.argv[1]; return Boolean( diff --git a/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts index 20c654be597..1128356eb6e 100644 --- a/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-input-validation.test.ts @@ -39,21 +39,8 @@ describe("MCP CLI input validation", () => { ).toThrow(/process arguments and shell history/); }); + // source-shape-contract: compatibility -- Pinned OpenShell child-visible keys must drive credential rejection through every MCP boundary it("rejects OpenShell child-environment compatibility keys as MCP credentials", () => { - expect(childVisibleCredentialManifest).toMatchObject({ - openshellVersion: "0.0.72", - openshellCommit: "8cb16de9eae4c44d7d31e1493747d8c10abb5963", - }); - expect(childVisibleCredentialManifest.rawChildValueKeys).toEqual([ - "GCP_PROJECT_ID", - "GOOGLE_CLOUD_PROJECT", - "CLOUD_ML_REGION", - "GCP_LOCATION", - "GCP_SERVICE_ACCOUNT_EMAIL", - "GOOSE_PROVIDER", - "ANTHROPIC_VERTEX_PROJECT_ID", - "VERTEX_LOCATION", - ]); for (const name of childVisibleCredentialManifest.rawChildValueKeys) { expect(() => parseMcpAddArgs(["github", "--url", "https://mcp.example.test/mcp", "--env", name]), @@ -68,11 +55,6 @@ describe("MCP CLI input validation", () => { ).toThrow(/materialized as a raw child-process value/); } - expect(childVisibleCredentialManifest.rewrittenChildValueKeys).toEqual([ - "GCE_METADATA_HOST", - "GCE_METADATA_IP", - "METADATA_SERVER_DETECTION", - ]); for (const name of childVisibleCredentialManifest.rewrittenChildValueKeys) { expect(() => parseMcpAddArgs(["github", "--url", "https://mcp.example.test/mcp", "--env", name]), @@ -80,6 +62,7 @@ describe("MCP CLI input validation", () => { } }); + // source-shape-contract: compatibility -- Host subprocess controls must stay synchronized with the pinned OpenShell child environment boundary it("rejects host subprocess control and allowlist names as MCP credentials", () => { for (const name of SUBPROCESS_ENV_ALLOWED_NAMES) { expect(childVisibleCredentialManifest.runtimeControlKeys).toContain(name); diff --git a/src/lib/actions/sandbox/mcp-bridge-runtime-validation.test.ts b/src/lib/actions/sandbox/mcp-bridge-runtime-validation.test.ts index 75dfe44951b..842d815b3f4 100644 --- a/src/lib/actions/sandbox/mcp-bridge-runtime-validation.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-runtime-validation.test.ts @@ -5,10 +5,8 @@ import { describe, expect, it } from "vitest"; import { assertMcpCredentialBoundaryRuntimeVersion, - MCP_CREDENTIAL_BOUNDARY_OPENSHELL_VERSION, McpCredentialBoundaryRuntimeVersionError, } from "./mcp-bridge-validation"; -import childVisibleCredentialManifest from "./openshell-child-visible-credentials.v0.0.72.json"; function matchingOpenshellRuntime() { return { @@ -56,9 +54,6 @@ describe("MCP credential-boundary runtime validation", () => { detail: "version mismatch", reason: "version-mismatch", }); - expect(MCP_CREDENTIAL_BOUNDARY_OPENSHELL_VERSION).toBe( - childVisibleCredentialManifest.openshellVersion, - ); }); it("fails closed when the runtime binary is missing (#6426)", () => { diff --git a/src/lib/agent/base-image-hermes.test.ts b/src/lib/agent/base-image-hermes.test.ts index e2204763c2e..64480801924 100644 --- a/src/lib/agent/base-image-hermes.test.ts +++ b/src/lib/agent/base-image-hermes.test.ts @@ -39,6 +39,7 @@ describe("agent base image provisioning", () => { }); }); + // source-shape-contract: security -- Tracked immutable Hermes base digest must feed the production image resolver it("accepts only the tracked published Hermes base digest", () => { const dockerfilePath = path.resolve(import.meta.dirname, "../../../agents/hermes/Dockerfile"); const dockerfile = fs.readFileSync(dockerfilePath, "utf8"); diff --git a/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts b/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts index 8eb070e598b..421e6084942 100644 --- a/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts +++ b/src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts @@ -12,6 +12,7 @@ import { } from "./compatible-endpoint-gateway-route"; describe("compatible endpoint gateway routing", () => { + // source-shape-contract: compatibility -- Bundled loopback routing must match the shipped host-gateway policy ports it("matches the bundled local-inference host-gateway ports (#5744)", () => { const policyPath = path.resolve( import.meta.dirname, diff --git a/test/brev-nightly-workflow.test.ts b/test/brev-nightly-workflow.test.ts index 7cc06931786..40ccdfb4665 100644 --- a/test/brev-nightly-workflow.test.ts +++ b/test/brev-nightly-workflow.test.ts @@ -49,6 +49,7 @@ describe("Brev nightly workflow contract", () => { const nightly = readYaml(".github/workflows/brev-nightly-e2e.yaml"); const branchValidation = readYaml(".github/workflows/e2e-branch-validation.yaml"); + // source-shape-contract: compatibility -- Caller arguments must remain within the reusable branch-validation interface it("passes only declared inputs and secrets to branch validation", () => { const declaredInputs = new Set(Object.keys(branchValidation.on?.workflow_call?.inputs ?? {})); const declaredSecrets = new Set(Object.keys(branchValidation.on?.workflow_call?.secrets ?? {})); @@ -68,6 +69,7 @@ describe("Brev nightly workflow contract", () => { } }); + // source-shape-contract: security -- Caller permissions must equal the reviewed reusable-workflow write ceiling it("grants the reusable workflow permission ceiling so GitHub can start the run", () => { expect(nightly.permissions).toEqual(branchValidation.permissions); expect(nightly.permissions).toEqual({ @@ -77,6 +79,7 @@ describe("Brev nightly workflow contract", () => { }); }); + // source-shape-contract: security -- Secret-bearing validation stays read-only while reporting writes remain isolated it("keeps write permissions out of the secret-bearing target-branch job", () => { const caller = nightly.jobs?.["brev-nightly-e2e"]; const validation = branchValidation.jobs?.["e2e-branch-validation"]; @@ -118,10 +121,7 @@ describe("Brev nightly workflow contract", () => { expect(JSON.stringify(reporter)).not.toMatch(/BREV_|NVIDIA_INFERENCE_API_KEY/); }); - it("keeps every suite in the nightly matrix in a distinct concurrency group", () => { - expect(branchValidation.concurrency?.group).toContain("inputs.test_suite"); - }); - + // source-shape-contract: security -- Suite validation must reject unsupported input before any target checkout it("fails closed on unsupported reusable test-suite values before checkout", () => { const steps = branchValidation.jobs?.["e2e-branch-validation"]?.steps ?? []; const validation = steps.find((step) => step.name === "Validate test suite"); @@ -137,22 +137,10 @@ describe("Brev nightly workflow contract", () => { ); }); - it("runs stateful messaging targets on separate fresh instances", () => { - expect(nightly.jobs?.["brev-nightly-e2e"]?.strategy?.matrix?.test_suite).toEqual([ - "all", - "messaging-providers", - "messaging-compatible-endpoint", - "full", - ]); - expect(branchValidation.jobs?.["e2e-branch-validation"]?.["timeout-minutes"]).toBe(130); - }); - - it("keeps failure diagnostics ahead of workflow-owned instance deletion", () => { + // source-shape-contract: security -- Ownership and keep-alive guards prevent deleting contributor-managed Brev instances + it("keeps instance deletion inside the workflow ownership boundary", () => { const steps = branchValidation.jobs?.["e2e-branch-validation"]?.steps ?? []; const run = steps.find((step) => step.name === "Run ephemeral Brev E2E"); - const collect = steps.find((step) => step.name === "Collect Brev debug bundle on failure"); - const uploadDebug = steps.find((step) => step.name === "Upload Brev debug bundle on failure"); - const uploadLogs = steps.find((step) => step.name === "Upload test logs"); const cleanup = steps.find((step) => step.name === "Delete Brev instance"); expect(branchValidation.on?.workflow_call?.inputs?.keep_alive).toMatchObject({ @@ -161,28 +149,14 @@ describe("Brev nightly workflow contract", () => { expect(run?.env?.[BREV_WORKFLOW_OWNERSHIP_ENV]).toBe("1"); expect(cleanup?.if).toBe("always() && !inputs.keep_alive"); expect(cleanup?.env?.INSTANCE).toBe("${{ env.BREV_E2E_INSTANCE_NAME }}"); - expect(uploadDebug?.with?.name).toBe( - "brev-debug-bundle-${{ inputs.test_suite }}-${{ github.run_attempt }}", - ); - expect(uploadLogs?.with?.name).toBe( - "e2e-branch-validation-logs-${{ inputs.test_suite }}-${{ github.run_attempt }}", - ); expect(cleanup?.run).toContain("for attempt in 1 2 3"); expect(cleanup?.run).toContain('timeout 30s brev delete "$INSTANCE"'); expect(cleanup?.run).toContain("timeout 30s brev ls --json"); expect(cleanup?.run).toContain("timeout 30s brev refresh"); expect(cleanup?.run).not.toMatch(/grep.*not found/); - expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( - steps.indexOf(collect as NonNullable), - ); - expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( - steps.indexOf(uploadDebug as NonNullable), - ); - expect(steps.indexOf(cleanup as NonNullable)).toBeGreaterThan( - steps.indexOf(uploadLogs as NonNullable), - ); }); + // source-shape-contract: security -- Brev credentials must originate only from repository secrets it("keeps manual dispatch inputs out of the Brev credential boundary", () => { const validation = branchValidation.jobs?.["e2e-branch-validation"]; const install = validation?.steps?.find((step) => step.name === "Install Brev CLI"); @@ -194,6 +168,7 @@ describe("Brev nightly workflow contract", () => { expect(JSON.stringify(validation)).not.toContain("inputs.brev_token"); }); + // source-shape-contract: security -- Exact Brev archive integrity must be verified before executable extraction it("verifies the pinned Brev CLI digest before extracting it", () => { const validation = branchValidation.jobs?.["e2e-branch-validation"]; const install = validation?.steps?.find((step) => step.name === "Install Brev CLI"); @@ -210,6 +185,7 @@ describe("Brev nightly workflow contract", () => { expect(script.indexOf("tar -xzf")).toBeGreaterThan(script.indexOf("sha256sum -c -")); }); + // source-shape-contract: security -- Removed launchable inputs must not restore remote setup-script execution it("does not expose stale published-launchable controls", () => { const dispatchInputs = Object.keys(nightly.on?.workflow_dispatch?.inputs ?? {}); const reusableInputs = Object.keys(branchValidation.on?.workflow_call?.inputs ?? {}); diff --git a/test/cloudflared-update-check-workflow.test.ts b/test/cloudflared-update-check-workflow.test.ts index 99ee32e3463..fae00f17944 100644 --- a/test/cloudflared-update-check-workflow.test.ts +++ b/test/cloudflared-update-check-workflow.test.ts @@ -19,7 +19,7 @@ const FULL_SHA_ACTION = /@[0-9a-f]{40}$/iu; type CloudflaredUpdateWorkflow = { on?: { schedule?: Array<{ cron?: string }>; - workflow_dispatch?: Record; + workflow_dispatch?: unknown; }; permissions?: Record; jobs?: Record< @@ -49,7 +49,10 @@ function writePinFixture(file: string, version: string, sha256: string): void { ); } -function runFixtureCheck(options: { pinnedVersion: string; latestVersion: string }) { +function runFixtureCheck( + options: { pinnedVersion: string; latestVersion: string }, + command?: string, +) { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloudflared-update-")); const workflowPath = path.join(tempDir, "e2e.yaml"); const releasePath = path.join(tempDir, "release.json"); @@ -97,7 +100,8 @@ esac { mode: 0o755 }, ); - const result = spawnSync("bash", [CHECK_SCRIPT], { + const commandArguments = command ? ["-e", "-o", "pipefail", "-c", command] : [CHECK_SCRIPT]; + const result = spawnSync("bash", commandArguments, { cwd: ROOT, encoding: "utf8", env: { @@ -123,21 +127,26 @@ describe("cloudflared update-check workflow contract", () => { ".github/workflows/cloudflared-update-check.yaml", ); const e2e = fs.readFileSync(E2E_WORKFLOW, "utf8"); - - it("runs weekly and manually with read-only permissions and a credential-free checkout", () => { - expect(workflow.on?.schedule).toEqual([{ cron: "23 13 * * 1" }]); - expect(workflow.on?.workflow_dispatch).toEqual({}); + const configuredCheckCommand = + workflow.jobs?.["check-cloudflared"]?.steps?.find((step) => typeof step.run === "string") + ?.run ?? ""; + + // source-shape-contract: security -- Automatic and on-demand checks must preserve the credential-free dependency monitoring boundary + it("keeps automatic and on-demand update checks reachable and credential-free", () => { + expect({ + automatic: + workflow.on?.schedule?.some( + (entry) => typeof entry.cron === "string" && entry.cron.trim() !== "", + ) ?? false, + onDemand: Object.hasOwn(workflow.on ?? {}, "workflow_dispatch"), + }).toEqual({ automatic: true, onDemand: true }); expect(workflow.permissions).toEqual({ contents: "read" }); const job = workflow.jobs?.["check-cloudflared"]; const checkout = job?.steps?.find((step) => step.uses?.startsWith("actions/checkout@")); - const check = job?.steps?.find( - (step) => step.name === "Compare reviewed pin with the latest upstream release", - ); expect(job?.permissions).toBeUndefined(); expect(checkout?.uses).toMatch(FULL_SHA_ACTION); expect(checkout?.with?.["persist-credentials"]).toBe(false); - expect(check?.run).toBe("bash scripts/checks/check-cloudflared-update.sh"); }); it("extracts exactly three identical reviewed version and SHA256 pins", () => { @@ -165,7 +174,10 @@ describe("cloudflared update-check workflow contract", () => { }); it("passes only when the latest release asset matches the reviewed SHA256", () => { - const fixture = runFixtureCheck({ pinnedVersion: "2026.7.1", latestVersion: "2026.7.1" }); + const fixture = runFixtureCheck( + { pinnedVersion: "2026.7.1", latestVersion: "2026.7.1" }, + configuredCheckCommand, + ); try { expect(fixture.result.status, fixture.result.stderr).toBe(0); expect(fixture.result.stdout).toContain("cloudflared pin is current"); @@ -177,7 +189,10 @@ describe("cloudflared update-check workflow contract", () => { }); it("fails an outdated pin with the latest version, hash, and all update locations", () => { - const fixture = runFixtureCheck({ pinnedVersion: "2026.6.1", latestVersion: "2026.7.1" }); + const fixture = runFixtureCheck( + { pinnedVersion: "2026.6.1", latestVersion: "2026.7.1" }, + configuredCheckCommand, + ); try { expect(fixture.result.status).toBe(1); expect(fixture.result.stderr).toContain("cloudflared update required"); diff --git a/test/code-scanning-workflow.test.ts b/test/code-scanning-workflow.test.ts index fde110256de..9b603ee2dc5 100644 --- a/test/code-scanning-workflow.test.ts +++ b/test/code-scanning-workflow.test.ts @@ -16,6 +16,7 @@ const dependabot = readYaml<{ updates?: DependabotUpdate[] }>(".github/dependabo const codeqlActionPrefix = "github/codeql-action/"; describe("Code scanning workflow dependency updates", () => { + // source-shape-contract: security -- One immutable CodeQL revision prevents partial scanner action upgrades it("keeps every CodeQL action on one immutable revision", () => { const codeqlActions = Object.values(workflow.jobs ?? {}) .flatMap((job) => job.steps ?? []) @@ -34,6 +35,7 @@ describe("Code scanning workflow dependency updates", () => { expect(new Set(revisions).size).toBe(1); }); + // source-shape-contract: security -- Grouped CodeQL updates preserve the reviewed single-revision scanner boundary it("groups CodeQL action updates so Dependabot keeps the shared revision synchronized", () => { const githubActionsUpdate = dependabot.updates?.find( (update) => update["package-ecosystem"] === "github-actions" && update.directory === "/", diff --git a/test/dcode-base-image-workflow.test.ts b/test/dcode-base-image-workflow.test.ts index 2b68f094bcb..94a1938e431 100644 --- a/test/dcode-base-image-workflow.test.ts +++ b/test/dcode-base-image-workflow.test.ts @@ -128,6 +128,7 @@ function pinnedAptVersion(dockerfile: string, packageName: string): string { } describe("base-image publication behavior", () => { + // source-shape-contract: security -- Publisher mutations must preserve immutable actions and guarded production build arguments it("accepts every discovered publisher and rejects supply-chain mutations", () => { const publishers = publisherJobs(workflow); expect(publishers.length).toBeGreaterThan(0); diff --git a/test/dcode-start-keepalive.test.ts b/test/dcode-start-keepalive.test.ts index 5ea84270af9..73d2c92c50e 100644 --- a/test/dcode-start-keepalive.test.ts +++ b/test/dcode-start-keepalive.test.ts @@ -98,6 +98,7 @@ afterEach(() => { }); describe("Deep Agents Code sandbox entrypoint keep-alive (#5717)", () => { + // source-shape-contract: compatibility -- Executes the shipped entrypoint to protect no-command sandbox liveness it("stays alive as a long-running process when invoked with no command", () => { // The terminal-runtime sandbox runs this entrypoint with no args as its // sole foreground process. It must NOT exit on its own — a self-exiting @@ -125,6 +126,7 @@ describe("Deep Agents Code sandbox entrypoint keep-alive (#5717)", () => { expect(fs.readFileSync(rlimitMarker, "utf8")).toBe("hardened\nverified\n"); }); + // source-shape-contract: compatibility -- Executes the shipped entrypoint to protect explicit command delegation it("execs an explicitly supplied command instead of idling", () => { const { scriptPath, rlimitMarker } = makeStartFixture(); const result = spawnSync(scriptPath, ["printf", "RAN_CMD"], { @@ -137,6 +139,7 @@ describe("Deep Agents Code sandbox entrypoint keep-alive (#5717)", () => { expect(fs.readFileSync(rlimitMarker, "utf8")).toBe("hardened\nverified\n"); }); + // source-shape-contract: security -- Executes the shipped entrypoint to prove missing rlimit enforcement fails closed it("refuses to launch when the required rlimit helper is missing (#6545)", () => { const { scriptPath, rlimitMarker } = makeStartFixture({ installRlimitHelper: () => undefined, @@ -155,6 +158,7 @@ describe("Deep Agents Code sandbox entrypoint keep-alive (#5717)", () => { expect(fs.existsSync(rlimitMarker)).toBe(false); }); + // source-shape-contract: security -- Executes the shipped entrypoint to prove ineffective rlimits fail closed it("refuses to launch when effective rlimits fail verification (#6545)", () => { const { scriptPath, rlimitMarker } = makeStartFixture({ installRlimitHelper: installFailingVerificationRlimitHelper, diff --git a/test/e2e-advisor.test.ts b/test/e2e-advisor.test.ts index 377dafaccbe..ead7987f08d 100644 --- a/test/e2e-advisor.test.ts +++ b/test/e2e-advisor.test.ts @@ -303,6 +303,7 @@ describe("E2E recommendation advisor prompt", () => { } }); + // source-shape-contract: security -- Symlink removal must precede every credential-bearing advisor step it("strips untrusted symlinks before secret-bearing advisor steps", () => { const steps = readAdvisorWorkflow().jobs?.advise?.steps ?? []; const removeSymlinksIndex = steps.findIndex( diff --git a/test/e2e-fixture-dependency-review.test.ts b/test/e2e-fixture-dependency-review.test.ts index 4a203520b9d..2ee548fe89e 100644 --- a/test/e2e-fixture-dependency-review.test.ts +++ b/test/e2e-fixture-dependency-review.test.ts @@ -51,6 +51,7 @@ describe("E2E fixture dependency review", () => { } }); + // source-shape-contract: security -- Exact fixture pins and reviewed lock integrity constrain untrusted dependency code it("keeps installed fixture dependencies on exact versions", () => { const weatherFixture = path.join(FIXTURES_ROOT, "plugins", "weather"); const manifest = JSON.parse( diff --git a/test/e2e-release-gate-workflow.test.ts b/test/e2e-release-gate-workflow.test.ts index 3175a78b83d..9ed105636ae 100644 --- a/test/e2e-release-gate-workflow.test.ts +++ b/test/e2e-release-gate-workflow.test.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { readFileSync } from "node:fs"; - import { describe, expect, it } from "vitest"; import { readYaml, type WorkflowJob } from "./helpers/e2e-workflow-contract"; @@ -19,24 +17,7 @@ type E2eWorkflow = { const e2eWorkflow = readYaml(".github/workflows/e2e.yaml"); describe("release gate workflow resource contracts", () => { - it("starts hosted agent proofs in the first wave after matrix generation", () => { - const fullJob = e2eWorkflow.jobs["full-e2e"]; - const tuiJob = e2eWorkflow.jobs["openclaw-tui-chat-correlation"]; - - expect(fullJob.needs).toBe("generate-matrix"); - expect(fullJob.if).not.toContain("always()"); - expect(fullJob.if).toContain(",full-e2e,"); - const fullE2ERun = fullJob.steps?.find( - (step) => step.name === "Run full-e2e live Vitest test", - )?.run; - expect(fullE2ERun).toMatch(/npx vitest run --project e2e-live[\s\S]*full-e2e\.test\.ts/u); - expect(fullE2ERun).not.toContain("onboard-progress-budget.test.ts"); - expect(fullE2ERun?.match(/npx vitest run --project e2e-live/gu)).toHaveLength(1); - expect(tuiJob.needs).toBe("generate-matrix"); - expect(tuiJob.if).not.toContain("always()"); - expect(tuiJob.if).toContain(",openclaw-tui-chat-correlation,"); - }); - + // source-shape-contract: security -- Trusted checkout selection binds TUI evidence to the validated controller commit it("replaces legacy target_ref dispatches with the validated checkout contract", () => { const inputs = e2eWorkflow.on?.workflow_dispatch?.inputs; const tuiJob = e2eWorkflow.jobs["openclaw-tui-chat-correlation"]; @@ -50,40 +31,4 @@ describe("release gate workflow resource contracts", () => { "${{ inputs.checkout_sha || github.sha }}", ); }); - - it("budgets cold Ollama pulls in the consolidated GPU lane", () => { - const gpuJob = e2eWorkflow.jobs["gpu-e2e"]; - const liveTest = readFileSync(new URL("./e2e/live/gpu-e2e.test.ts", import.meta.url), "utf8"); - - expect(gpuJob["timeout-minutes"]).toBe(90); - expect(gpuJob.env?.NEMOCLAW_OLLAMA_PULL_TIMEOUT).toBe("2400"); - expect(liveTest).toContain("timeoutMs: 55 * 60_000"); - }); - - it("authenticates Spark image pulls through the shared guarded steps", () => { - const steps = e2eWorkflow.jobs["spark-install"].steps ?? []; - const stepIndex = (name: string) => steps.findIndex((step) => step.name === name); - const auth = steps.find((step) => step.name === "Authenticate to Docker Hub"); - const cleanup = steps.find((step) => step.name === "Clean up Docker auth"); - - expect(steps.some((step) => step.name === "Configure isolated Docker auth directory")).toBe( - false, - ); - expect(auth?.uses).toBeUndefined(); - expect(auth?.if).toBeUndefined(); - expect(auth?.["continue-on-error"]).toBeUndefined(); - expect(auth?.env).toHaveProperty("DOCKERHUB_AUTH_REQUIRED"); - expect(auth?.run).toEqual(expect.any(String)); - expect(stepIndex("Authenticate to Docker Hub")).toBeLessThan( - stepIndex("Prepare E2E workspace"), - ); - expect(stepIndex("Authenticate to Docker Hub")).toBeLessThan( - stepIndex("Run Spark install live test"), - ); - expect(cleanup?.if).toBe("always()"); - expect(cleanup?.run).toBe("bash .github/scripts/docker-auth-cleanup.sh"); - expect(stepIndex("Run Spark install live test")).toBeLessThan( - stepIndex("Clean up Docker auth"), - ); - }); }); diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index ff9e662f674..a24e56b9379 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -237,6 +237,7 @@ async function retryHostedInference( ); } +// source-shape-contract: security -- Live execution proves the shipped Hermes manifest remains healthy and credential-safe test("hermes-e2e: install.sh onboards Hermes and proves health plus live inference", { timeout: LIVE_TIMEOUT_MS, }, async ({ artifacts, cleanup, host, provider, sandbox, secrets }) => { diff --git a/test/e2e/support/e2e-expected-state.test.ts b/test/e2e/support/e2e-expected-state.test.ts index 21abdad404d..edd95b0361f 100644 --- a/test/e2e/support/e2e-expected-state.test.ts +++ b/test/e2e/support/e2e-expected-state.test.ts @@ -16,6 +16,7 @@ import { listTargets } from "../registry/registry.ts"; import type { ExpectedState, StateProbeId } from "../registry/types.ts"; describe("typed expected-state registry behavior", () => { + // source-shape-contract: compatibility -- Registry indexing keeps every shipped expected-state selector resolvable it("indexes every registered state by its unique id", () => { const states = listExpectedStates(); const ids = states.map((state) => state.id); @@ -28,6 +29,7 @@ describe("typed expected-state registry behavior", () => { } }); + // source-shape-contract: compatibility -- Unknown expected-state selectors must retain actionable failure diagnostics it("rejects an unknown state with an actionable inventory", () => { const unknown = "synthetic-unknown-state"; @@ -117,6 +119,7 @@ describe("expected-state probe compilation", () => { }); describe("target expected-state references", () => { + // source-shape-contract: compatibility -- Every shipped target must resolve its expected-state runtime contract it("resolves every state id consumed by the typed target registry", () => { const referenced = listTargets() .map((target) => target.expectedStateId) @@ -128,6 +131,7 @@ describe("target expected-state references", () => { } }); + // source-shape-contract: security -- Fail-closed targets must compile probes that forbid gateway and sandbox side effects it("compiles fail-closed absence probes for targets that forbid runtime side effects", () => { const failClosedTargets = listTargets().filter((target) => { const forbidden = target.expectedFailure?.forbiddenSideEffects ?? []; @@ -143,6 +147,7 @@ describe("target expected-state references", () => { } }); + // source-shape-contract: security -- Every preflight failure must compile absence probes before privileged runtime creation it("compiles absence probes for every preflight failure contract", () => { const preflightFailures = listTargets().filter( (target) => target.expectedFailure?.phase === "preflight", @@ -157,6 +162,7 @@ describe("target expected-state references", () => { } }); + // source-shape-contract: security -- Policy-selection failures must stop before gateway or sandbox side effects it("keeps policy-selection failures limited to the installed CLI", () => { const policySelectionFailures = listTargets().filter( (target) => target.expectedFailure?.errorClass === "policy-presets-required", @@ -170,6 +176,7 @@ describe("target expected-state references", () => { } }); + // source-shape-contract: compatibility -- Terminal agent targets must not require an unsupported host gateway probe it("omits host gateway probes for targets whose loaded agent runtime is terminal", () => { const targetAgents = listTargets() .filter((target) => target.manifestPath !== undefined) @@ -193,6 +200,7 @@ describe("target expected-state references", () => { } }); + // source-shape-contract: security -- Post-reboot targets must retain host registry and container preservation probes it("compiles host-preservation probes for every post-reboot recovery target", () => { const recoveryTargets = listTargets().filter( (target) => target.environment?.lifecycle === "post-reboot-recovery", diff --git a/test/e2e/support/e2e-live-project-config.test.ts b/test/e2e/support/e2e-live-project-config.test.ts index 5f2587abb8e..0708ef6628c 100644 --- a/test/e2e/support/e2e-live-project-config.test.ts +++ b/test/e2e/support/e2e-live-project-config.test.ts @@ -85,6 +85,7 @@ describe("gated E2E Vitest projects", () => { expect(deterministicProjects.every((project) => project.test?.retry === undefined)).toBe(true); }); + // source-shape-contract: security -- Live projects must not inherit the deterministic drift-preflight bypass it("keeps the drift-preflight bypass out of live projects (#6692)", () => { const statefulProjects = projectConfigs().filter( (project) => !project.test?.setupFiles?.includes(FIXTURE_UMASK_SETUP), diff --git a/test/e2e/support/e2e-live-registry-discovery.test.ts b/test/e2e/support/e2e-live-registry-discovery.test.ts index 53182ed1e52..e2a17f9d3cc 100644 --- a/test/e2e/support/e2e-live-registry-discovery.test.ts +++ b/test/e2e/support/e2e-live-registry-discovery.test.ts @@ -26,6 +26,7 @@ function syntheticTarget(environment: TargetEnvironment = SUPPORTED_ENVIRONMENT) } describe("live target registry discovery support", () => { + // source-shape-contract: compatibility -- Every shipped target must classify as runnable or expose a concrete skip reason it("classifies every shipped target as supported or with a concrete reason", () => { const targets = listTargets(); diff --git a/test/e2e/support/e2e-live-skip-name-contract.test.ts b/test/e2e/support/e2e-live-skip-name-contract.test.ts index 721e05cd224..85cda2a0015 100644 --- a/test/e2e/support/e2e-live-skip-name-contract.test.ts +++ b/test/e2e/support/e2e-live-skip-name-contract.test.ts @@ -29,6 +29,7 @@ function syntheticTarget(platform: string): TargetDefinition { * tests and Vitest would exit non-zero with no structured skip reason. */ describe("live registry-targets skip-name contract", () => { + // source-shape-contract: compatibility -- Live test registration names must remain addressable by stable target identifiers it("registers every target under a name equal to its id", () => { const targets = listTargets(); expect(targets.length).toBeGreaterThan(0); @@ -37,6 +38,7 @@ describe("live registry-targets skip-name contract", () => { } }); + // source-shape-contract: compatibility -- Workflow target filters must select every registered live test by exact identifier it('matches the workflow\'s exact `-t "^${TARGET_ID}$"` regex for every target', () => { for (const target of listTargets()) { const name = liveTargetTestName(target); diff --git a/test/e2e/support/e2e-manifests.test.ts b/test/e2e/support/e2e-manifests.test.ts index 05bc2407fcf..387e1ded150 100644 --- a/test/e2e/support/e2e-manifests.test.ts +++ b/test/e2e/support/e2e-manifests.test.ts @@ -53,6 +53,7 @@ describe("NemoClawInstance manifests", () => { ); }); + // source-shape-contract: compatibility -- Every shipped target path must load through the validated E2E manifest consumer it("resolves every typed target manifest path to a validated manifest", () => { const manifestPaths = new Set( loadManifestsFromDir(MANIFEST_DIR).map((manifest) => path.resolve(manifest.filePath)), diff --git a/test/e2e/support/e2e-matrix.test.ts b/test/e2e/support/e2e-matrix.test.ts index 9543db98ba6..e22da178ca5 100644 --- a/test/e2e/support/e2e-matrix.test.ts +++ b/test/e2e/support/e2e-matrix.test.ts @@ -83,6 +83,7 @@ describe("live E2E target matrix", () => { expect(() => resolveRunnerForTarget(broken)).toThrow(/no default for platform/); }); + // source-shape-contract: compatibility -- Default live matrix output must cover every fixture-supported registered target once it("builds the default live matrix from every fixture-supported target", () => { const targets = listTargets(); const supportedTargets = targets.filter((entry) => liveTargetSupport(entry).supported); diff --git a/test/e2e/support/e2e-registry.test.ts b/test/e2e/support/e2e-registry.test.ts index a2c15486e42..9e93a77a3f4 100644 --- a/test/e2e/support/e2e-registry.test.ts +++ b/test/e2e/support/e2e-registry.test.ts @@ -42,6 +42,7 @@ describe("deterministic target registry", () => { ); }); + // source-shape-contract: compatibility -- The target CLI must reject unknown selectors with actionable registered choices it("should return actionable unknown target error", () => { const result = runTargetCli(["--emit-live-matrix", "--targets", "does-not-exist"]); const output = `${result.stdout}${result.stderr}`; @@ -54,6 +55,7 @@ describe("deterministic target registry", () => { } }); + // source-shape-contract: compatibility -- The target CLI must preserve requested ordering for multiple live selectors it("CLI should emit multiple selected live matrix entries", () => { const selectedIds = listTargets() .slice(0, 2) diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index c4cdcce1290..2f4807bcca6 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -14,6 +14,7 @@ import { } from "../../../tools/e2e/credential-free-tests.mts"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; import { buildE2eWorkflowPlan } from "../../../tools/e2e/workflow-plan.mts"; +import { requireFixture } from "./require-fixture"; function readWorkflow(): Record { return YAML.parse( @@ -212,7 +213,7 @@ it("rejects report-to-pr PR number validation drift", () => { const reportStep = workflow.jobs["report-to-pr"].steps.find( (step) => step.name === "Post E2E target results to PR", ); - expect(reportStep?.with?.script).toEqual(expect.any(String)); + requireFixture(typeof reportStep?.with?.script === "string", "missing report-to-pr script"); reportStep!.with!.script = String(reportStep!.with!.script) .replace(/\/\^\[1-9\]\[0-9\]\*\$\/\.test\(prNumberInput\)/, "prNumberInput.length > 0") .replace("Number(prNumberInput)", "Number.parseInt(prNumberInput, 10)") diff --git a/test/e2e/support/e2e-workflow-trace.test.ts b/test/e2e/support/e2e-workflow-trace.test.ts index 66f30e81b33..30a71df98f8 100644 --- a/test/e2e/support/e2e-workflow-trace.test.ts +++ b/test/e2e/support/e2e-workflow-trace.test.ts @@ -35,18 +35,16 @@ function liveStep(workflow: E2eWorkflow, name: string): Record } describe("e2e workflow live trace boundary", () => { - it("rejects missing live trace boundary steps", () => { - for (const name of [ - "Configure live E2E trace directory", - "Build trusted live E2E timing summary", - "Delete raw live E2E traces", - ]) { - const errors = validateMutatedWorkflow((workflow) => { - workflow.jobs.live.steps = workflow.jobs.live.steps.filter((step) => step.name !== name); - }); - - expect(errors).toContain(`run-target job missing step: ${name}`); - } + it.each([ + "Configure live E2E trace directory", + "Build trusted live E2E timing summary", + "Delete raw live E2E traces", + ])("rejects a missing live trace boundary step: %s", (name) => { + const errors = validateMutatedWorkflow((workflow) => { + workflow.jobs.live.steps = workflow.jobs.live.steps.filter((step) => step.name !== name); + }); + + expect(errors).toContain(`run-target job missing step: ${name}`); }); it("rejects live sanitizer and cleanup steps without always guards", () => { diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 7e69b148da2..7d385b40adf 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -78,6 +78,7 @@ describe("e2e workflow boundary", () => { expect(validateE2eWorkflowBoundary()).toEqual([]); }); + // source-shape-contract: security -- Mutates the shipped workflow to prove PR-safe routing rejects credential-backed smokes it("rejects credential-backed provider smokes in the PR-safe inference-routing job", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-inference-routing-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); @@ -136,6 +137,7 @@ describe("e2e workflow boundary", () => { } }); + // source-shape-contract: security -- Mutates the shipped workflow to prove artifact uploads reject unmanaged temporary paths it("rejects free-standing E2E artifact uploads from raw temp paths", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); @@ -213,6 +215,7 @@ describe("e2e workflow boundary", () => { }, ); + // source-shape-contract: compatibility -- Cross-checks generated selectors against the executable workflow job registry it("derives test selectors from code and workflow jobs from workflow metadata", { timeout: 60_000, }, () => { @@ -778,6 +781,7 @@ jobs: } }); + // source-shape-contract: security -- Mutates the shipped workflow to prove channel lifecycle secrets and artifacts fail closed it("rejects channels stop/start workflow-boundary drift for secret and artifact handling", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); @@ -891,6 +895,7 @@ jobs: } }); + // source-shape-contract: security -- Mutates the shipped workflow to reject duplicate unguarded Docker credential exposure it("rejects duplicate unguarded Docker Hub auth in messaging-compatible-endpoint", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); @@ -926,6 +931,7 @@ jobs: } }); + // source-shape-contract: security -- Mutates the shipped diagnostics job to reject secret and Docker auth leakage it("rejects diagnostics workflow-boundary drift for secret and Docker auth handling", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/test/e2e/support/mcp-workflow-boundary.test.ts b/test/e2e/support/mcp-workflow-boundary.test.ts index 51eabd2671b..c753e51a6de 100644 --- a/test/e2e/support/mcp-workflow-boundary.test.ts +++ b/test/e2e/support/mcp-workflow-boundary.test.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -10,6 +9,7 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; import { validateMcpOpenShellWorkflowBoundary } from "../../../tools/e2e/mcp-workflow-boundary.mts"; +import { requireFixture } from "./require-fixture"; describe("MCP workflow artifact boundary", () => { it("rejects upload action or path drift from the reviewed shared boundary", () => { @@ -25,7 +25,7 @@ describe("MCP workflow artifact boundary", () => { const upload = workflow.jobs["mcp-bridge"].steps.find( (step) => step.name === "Upload MCP server artifacts", ); - assert(upload?.with, "MCP artifact upload fixture is missing"); + requireFixture(upload?.with, "MCP artifact upload fixture is missing"); upload.uses = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@main"; upload.with.path = "e2e-artifacts/live/unscanned/"; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); @@ -60,7 +60,7 @@ describe("MCP workflow artifact boundary", () => { const cloudflared = workflow.jobs["mcp-bridge-dev"].steps.find( (step) => step.name === "Install and verify cloudflared prerequisite", ); - assert(cloudflared?.env, "MCP cloudflared installer fixture is missing"); + requireFixture(cloudflared?.env, "MCP cloudflared installer fixture is missing"); cloudflared.env.CLOUDFLARED_DEB_SHA256 = "mutable"; cloudflared.run = "sudo apt-get install -y cloudflared"; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); diff --git a/test/e2e/support/mcp-workflow-compatibility.test.ts b/test/e2e/support/mcp-workflow-compatibility.test.ts index 288065e222c..63edffcd45d 100644 --- a/test/e2e/support/mcp-workflow-compatibility.test.ts +++ b/test/e2e/support/mcp-workflow-compatibility.test.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -10,6 +9,7 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; import { validateMcpOpenShellWorkflowBoundary } from "../../../tools/e2e/mcp-workflow-boundary.mts"; +import { requireFixture } from "./require-fixture"; describe("MCP workflow runtime compatibility", () => { it("accepts harmless classifier key reordering (#6426)", () => { @@ -23,7 +23,7 @@ describe("MCP workflow runtime compatibility", () => { const classifierIndex = steps.findIndex( (step) => step.name === "Classify OpenShell credential-boundary compatibility", ); - assert(classifierIndex >= 0, "MCP dev classifier fixture is missing"); + requireFixture(classifierIndex >= 0, "MCP dev classifier fixture is missing"); const classifier = steps[classifierIndex]!; steps[classifierIndex] = { run: classifier.run, @@ -56,7 +56,7 @@ describe("MCP workflow runtime compatibility", () => { const lifecycle = workflow.jobs["mcp-bridge-dev"].steps.find( (step) => step.name === "Run MCP OpenShell provider live test", ); - assert(lifecycle, "MCP dev lifecycle fixture is missing"); + requireFixture(lifecycle, "MCP dev lifecycle fixture is missing"); delete lifecycle.if; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); @@ -87,8 +87,8 @@ describe("MCP workflow runtime compatibility", () => { const lifecycleIndex = steps.findIndex( (step) => step.name === "Run MCP OpenShell provider live test", ); - assert(classifierIndex >= 0, "MCP dev classifier fixture is missing"); - assert(lifecycleIndex >= 0, "MCP dev lifecycle fixture is missing"); + requireFixture(classifierIndex >= 0, "MCP dev classifier fixture is missing"); + requireFixture(lifecycleIndex >= 0, "MCP dev lifecycle fixture is missing"); const classifier = steps[classifierIndex]!; classifier.id = "uncanonical_classifier"; classifier.run = "npx tsx tools/e2e/unreviewed-classifier.mts"; @@ -130,7 +130,7 @@ describe("MCP workflow runtime compatibility", () => { const classifier = workflow.jobs["mcp-bridge-dev"].steps.find( (step) => step.name === "Classify OpenShell credential-boundary compatibility", ); - assert(classifier?.run, "MCP dev classifier fixture is missing"); + requireFixture(classifier?.run, "MCP dev classifier fixture is missing"); classifier.if = "false"; classifier["continue-on-error"] = true; classifier.env = { E2E_ARTIFACT_DIR: "/tmp/unreviewed" }; @@ -158,7 +158,7 @@ describe("MCP workflow runtime compatibility", () => { const lifecycle = workflow.jobs["mcp-bridge"].steps.find( (step) => step.name === "Run MCP OpenShell provider live test", ); - assert(lifecycle, "MCP stable lifecycle fixture is missing"); + requireFixture(lifecycle, "MCP stable lifecycle fixture is missing"); lifecycle.if = "${{ steps.mcp_runtime_compatibility.outputs.mode == 'full-lifecycle' }}"; workflow.jobs["mcp-bridge"].steps.splice( workflow.jobs["mcp-bridge"].steps.indexOf(lifecycle), diff --git a/test/e2e/support/require-fixture.ts b/test/e2e/support/require-fixture.ts new file mode 100644 index 00000000000..9fe952daad9 --- /dev/null +++ b/test/e2e/support/require-fixture.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function requireFixture(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} diff --git a/test/e2e/support/spark-install-workflow-boundary.test.ts b/test/e2e/support/spark-install-workflow-boundary.test.ts index 2e990e2482e..8ffe6df3e02 100644 --- a/test/e2e/support/spark-install-workflow-boundary.test.ts +++ b/test/e2e/support/spark-install-workflow-boundary.test.ts @@ -9,7 +9,7 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; -import { assertSparkInstallSandboxName } from "../live/spark-install-helpers.ts"; +import { requireFixture } from "./require-fixture"; function readWorkflow(): Record { return YAML.parse( @@ -18,15 +18,6 @@ function readWorkflow(): Record { } describe("spark install workflow boundary", () => { - it("keeps the configured sandbox inside the live cleanup ownership boundary", () => { - const workflow = readWorkflow() as { - jobs: Record }>; - }; - const sandboxName = workflow.jobs["spark-install"]?.env?.NEMOCLAW_SANDBOX_NAME; - - expect(assertSparkInstallSandboxName(String(sandboxName))).toBe(sandboxName); - }); - it("rejects Spark install trusted-boundary drift", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); @@ -37,7 +28,7 @@ describe("spark install workflow boundary", () => { >; }; const job = workflow.jobs["spark-install"]; - expect(job).toBeDefined(); + requireFixture(job, "missing spark-install job"); job["runs-on" as keyof typeof job] = "self-hosted" as never; job["timeout-minutes" as keyof typeof job] = 30 as never; job.env = { @@ -57,7 +48,7 @@ describe("spark install workflow boundary", () => { const checkout = job.steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@"), ); - expect(checkout).toBeDefined(); + requireFixture(checkout, "missing spark-install checkout"); checkout!.uses = "actions/checkout@v6"; checkout!.with = { ...(checkout!.with as Record), @@ -65,12 +56,12 @@ describe("spark install workflow boundary", () => { }; const runSpark = job.steps.find((step) => step.name === "Run Spark install live test"); - expect(runSpark).toBeDefined(); + requireFixture(runSpark, "missing Spark install live test step"); runSpark!.env = {}; runSpark!.run = "npx vitest run --project e2e-live test/e2e/live/other.test.ts"; const upload = job.steps.find((step) => step.name === "Upload Spark install artifacts"); - expect(upload).toBeDefined(); + requireFixture(upload, "missing Spark install artifact upload"); upload!.with = { ...(upload!.with as Record), name: "spark-install-artifacts", diff --git a/test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts b/test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts index 1eb45861eda..c67b4cb6948 100644 --- a/test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts +++ b/test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; +import { requireFixture } from "./require-fixture"; function readWorkflow(): Record { return YAML.parse( @@ -24,7 +25,7 @@ describe("tunnel lifecycle workflow boundary", () => { jobs: Record }>; }; const job = workflow.jobs["tunnel-lifecycle"]; - expect(job).toBeDefined(); + requireFixture(job, "missing tunnel-lifecycle job"); job.env = { ...job.env }; delete job.env.NEMOCLAW_CLI_BIN; fs.writeFileSync(workflowPath, YAML.stringify(workflow)); @@ -51,7 +52,7 @@ describe("tunnel lifecycle workflow boundary", () => { >; }; const job = workflow.jobs["tunnel-lifecycle"]; - expect(job).toBeDefined(); + requireFixture(job, "missing tunnel-lifecycle job"); job["runs-on"] = "self-hosted"; job["timeout-minutes"] = 30; job.env = { @@ -63,7 +64,7 @@ describe("tunnel lifecycle workflow boundary", () => { const checkout = job.steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@"), ); - expect(checkout).toBeDefined(); + requireFixture(checkout, "missing tunnel-lifecycle checkout"); checkout!.uses = "actions/checkout@v6"; checkout!.with = { ...(checkout!.with as Record), @@ -73,7 +74,7 @@ describe("tunnel lifecycle workflow boundary", () => { const cloudflared = job.steps.find( (step) => step.name === "Install and verify cloudflared prerequisite", ); - expect(cloudflared).toBeDefined(); + requireFixture(cloudflared, "missing cloudflared prerequisite step"); cloudflared!.env = { NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", @@ -81,11 +82,11 @@ describe("tunnel lifecycle workflow boundary", () => { cloudflared!.run = "cloudflared --version"; const runTunnel = job.steps.find((step) => step.name === "Run tunnel lifecycle live test"); - expect(runTunnel).toBeDefined(); + requireFixture(runTunnel, "missing tunnel lifecycle live test step"); runTunnel!.run = `${String(runTunnel!.run ?? "")}\nsudo apt-get install -y cloudflared`; const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts"); - expect(upload).toBeDefined(); + requireFixture(upload, "missing tunnel lifecycle artifact upload"); upload!.with = { ...(upload!.with as Record), path: "e2e-artifacts/live/", @@ -132,11 +133,11 @@ describe("tunnel lifecycle workflow boundary", () => { jobs: Record> }>; }; const job = workflow.jobs["tunnel-lifecycle"]; - expect(job).toBeDefined(); + requireFixture(job, "missing tunnel-lifecycle job"); const cloudflared = job.steps.find( (step) => step.name === "Install and verify cloudflared prerequisite", ); - expect(cloudflared).toBeDefined(); + requireFixture(cloudflared, "missing cloudflared prerequisite step"); cloudflared!.env = { CLOUDFLARED_VERSION: "2026.6.1" }; cloudflared!.run = [ "set -euo pipefail", diff --git a/test/fetch-guard-patch-regression.test.ts b/test/fetch-guard-patch-regression.test.ts index 083c6e1350a..1c8c40d995e 100644 --- a/test/fetch-guard-patch-regression.test.ts +++ b/test/fetch-guard-patch-regression.test.ts @@ -454,6 +454,7 @@ describe("fetch-guard patch regression guard", () => { ).toContain("rm -rf /usr/local/lib/node_modules/mcporter /usr/local/bin/mcporter"); }); + // source-shape-contract: security -- Cross-file OpenClaw pins bind reviewed classifiers to verified package integrity it("requires classifier review and integrity evidence when the OpenClaw build pin changes", () => { const reviewMessage = "Update fetch-guard classifier expectations before changing the OpenClaw build version."; diff --git a/test/hermes-mcp-config-transaction.test.ts b/test/hermes-mcp-config-transaction.test.ts index f4972996543..34f76081bdb 100644 --- a/test/hermes-mcp-config-transaction.test.ts +++ b/test/hermes-mcp-config-transaction.test.ts @@ -13,7 +13,6 @@ import { normalizeMcpServerUrl, validateMcpCredentialEnvName, } from "../src/lib/actions/sandbox/mcp-bridge-validation"; -import credentialBoundaryManifest from "../src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.72.json"; const TRANSACTION = path.resolve( import.meta.dirname, @@ -187,12 +186,11 @@ print(json.dumps({"ok": True})) }); it("shares the host credential-name boundary while preserving exact cleanup", () => { - const blockedNames = [ - ...credentialBoundaryManifest.rawChildValueKeys, - ...credentialBoundaryManifest.rewrittenChildValueKeys, - ...credentialBoundaryManifest.runtimeControlKeys, - ...credentialBoundaryManifest.runtimeControlPrefixes.map((prefix) => `${prefix}MCP_TOKEN`), - ]; + // One representative from each production category: OpenShell raw child + // value, OpenShell rewritten child value, exact process control, and + // process-control prefix. The exhaustive manifest-driven matrix lives at + // the shared TypeScript validator boundary. + const blockedNames = ["GCP_PROJECT_ID", "GCE_METADATA_HOST", "PATH", "NEMOCLAW_MCP_TOKEN"]; const result = runPython( ` import importlib.util, json, sys @@ -225,23 +223,19 @@ for name in blocked: cleanup_accepted.append(name) module._validate_payload("add", payload("MY_SERVICE_MCP_TOKEN", "add")) print(json.dumps({ - "addRejected": add_rejected, - "cleanupAccepted": cleanup_accepted, + "allBlocked": add_rejected == blocked, + "cleanupPreserved": cleanup_accepted == blocked, "safeAccepted": True, })) `, [JSON.stringify(blockedNames)], ); - expect(credentialBoundaryManifest.openshellVersion).toBe("0.0.72"); - for (const name of blockedNames) { - expect(() => validateMcpCredentialEnvName(name)).toThrow(); - } expect(() => validateMcpCredentialEnvName("MY_SERVICE_MCP_TOKEN")).not.toThrow(); expect(result.status, result.stderr).toBe(0); expect(JSON.parse(result.stdout)).toEqual({ - addRejected: blockedNames, - cleanupAccepted: blockedNames, + allBlocked: true, + cleanupPreserved: true, safeAccepted: true, }); }); diff --git a/test/hermes-sandbox-workflow.test.ts b/test/hermes-sandbox-workflow.test.ts deleted file mode 100644 index 54c1bc00d0d..00000000000 --- a/test/hermes-sandbox-workflow.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { readYaml, type WorkflowJob, type WorkflowStep } from "./helpers/e2e-workflow-contract"; - -const workflow = readYaml<{ jobs: Record }>( - ".github/workflows/sandbox-images-and-e2e.yaml", -); - -function requireStep(steps: WorkflowStep[], name: string): { index: number; step: WorkflowStep } { - const index = steps.findIndex((step) => step.name === name); - expect(index, `missing workflow step: ${name}`).toBeGreaterThanOrEqual(0); - return { index, step: steps[index] }; -} - -describe("Hermes sandbox image workflow", () => { - it("installs pinned root dependencies before either Vitest invocation", () => { - const steps = workflow.jobs["build-hermes-sandbox-image"].steps ?? []; - const setup = requireStep(steps, "Set up Node"); - const install = requireStep(steps, "Install root dependencies"); - const secretBoundary = requireStep(steps, "Run Hermes sandbox secret boundary test"); - const rootEntrypoint = requireStep(steps, "Run Hermes root entrypoint smoke Vitest test"); - - expect(setup.step.uses).toBe("actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e"); - expect(install.step.run).toBe("npm ci --ignore-scripts"); - expect(setup.index).toBeLessThan(install.index); - expect(install.index).toBeLessThan(secretBoundary.index); - expect(install.index).toBeLessThan(rootEntrypoint.index); - }); - - it("builds Hermes once, reuses that image for both probes, and cleans up last", () => { - const steps = workflow.jobs["build-hermes-sandbox-image"].steps ?? []; - const build = requireStep(steps, "Build Hermes production image"); - const secretBoundary = requireStep(steps, "Run Hermes sandbox secret boundary test"); - const secretArtifacts = requireStep(steps, "Upload Hermes sandbox secret boundary artifacts"); - const rootEntrypoint = requireStep(steps, "Run Hermes root entrypoint smoke Vitest test"); - const rootArtifacts = requireStep(steps, "Upload Hermes root entrypoint smoke artifacts"); - const cleanup = requireStep(steps, "Clean up Docker auth"); - const buildCommand = 'docker build "${build_args[@]}" -t nemoclaw-hermes-production .'; - - expect(steps.filter((step) => step.run?.includes(buildCommand))).toHaveLength(1); - expect(build.step.run).toContain("build_args=(-f agents/hermes/Dockerfile"); - expect(build.step.run).toContain('scripts/check-production-build-args.sh "${build_args[@]}"'); - expect(build.step.run).toContain(buildCommand); - expect(secretBoundary.step.env?.NEMOCLAW_HERMES_TEST_IMAGE).toBe("nemoclaw-hermes-production"); - expect(rootEntrypoint.step.env?.NEMOCLAW_HERMES_TEST_IMAGE).toBe("nemoclaw-hermes-production"); - expect(build.index).toBeLessThan(secretBoundary.index); - expect(secretBoundary.index).toBeLessThan(secretArtifacts.index); - expect(secretArtifacts.index).toBeLessThan(rootEntrypoint.index); - expect(rootEntrypoint.index).toBeLessThan(rootArtifacts.index); - expect(rootArtifacts.index).toBeLessThan(cleanup.index); - expect(cleanup.index).toBe(steps.length - 1); - expect(cleanup.step.if).toBe("always()"); - }); -}); diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 588fce76b82..09e81f4c1ec 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -162,6 +162,7 @@ function stripFencedCodeBlocks(markdown: string): string { } describe("inference options model task-fit docs (#4755)", () => { + // source-shape-contract: compatibility -- Published task-fit guidance must cover every curated onboarding model identifier it("keeps a per-model task-fit comparison table for curated onboarding models", () => { const markdown = fs.readFileSync(chooseModelPath, "utf8"); const start = markdown.indexOf("## Model Task Fit"); diff --git a/test/install-clone-ref.test.ts b/test/install-clone-ref.test.ts new file mode 100644 index 00000000000..c2675b92638 --- /dev/null +++ b/test/install-clone-ref.test.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { INSTALLER_PAYLOAD } from "./helpers/installer-sourced-env"; + +const CURL_PIPE_INSTALLER = path.join(import.meta.dirname, "..", "install.sh"); + +describe("installer git checkout", () => { + it("fetches fully-qualified refs into a detached checkout", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-clone-ref-")); + const origin = path.join(tmp, "origin"); + fs.mkdirSync(origin); + const git = (args: string[], cwd = origin) => spawnSync("git", args, { cwd, encoding: "utf8" }); + + try { + expect(git(["init", "--initial-branch=topic"]).status).toBe(0); + expect(git(["config", "user.name", "NemoClaw Test"]).status).toBe(0); + expect(git(["config", "user.email", "nemoclaw-test@example.invalid"]).status).toBe(0); + fs.writeFileSync(path.join(origin, "README.md"), "fixture\n"); + expect(git(["add", "README.md"]).status).toBe(0); + expect(git(["-c", "commit.gpgsign=false", "commit", "-m", "fixture"]).status).toBe(0); + const expectedHead = git(["rev-parse", "HEAD"]).stdout.trim(); + + for (const [index, installer] of [INSTALLER_PAYLOAD, CURL_PIPE_INSTALLER].entries()) { + const destination = path.join(tmp, `checkout-${index}`); + const result = spawnSync( + "bash", + [ + "-c", + 'source "$INSTALLER_UNDER_TEST"\nclone_nemoclaw_ref refs/heads/topic "$DESTINATION"', + ], + { + encoding: "utf8", + env: { + ...process.env, + DESTINATION: destination, + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `url.file://${origin}.insteadOf`, + GIT_CONFIG_VALUE_0: "https://github.com/NVIDIA/NemoClaw.git", + INSTALLER_UNDER_TEST: installer, + }, + }, + ); + expect(result.status, result.stderr).toBe(0); + expect(git(["-C", destination, "rev-parse", "HEAD"], tmp).stdout.trim()).toBe(expectedHead); + expect(git(["-C", destination, "symbolic-ref", "-q", "HEAD"], tmp).status).not.toBe(0); + } + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); diff --git a/test/install-preflight.test.ts b/test/install-preflight.test.ts index 32b2d80217b..12a2e7301ff 100644 --- a/test/install-preflight.test.ts +++ b/test/install-preflight.test.ts @@ -2151,19 +2151,6 @@ exit 99`, expect(result.stdout.trim()).toBe("v0.2.0"); }); - it("clone_nemoclaw_ref uses fetch checkout so fully-qualified refs work", () => { - const payload = fs.readFileSync(INSTALLER_PAYLOAD, "utf-8"); - const bootstrap = fs.readFileSync(CURL_PIPE_INSTALLER, "utf-8"); - for (const src of [payload, bootstrap]) { - const fn = src.match(/clone_nemoclaw_ref\(\) \{([\s\S]*?)^}/m); - expect(fn).toBeTruthy(); - expect(fn![1]).toContain('git init --quiet "$dest"'); - expect(fn![1]).toContain('git -C "$dest" fetch --quiet --depth 1 origin "$ref"'); - expect(fn![1]).toContain("checkout --quiet --detach FETCH_HEAD"); - expect(fn![1]).not.toContain("clone --quiet --depth 1 --branch"); - } - }); - it("source-checkout path does NOT call resolve_release_tag / git clone", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-source-notag-")); const fakeBin = path.join(tmp, "bin"); diff --git a/test/label-merged-pr-release-target-workflow.test.ts b/test/label-merged-pr-release-target-workflow.test.ts index 41d6cb5d8ab..fcfadb250f5 100644 --- a/test/label-merged-pr-release-target-workflow.test.ts +++ b/test/label-merged-pr-release-target-workflow.test.ts @@ -169,12 +169,12 @@ async function runScript(harness: ReturnType): Promise { + // source-shape-contract: security -- Pull-request-target labeling must never execute or checkout untrusted contributor code it("keeps fork-safe labeling inside the trusted metadata boundary", () => { expect(workflow.on?.pull_request_target).toEqual({ branches: ["main"], types: ["closed"], }); - expect(workflow.on?.schedule).toEqual([{ cron: "17 */6 * * *" }]); expect(workflow.on).toHaveProperty("workflow_dispatch"); expect(workflow.permissions).toEqual({ contents: "read", @@ -184,7 +184,6 @@ describe("merged PR release target workflow", () => { expect(job.if).toBe( "${{ github.event_name != 'pull_request_target' || github.event.pull_request.merged == true }}", ); - expect(job["timeout-minutes"]).toBe(10); expect(actionStep?.uses).toMatch(/^actions\/github-script@[0-9a-f]{40}$/u); expect(job.steps).toHaveLength(1); expect(job.steps?.some((step) => step.uses?.startsWith("actions/checkout@"))).toBe(false); @@ -218,7 +217,6 @@ describe("merged PR release target workflow", () => { Object.assign(harness.context.payload, { pull_request: pullRequest }); await expect(runScript(harness)).rejects.toThrow(error); - expect(harness.listTags).not.toHaveBeenCalled(); expect(harness.addLabels).not.toHaveBeenCalled(); }); @@ -557,7 +555,6 @@ describe("merged PR release target workflow", () => { await expect(runScript(harness)).rejects.toThrow( "Newest release tag kept changing during reconciliation", ); - expect(harness.listTags).toHaveBeenCalledTimes(4); expect(harness.warning).toHaveBeenCalledTimes(2); expect(harness.getBranch).not.toHaveBeenCalled(); diff --git a/test/macos-e2e-workflow-boundary.test.ts b/test/macos-e2e-workflow-boundary.test.ts index c030f129977..3ee574ca329 100644 --- a/test/macos-e2e-workflow-boundary.test.ts +++ b/test/macos-e2e-workflow-boundary.test.ts @@ -48,6 +48,7 @@ function stepNamed(name: string, jobName = "macos-e2e"): WorkflowStep { } describe("macOS E2E workflow boundary", () => { + // source-shape-contract: security -- Live credentials must stay gated to trusted main-branch workflow code it("keeps secret-bearing live E2E on trusted main-branch code", () => { expect(readMacosWorkflow().on?.pull_request).toBeDefined(); @@ -66,41 +67,40 @@ describe("macOS E2E workflow boundary", () => { expect(jobNamed("macos-docker-final-destroy").if).toContain("github.ref == 'refs/heads/main'"); }); - it("runs final-destroy against a commit-pinned Docker setup on trusted Intel macOS", () => { - const job = jobNamed("macos-docker-final-destroy"); - const docker = stepNamed("Set up pinned Docker Engine", "macos-docker-final-destroy"); - const live = stepNamed("Run macOS Docker final-destroy E2E", "macos-docker-final-destroy"); + // source-shape-contract: compatibility -- Real Docker cleanup must invoke its gated live lane on the reviewed Intel runner and engine + it("runs final-destroy against a pinned Docker setup on trusted Intel macOS", () => { + const job = readMacosWorkflow().jobs?.["macos-docker-final-destroy"]; + const docker = job?.steps?.find((step) => step.name === "Set up pinned Docker Engine"); + const live = job?.steps?.find((step) => step.name === "Run macOS Docker final-destroy E2E"); - expect(job["runs-on"]).toBe("macos-15-intel"); - expect(job.permissions).toEqual({ contents: "read" }); - expect(docker.uses).toBe("docker/setup-docker-action@6d7cfa65f60a9dda7b46e5513fa982536f3c9877"); - expect(docker.with?.version).toBe("v27.4.0"); - expect(String(docker.env?.LIMA_START_ARGS)).toContain("--cpus 4 --memory 8"); - expect(live.run).toContain("test/e2e/live/sandbox-operations.test.ts"); - expect(live.env?.NEMOCLAW_NON_INTERACTIVE).toBe("1"); + expect(job?.["runs-on"]).toBe("macos-15-intel"); + expect(job?.permissions).toEqual({ contents: "read" }); + expect(docker?.uses).toBe( + "docker/setup-docker-action@6d7cfa65f60a9dda7b46e5513fa982536f3c9877", + ); + expect(docker?.with?.version).toBe("v27.4.0"); + expect(live?.run).toContain("npx vitest run --project e2e-live"); + expect(live?.run).toContain("test/e2e/live/sandbox-operations.test.ts"); + expect(live?.env?.NEMOCLAW_RUN_LIVE_E2E).toBe("1"); + expect(live?.env?.NEMOCLAW_NON_INTERACTIVE).toBe("1"); }); - it("uploads live macOS E2E artifacts when the workflow fails", () => { - const upload = stepNamed("Upload logs on failure"); - const dockerUpload = stepNamed( - "Upload macOS Docker logs on failure", - "macos-docker-final-destroy", + // source-shape-contract: security -- Failure-only macOS artifact publishers must retain diagnostic paths and immutable actions + it("pins live macOS artifact publishers to an immutable action", () => { + const workflow = readMacosWorkflow(); + const upload = workflow.jobs?.["macos-e2e"]?.steps?.find( + (step) => step.name === "Upload logs on failure", ); - - expect(upload.if).toBe("failure() && github.event_name == 'pull_request'"); - expect(String(upload.with?.path)).toContain("/tmp/nemoclaw-e2e-*.log"); - expect(String(upload.with?.path)).toContain("${{ github.workspace }}/e2e-artifacts/live"); - - expect(dockerUpload.if).toBe("failure()"); - expect(dockerUpload.uses).toBe( - "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + const dockerUpload = workflow.jobs?.["macos-docker-final-destroy"]?.steps?.find( + (step) => step.name === "Upload macOS Docker logs on failure", ); - expect(String(dockerUpload.with?.path)).toContain("/tmp/nemoclaw-e2e-*.log"); - expect(String(dockerUpload.with?.path)).toContain("${{ github.workspace }}/e2e-artifacts/live"); - }); - it("bounds the fast and real-Docker macOS jobs independently", () => { - expect(jobNamed("macos-e2e")["timeout-minutes"]).toBe(30); - expect(jobNamed("macos-docker-final-destroy")["timeout-minutes"]).toBe(90); + for (const step of [upload, dockerUpload]) { + expect(step?.uses).toBe("actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"); + expect(String(step?.with?.path)).toContain("/tmp/nemoclaw-e2e-*.log"); + expect(String(step?.with?.path)).toContain("${{ github.workspace }}/e2e-artifacts/live"); + } + expect(upload?.if).toBe("failure() && github.event_name == 'pull_request'"); + expect(dockerUpload?.if).toBe("failure()"); }); }); diff --git a/test/mcp-openshell-workflow.test.ts b/test/mcp-openshell-workflow.test.ts index cec0dcd2041..0f2bea654d5 100644 --- a/test/mcp-openshell-workflow.test.ts +++ b/test/mcp-openshell-workflow.test.ts @@ -9,6 +9,7 @@ import credentialBoundaryManifest from "../src/lib/actions/sandbox/openshell-chi import { validateMcpOpenShellWorkflowBoundary } from "../tools/e2e/mcp-workflow-boundary.mts"; describe("MCP OpenShell workflow boundary", () => { + // source-shape-contract: compatibility -- Setup guidance must advertise the pinned runtime compatibility boundary it("keeps the setup docs aligned with the stable default", () => { const setupDocs = fs.readFileSync("docs/deployment/set-up-mcp-bridge.mdx", "utf8"); diff --git a/test/openclaw-dependency-review.test.ts b/test/openclaw-dependency-review.test.ts index 6d069ac87ea..9f8d9769cbf 100644 --- a/test/openclaw-dependency-review.test.ts +++ b/test/openclaw-dependency-review.test.ts @@ -58,12 +58,6 @@ function requiredStep(job: WorkflowJob, name: string): WorkflowStep { return step as WorkflowStep; } -function requiredStepIndex(job: WorkflowJob, name: string): number { - const index = job.steps?.findIndex((candidate) => candidate.name === name) ?? -1; - expect(index, `Missing workflow step: ${name}`).toBeGreaterThanOrEqual(0); - return index; -} - function findProductionBuildGuardCoverage( workflowName: string, workflow: Workflow, @@ -459,37 +453,10 @@ grep -Fq -- '--phase post-agent-install' Dockerfile } }); - it("guards and exports the base-image dispatch version as one scalar", () => { + it("accepts reviewed base-image versions and rejects injected build arguments", () => { const baseImages = readYaml(".github/workflows/base-image.yaml"); const buildAndPush = baseImages.jobs["build-and-push"] as WorkflowJob; const guard = requiredStep(buildAndPush, "Validate production Docker build args"); - const build = requiredStep(buildAndPush, "Build and push"); - - expect(guard.id).toBe("production-build-args"); - expect(guard.env).toEqual({ - OPENCLAW_VERSION_INPUT: "${{ inputs.openclaw_version }}", - }); - expect(guard.run).toContain(`"$OPENCLAW_VERSION_INPUT" == *$'\\r'*`); - expect(guard.run).toContain(`"$OPENCLAW_VERSION_INPUT" == *$'\\n'*`); - expect(guard.run).toContain(`"$OPENCLAW_VERSION_INPUT" =~ ^[0-9]+([.][0-9]+)*$`); - expect(guard.run).toContain('scripts/check-production-build-args.sh "${build_args[@]}"'); - expect(guard.run).toContain( - `printf 'openclaw_build_arg=%s\\n' "$openclaw_build_arg" >> "$GITHUB_OUTPUT"`, - ); - expect(build.with?.["build-args"]).toBe( - "${{ steps.production-build-args.outputs.openclaw_build_arg }}", - ); - expect(requiredStepIndex(buildAndPush, "Validate production Docker build args")).toBeLessThan( - requiredStepIndex(buildAndPush, "Build and push"), - ); - - for (const [jobName, job] of Object.entries(baseImages.jobs)) { - for (const step of job.steps ?? []) { - expect(step.run ?? "", `${jobName}:${step.name ?? "unnamed step"}`).not.toContain( - "${{ inputs.openclaw_version }}", - ); - } - } for (const [input, expectedOutput] of [ ["", "openclaw_build_arg=\n"], @@ -521,6 +488,7 @@ grep -Fq -- '--phase post-agent-install' Dockerfile } }); + // source-shape-contract: security -- Network-fetched distribution audits must execute only from trusted main workflow code it("runs and gates the real patched-distribution harness only from trusted main code", () => { const pr = readYaml(".github/workflows/pr.yaml"); const main = readYaml(".github/workflows/main.yaml"); @@ -531,7 +499,6 @@ grep -Fq -- '--phase post-agent-install' Dockerfile expect(pr.permissions).toEqual({ contents: "read" }); expect(prJob).toBeUndefined(); - expect(mainJob?.["timeout-minutes"]).toBe(20); expect(requiredStep(mainJob, "Audit the real patched OpenClaw distribution").env).toMatchObject( { NEMOCLAW_REAL_OPENCLAW_DIST_HARNESS: "1", @@ -547,12 +514,6 @@ grep -Fq -- '--phase post-agent-install' Dockerfile requiredStep(mainJob, "Audit managed OpenClaw security finding suppressions").run, ).toContain("test/openclaw-security-audit-suppressions-real.test.ts"); expect(requiredStep(mainJob, "Install test dependencies").run).toBe("npm ci --ignore-scripts"); - expect(mainJob.env).toMatchObject({ - npm_config_fetch_retries: "3", - npm_config_fetch_retry_mintimeout: "10000", - npm_config_fetch_retry_maxtimeout: "60000", - }); - expect(prChecks.needs).not.toContain("real-openclaw-dist-harness"); expect(mainChecks.needs).toContain("real-openclaw-dist-harness"); const prGate = requiredStep(prChecks, "Verify required PR checks"); diff --git a/test/openclaw-lifecycle-policy.test.ts b/test/openclaw-lifecycle-policy.test.ts index e5565c33e27..bbcc1d5df06 100644 --- a/test/openclaw-lifecycle-policy.test.ts +++ b/test/openclaw-lifecycle-policy.test.ts @@ -100,6 +100,7 @@ console.log(JSON.stringify({ `; describe("reviewed npm lifecycle policy", () => { + // source-shape-contract: security -- Every executable archive install must match the reviewed fail-closed lifecycle allowlist it("cross-checks the allowlist against every production archive install boundary", () => { expect(policy).toMatchObject({ schemaVersion: 1, defaultPolicy: "deny" }); expect(policy.allowedLifecycleScripts).not.toHaveLength(0); diff --git a/test/openclaw-real-patched-dist-harness.test.ts b/test/openclaw-real-patched-dist-harness.test.ts index 6e7a458307d..93d9ca40d83 100644 --- a/test/openclaw-real-patched-dist-harness.test.ts +++ b/test/openclaw-real-patched-dist-harness.test.ts @@ -234,7 +234,8 @@ describe("OpenClaw real patched-dist materialization guard", () => { }; }; - expect(() => { + let failure: unknown; + try { materializeReviewedTarball( "https://registry.npmjs.org/openclaw/-/openclaw-drifted.tgz", tmp, @@ -242,7 +243,12 @@ describe("OpenClaw real patched-dist materialization guard", () => { fakePack, ); installStarted = true; - }).toThrow(/OpenClaw tarball SRI/); + } catch (caught) { + failure = caught; + } + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toMatch(/OpenClaw tarball SRI/); expect(installStarted).toBe(false); } finally { fs.rmSync(tmp, { recursive: true, force: true }); @@ -259,7 +265,8 @@ describe("OpenClaw real patched-dist materialization guard", () => { stderr: "", }); - expect(() => { + let failure: unknown; + try { materializeReviewedTarball( "https://registry.npmjs.org/openclaw/-/openclaw-unsafe.tgz", tmp, @@ -267,7 +274,12 @@ describe("OpenClaw real patched-dist materialization guard", () => { fakePack, ); installStarted = true; - }).toThrow(/unsafe archive filename/); + } catch (caught) { + failure = caught; + } + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toMatch(/unsafe archive filename/); expect(installStarted).toBe(false); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/plugin-vitest-project.test.ts b/test/plugin-vitest-project.test.ts index 2feec42e4d9..7534ed43df5 100644 --- a/test/plugin-vitest-project.test.ts +++ b/test/plugin-vitest-project.test.ts @@ -38,6 +38,7 @@ function listedTypeScriptFiles(configPath: string): string[] { } describe("plugin Vitest project contract", () => { + // source-shape-contract: compatibility -- Root and standalone plugin runners must consume one canonical project contract it("defines one canonical plugin project for root and standalone runs", () => { const sourceTransform = pluginVitestProjectOptions.oxc; const policyAliases = pluginVitestProjectOptions.test.alias as PolicyAlias[]; @@ -72,6 +73,7 @@ describe("plugin Vitest project contract", () => { }); }); + // source-shape-contract: compatibility -- Assertion enforcement must remain scoped to the expect-based plugin project it("pilots assertion presence only in expect-based plugin tests (#6692)", () => { const rootTest = rootVitestConfig.test as { expect?: { requireAssertions?: boolean }; diff --git a/test/pr-e2e-gate-workflow.test.ts b/test/pr-e2e-gate-workflow.test.ts index 3754d531a68..388f5a207dd 100644 --- a/test/pr-e2e-gate-workflow.test.ts +++ b/test/pr-e2e-gate-workflow.test.ts @@ -163,6 +163,40 @@ function runStartStep(headBranch: string, prNumber = "42") { } } +function runCancelStep(prNumber: string) { + const workflow = readYaml(PR_GATE_PATH); + const cancel = step(workflow.jobs["cancel-superseded"], "Cancel superseded E2E runs"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-cancel-step-")); + const binDir = path.join(tempDir, "bin"); + const argumentsPath = path.join(tempDir, "node-arguments"); + fs.mkdirSync(binDir); + fs.writeFileSync( + path.join(binDir, "node"), + '#!/usr/bin/env bash\nset -euo pipefail\nprintf \'%s\\0\' "$@" > "$FAKE_NODE_ARGUMENTS"\n', + { mode: 0o755 }, + ); + + try { + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", cancel.run!], { + encoding: "utf8", + env: { + ...process.env, + FAKE_NODE_ARGUMENTS: argumentsPath, + GITHUB_TOKEN: "token", + PATH: `${binDir}:${process.env.PATH ?? ""}`, + PR_NUMBER: prNumber, + }, + timeout: 5_000, + }); + return { + arguments: fs.readFileSync(argumentsPath, "utf8").split("\0").slice(0, -1), + result, + }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + function runChildValidation(currentPullSha: string) { const workflow = readYaml(E2E_PATH); const validation = step(workflow.jobs["generate-matrix"], "Validate controller dispatch"); @@ -219,6 +253,7 @@ esac } describe("PR E2E gate workflow", () => { + // source-shape-contract: security -- Trusted metadata triggers and least privilege bound the write-capable controller it("limits triggers and job permissions", () => { const workflow = readYaml(PR_GATE_PATH); const cancel = workflow.jobs["cancel-superseded"]; @@ -254,6 +289,7 @@ describe("PR E2E gate workflow", () => { expect(collectStrings(workflow).some((value) => value.includes("${{ secrets."))).toBe(false); }); + // source-shape-contract: security -- Controller checkouts and dependency installs must not execute mutable contributor hooks it("pins both controller checkouts and installs without lifecycle scripts or caches", () => { const workflow = readYaml(PR_GATE_PATH); const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); @@ -286,15 +322,18 @@ describe("PR E2E gate workflow", () => { }); it("cancels superseded PR runs", () => { - const workflow = readYaml(PR_GATE_PATH); - const cancel = workflow.jobs["cancel-superseded"]; - const cancelStep = step(cancel, "Cancel superseded E2E runs"); + const execution = runCancelStep("42"); - expect(cancelStep.run).toContain("tools/e2e/pr-e2e-gate.mts --mode cancel"); - expect(cancelStep.run).toContain('--pr "$PR_NUMBER"'); - expect(cancelStep.run).not.toContain("${{ github.event."); - expect(cancelStep.env?.GITHUB_TOKEN).toBe("${{ github.token }}"); - expect(cancelStep.env?.PR_NUMBER).toBe("${{ github.event.pull_request.number }}"); + expect(execution.result.status).toBe(0); + expect(execution.result.stderr).toBe(""); + expect(execution.arguments).toEqual([ + "--experimental-strip-types", + "tools/e2e/pr-e2e-gate.mts", + "--mode", + "cancel", + "--pr", + "42", + ]); }); it.each([ @@ -322,104 +361,6 @@ describe("PR E2E gate workflow", () => { expect(execution.arguments[prFlag + 1]).toBe(""); }); - it("coordinates the check around one E2E run", () => { - const workflow = readYaml(PR_GATE_PATH); - const job = workflow.jobs.coordinate; - const workspace = step(job, "Create private workspace"); - const start = step(job, "Start evaluation"); - const upload = step(job, "Upload risk plan"); - const wait = step(job, "Wait for E2E run"); - const download = step(job, "Download evidence"); - const finish = step(job, "Verify evidence"); - const fallback = step(job, "Close incomplete check"); - const cleanup = step(job, "Remove private workspace"); - - expect(job.concurrency).toEqual({ - group: - "pr-e2e-gate-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}", - "cancel-in-progress": false, - }); - expect(job["timeout-minutes"]).toBe(180); - expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-pr-e2e-gate.XXXXXX"'); - expect(workspace.run).toContain('chmod 700 "$work_dir"'); - expect(start.run).toContain("tools/e2e/pr-e2e-gate.mts --mode start"); - expect(start.run).toContain('--head "$HEAD_SHA"'); - expect(start.run).toContain('--head-repo "$HEAD_REPOSITORY"'); - expect(start.run).toContain('--head-branch "$HEAD_BRANCH"'); - expect(start.run).toContain('--workflow-sha "$WORKFLOW_SHA"'); - expect(start.run).toContain('--ci-conclusion "$CI_CONCLUSION"'); - expect(start.run).toContain('--ci-run-attempt "$CI_RUN_ATTEMPT"'); - expect(start.run).toContain('--ci-run-id "$CI_RUN_ID"'); - expect(start.run).toContain('--pr "$PR_NUMBER"'); - expect(start.run).toContain('--work-dir "$WORK_DIR"'); - expect(start.run).not.toContain("${{ github.event."); - expect(start.env).toMatchObject({ - CI_CONCLUSION: "${{ github.event.workflow_run.conclusion }}", - CI_RUN_ATTEMPT: "${{ github.event.workflow_run.run_attempt }}", - CI_RUN_ID: "${{ github.event.workflow_run.id }}", - HEAD_BRANCH: "${{ github.event.workflow_run.head_branch }}", - HEAD_REPOSITORY: "${{ github.event.workflow_run.head_repository.full_name }}", - HEAD_SHA: "${{ github.event.workflow_run.head_sha }}", - PR_NUMBER: "${{ github.event.workflow_run.pull_requests[0].number }}", - WORKFLOW_SHA: "${{ github.workflow_sha }}", - WORK_DIR: "${{ steps.workspace.outputs.work_dir }}", - }); - expect(start.run).not.toContain("--mode initialize"); - expect(upload.if).toContain("steps.workspace.outputs.work_dir != ''"); - expect(upload.with?.name).toBe("pr-e2e-risk-plan-${{ github.event.workflow_run.head_sha }}"); - expect(upload.with?.path).toBe("${{ steps.workspace.outputs.work_dir }}/risk-plan.json"); - expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); - expect(wait.run).toContain('gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY"'); - expect(wait.run).toContain("--json status,conclusion"); - expect(wait.run).toContain('if [[ "$state" != "$last_state" ]]'); - expect(wait.run).toContain("completed:success"); - expect(wait.run).toContain("completed:failure"); - expect(wait.run).toContain("sleep 10"); - expect(wait.run).toContain('if [ "$wait_status" -eq 124 ]'); - expect(wait.run).toContain('exit "$wait_status"'); - expect(wait.run).not.toContain("gh run watch"); - expect(wait.run).not.toContain("--json jobs"); - expect(wait.run).not.toContain("2>/dev/null"); - expect(wait["continue-on-error"]).toBe(true); - expect(download.if).toContain("always()"); - expect(download.run).toContain("timeout --signal=TERM --kill-after=30s 10m"); - expect(download.run).toContain('if [ "$download_status" -eq 124 ]'); - expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); - expect(download["continue-on-error"]).toBe(true); - expect(finish.if).toContain("always()"); - expect(finish.run).toContain("tools/e2e/pr-e2e-gate.mts --mode finish"); - expect(finish.run).toContain('--state-hash "${{ steps.start.outputs.state_hash }}"'); - expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); - expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); - expect(fallback.if).toContain("always()"); - expect(fallback.if).toContain("steps.start.outputs.check_id != ''"); - expect(fallback.if).toContain("steps.start.outputs.finalized != 'true'"); - expect(fallback.if).toContain("steps.finish.outputs.finalized != 'true'"); - expect(fallback.run).toContain("tools/e2e/pr-e2e-gate.mts --mode abandon"); - expect(fallback.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); - expect(cleanup.if).toContain("always() && steps.workspace.outputs.work_dir != ''"); - expect(cleanup.run).toContain('rm -rf -- "${{ steps.workspace.outputs.work_dir }}"'); - expect(collectStrings(workflow).some((value) => value.includes("/tmp/"))).toBe(false); - }); - - it("uses one child dispatch protocol and one correlated run title", () => { - const workflow = readYaml(E2E_PATH); - const inputs = workflow.on.workflow_dispatch.inputs; - - expect(inputs).toEqual( - expect.objectContaining({ - jobs: expect.any(Object), - pr_number: expect.any(Object), - checkout_sha: expect.any(Object), - plan_hash: expect.any(Object), - correlation_id: expect.any(Object), - }), - ); - expect(workflow["run-name"]).toContain( - "format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id)", - ); - }); - it("validates the E2E run against the PR head commit", () => { const current = runChildValidation("a".repeat(40)); const stale = runChildValidation("c".repeat(40)); diff --git a/test/pr-review-advisor-workflow-boundary.test.ts b/test/pr-review-advisor-workflow-boundary.test.ts index e87c4f326fe..11474c0de92 100644 --- a/test/pr-review-advisor-workflow-boundary.test.ts +++ b/test/pr-review-advisor-workflow-boundary.test.ts @@ -134,6 +134,7 @@ fi expect(validatePrReviewAdvisorWorkflowBoundary()).toEqual([]); }); + // source-shape-contract: security -- Exactly one advisor lane may perform write-capable PR comment publication it("requires one advisor lane to publish the PR comment", () => { const source = fs.readFileSync( path.join(ROOT, ".github/workflows/pr-review-advisor.yaml"), diff --git a/test/pr-workflow-contract.test.ts b/test/pr-workflow-contract.test.ts index e8beaa1859b..2325878c967 100644 --- a/test/pr-workflow-contract.test.ts +++ b/test/pr-workflow-contract.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -111,42 +111,6 @@ function requiredStepIndex(action: CompositeAction, stepName: string): number { return stepIndex; } -function uploadsCompiledCliArtifact( - action: CompositeAction, - shard: number, - shardCount: number, -): boolean { - const validationRun = requiredStep(action, "Validate shard inputs").run ?? ""; - const outputDirectory = mkdtempSync(join(tmpdir(), "nemoclaw-cli-shard-output-")); - const outputPath = join(outputDirectory, "github-output"); - try { - // Execute the repository-owned action body so producer selection stays a behavioral contract. - const result = spawnSync("bash", ["-c", validationRun], { - encoding: "utf8", - env: { - ...process.env, - CLI_SHARD: String(shard), - CLI_SHARD_COUNT: String(shardCount), - GITHUB_OUTPUT: outputPath, - }, - }); - expect( - result.status, - `Shard validation failed for ${shard}/${shardCount}: ${result.stderr}`, - ).toBe(0); - const output = readFileSync(outputPath, "utf8").match( - /^upload_build_artifact=(true|false)$/mu, - )?.[1]; - expect( - output, - `Shard validation omitted its artifact output for ${shard}/${shardCount}`, - ).toBeDefined(); - return output === "true"; - } finally { - rmSync(outputDirectory, { force: true, recursive: true }); - } -} - function requiredWorkflowStep(job: WorkflowJob, stepName: string): WorkflowStep { const step = job.steps?.find((candidate) => candidate.name === stepName); if (!step) { @@ -281,6 +245,7 @@ describe("pull request and main workflow contracts", () => { ".github/actions/resolve-hermes-base-image/action.yaml", ); + // source-shape-contract: security -- Installer hashes must be verified by base-trusted or immutable bootstrap code it("runs pull request installer verification from immutable trusted code", () => { const job = installerHashWorkflow.jobs["check-hash"]; const parserRuntimeSetup = requiredWorkflowStep( @@ -522,6 +487,7 @@ describe("pull request and main workflow contracts", () => { } }); + // source-shape-contract: security -- The trusted action must invoke its bundled verifier without PR-controlled resolution it("keeps the installer verifier inside the trusted composite action", () => { const verification = requiredStep(installerHashAction, "Verify installer hashes are current"); @@ -534,6 +500,7 @@ describe("pull request and main workflow contracts", () => { ); }); + // source-shape-contract: compatibility -- Path-filter semantics keep documentation-only and code-changing PR lanes distinct it("routes only code-changing PRs through the code-check path", () => { const filterStep = prWorkflow.jobs.changes.steps?.find((step) => step.id === "filter"); @@ -556,30 +523,7 @@ describe("pull request and main workflow contracts", () => { ).toBe(true); }); - it("keeps ordinary hooks automatic and full coverage explicit", () => { - const hooks = prekConfig.repos.flatMap((repo) => repo.hooks ?? []); - const hook = (id: string) => hooks.find((candidate) => candidate.id === id); - - expect(prekConfig.default_stages).toEqual(["pre-commit"]); - expect(hook("test-cli")?.stages).toEqual(["manual"]); - expect(hook("test-cli")?.entry).toBe("npm run test:coverage:cli"); - expect(hook("test-plugin")?.stages).toEqual(["manual"]); - expect(hook("test-plugin")?.entry).toBe("npm run test:coverage:plugin"); - for (const id of [ - "trailing-whitespace", - "end-of-file-fixer", - "shfmt", - "check-added-large-files", - "check-executables-have-shebangs", - "check-shebang-scripts-are-executable", - ]) { - expect(hook(id)?.stages, id).toEqual(["pre-commit"]); - } - for (const id of ["tsc-plugin", "tsc-js", "tsc-cli", "version-tag-sync"]) { - expect(hook(id)?.stages, id).toEqual(["pre-push"]); - } - }); - + // source-shape-contract: compatibility -- Repository checks must follow every authoritative dependency-pin input and consumer it("runs repository checks for every operational dependency-pin authority and consumer", () => { const hooks = prekConfig.repos.flatMap((repo) => repo.hooks ?? []); const repositoryChecks = hooks.find((candidate) => candidate.id === "repository-checks"); @@ -609,6 +553,26 @@ describe("pull request and main workflow contracts", () => { expect(files.test("docs/reference/commands.mdx")).toBe(false); }); + // source-shape-contract: compatibility -- Pre-commit routing must apply the declarative guard to every supported test location + it("runs the source-shape guard for root and co-located tests", () => { + const hooks = prekConfig.repos.flatMap((repo) => repo.hooks ?? []); + const sourceShape = hooks.find((candidate) => candidate.id === "source-shape-test-budget"); + const files = new RegExp(sourceShape?.files ?? "(?!)", "u"); + + expect(sourceShape?.entry).toBe("npm run source-shape:check"); + for (const path of [ + "test/example.test.ts", + "src/lib/example.spec.ts", + "nemoclaw/src/example.test.ts", + "scripts/find-source-shape-tests.ts", + "ci/source-shape-test-budget.json", + ]) { + expect(files.test(path), path).toBe(true); + } + expect(files.test("src/lib/example.ts")).toBe(false); + }); + + // source-shape-contract: compatibility -- Changed-file routing must typecheck each project and its transitive configuration inputs it("scopes pre-push typechecks to project and transitive inputs", () => { const hooks = prekConfig.repos.flatMap((repo) => repo.hooks ?? []); const pluginTypecheck = hooks.find((candidate) => candidate.id === "tsc-plugin"); @@ -740,6 +704,7 @@ describe("pull request and main workflow contracts", () => { ]); }); + // source-shape-contract: security -- Pull requests must execute base-trusted actions while main uses reviewed repository actions it("reuses the same shared CI actions in PR and main workflows", () => { for (const [jobName, stepName, trustedActionPath, mainActionPath] of [ [ @@ -923,146 +888,15 @@ describe("pull request and main workflow contracts", () => { } }); - it("preserves the shared static, build, and coverage gates", () => { - const staticRuns = stepRuns(sharedActions.staticChecks); - const staticRunsJoined = staticRuns.join("\n"); - const staticPrekRun = staticRuns.find((run) => - run.includes("npx prek run --all-files --stage pre-commit"), - ); - const buildRuns = stepRuns(sharedActions.buildTypecheck); - const cliShardRuns = stepRuns(sharedActions.cliCoverageShard).join("\n"); - const cliMergeRuns = stepRuns(sharedActions.cliCoverageMerge).join("\n"); - const pluginRuns = stepRuns(sharedActions.pluginCoverage).join("\n"); - const installerRuns = stepRuns(sharedActions.installerIntegration).join("\n"); - - expect(staticRuns).toContain("npm install --ignore-scripts"); - expect(staticRuns).toContain("npm --prefix nemoclaw ci --ignore-scripts --dry-run"); - expect( - requiredStepIndex(sharedActions.staticChecks, "Validate sandbox payload lockfile"), - ).toBeLessThan(requiredStepIndex(sharedActions.staticChecks, "Install dependencies")); - expect(staticRuns).toContain("npm run validate:configs"); - expect(staticRuns).toContain("npm run typecheck:scorecard"); - expect(staticPrekRun).toContain("npx prek run --all-files --stage pre-commit"); - for (const skippedHook of [ - "source-shape-test-budget", - "test-file-size-budget", - "test-skills-yaml", - ]) { - expect(staticPrekRun).toContain(`--skip ${skippedHook}`); - } - expect(staticPrekRun).not.toContain("--skip test-cli"); - expect(staticPrekRun).not.toContain("--skip test-plugin"); - expect(staticRuns).toContain("npm run source-shape:check"); - expect(staticRuns).toContain("npm run test-size:check"); - expect(staticRuns).toContain("npx vitest run test/skills-frontmatter.test.ts"); - expect(staticRuns).toContain("python3 scripts/generate-platform-docs.py --check"); + // source-shape-contract: security -- Downloaded CI tooling must use a committed digest rather than upstream metadata + it("pins downloaded CI tooling to reviewed integrity", () => { + const staticRunsJoined = stepRuns(sharedActions.staticChecks).join("\n"); + expect(staticRunsJoined).toContain( 'HADOLINT_SHA256="6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc5a47"', ); expect(staticRunsJoined).not.toContain('"${HADOLINT_URL}.sha256"'); expect(staticRunsJoined).not.toContain("EXPECTED=$(curl"); - - expect(buildRuns.join("\n")).toContain("cd nemoclaw && npm install --ignore-scripts"); - expect(buildRuns).toContain("cd nemoclaw && npm run build"); - expect(buildRuns).toContain("npm run build:cli"); - expect(buildRuns).toContain("npx vitest run --project package-contract"); - expect(buildRuns).toContain("npm run typecheck:cli"); - expect(buildRuns).toContain("npm --prefix nemoclaw run typecheck"); - expect(buildRuns).toContain("npx tsc -p jsconfig.json"); - expect(buildRuns).toContain("bash scripts/check-version-tag-sync.sh"); - - expect(cliShardRuns).toContain("cd nemoclaw && npm run build"); - expect(cliShardRuns).toContain("npm run build:cli"); - expect(cliShardRuns).toContain("npx tsx scripts/check-dist-sourcemaps.ts dist"); - expect(cliShardRuns).toContain("npx vitest run --project cli --project integration"); - expect(cliShardRuns).toContain('--coverage.include="src/**/*.ts"'); - expect(cliShardRuns).not.toContain('--coverage.include="dist/lib/**/*.js"'); - expect(cliShardRuns).toContain('--shard="${CLI_SHARD}/${CLI_SHARD_COUNT}"'); - expect(cliShardRuns).toContain("--reporter=github-actions"); - expect(cliShardRuns).toContain("--reporter=blob"); - expect(cliShardRuns).toContain( - '--outputFile.blob=".vitest-reports/blob-${CLI_SHARD}-${CLI_SHARD_COUNT}.json"', - ); - expect(cliShardRuns).toContain('--coverage.reportsDirectory="coverage/cli/shard-${CLI_SHARD}"'); - expect(cliShardRuns).not.toContain("${{ inputs.shard"); - expect(cliShardRuns).not.toContain("scripts/check-coverage-ratchet.ts"); - - expect(cliMergeRuns).not.toContain("npm run build:cli"); - expect(cliMergeRuns).toContain("test -s dist/nemoclaw.js"); - expect(cliMergeRuns).toContain("npx tsx scripts/check-dist-sourcemaps.ts dist"); - expect(cliMergeRuns).toContain('blob=".vitest-reports/blob-${shard}-${CLI_SHARD_COUNT}.json"'); - expect(cliMergeRuns).toContain( - 'find .vitest-reports -maxdepth 1 -type f -name "blob-*-${CLI_SHARD_COUNT}.json"', - ); - expect(cliMergeRuns).not.toContain("${{ inputs.shard-count"); - expect(cliMergeRuns).toContain("npx vitest --mergeReports .vitest-reports"); - expect(cliMergeRuns).toContain("--reporter=json"); - expect(cliMergeRuns).toContain("--outputFile.json=coverage/cli/vitest-results.json"); - expect(cliMergeRuns).toContain("--coverage.reportsDirectory=coverage/cli"); - expect(cliMergeRuns).toContain('--coverage.include="src/**/*.ts"'); - expect(cliMergeRuns).not.toContain('--coverage.include="dist/lib/**/*.js"'); - expect(cliMergeRuns).toContain( - 'scripts/check-coverage-ratchet.ts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json "CLI coverage"', - ); - - expect(pluginRuns).toContain("npx vitest run --project plugin"); - expect(pluginRuns).toContain( - 'scripts/check-coverage-ratchet.ts coverage/plugin/coverage-summary.json ci/coverage-threshold-plugin.json "Plugin coverage"', - ); - - expect(installerRuns).toContain("npm install --ignore-scripts"); - expect(installerRuns).toContain("cd nemoclaw && npm install --ignore-scripts"); - expect(installerRuns).toContain("npm run build:cli"); - expect(installerRuns).toContain("cd nemoclaw && npm run build"); - expect(installerRuns).toContain("CI=true npx vitest run --project installer-integration"); - }); - - it("keeps PR coverage for non-opt-in Vitest projects after removing the self-hosted full run", () => { - const vitestConfig = readFileSync("vitest.config.ts", "utf8"); - const cliShardRuns = stepRuns(sharedActions.cliCoverageShard).join("\n"); - const installerRuns = stepRuns(sharedActions.installerIntegration).join("\n"); - const prInstallerRuns = stepRuns(prWorkflow.jobs["installer-integration"]).join("\n"); - - expect(installerRuns).toContain("CI=true npx vitest run --project installer-integration"); - expect(prInstallerRuns).toContain("CI=true npx vitest run --project installer-integration"); - expect(stepUses(prWorkflow.jobs["installer-integration"])).toContain( - trustedPrActionPaths.installerIntegration, - ); - expect(stepUses(mainWorkflow.jobs["installer-integration"])).toContain( - sharedActionPaths.installerIntegration, - ); - expect(vitestConfig).toContain('name: "installer-integration"'); - - // Source and integration coverage are sharded together, while support, - // installer, package, and live projects remain disjoint explicit lanes. - expect(cliShardRuns).toContain("npx vitest run --project cli --project integration"); - expect(vitestConfig).toContain('name: "cli"'); - expect(vitestConfig).toContain('include: ["src/**/*.test.ts"]'); - expect(vitestConfig).toContain('name: "integration"'); - expect(vitestConfig).toContain('include: ["test/**/*.test.{js,ts}"]'); - expect(vitestConfig).toContain('name: "e2e-support"'); - expect(stepRuns(prWorkflow.jobs["e2e-support"])).toEqual([ - "npm ci --ignore-scripts", - "npx tsx scripts/checks/e2e-mock-parity.ts --base HEAD^1 --head HEAD^2", - "npm run build:cli", - "npx vitest run --project e2e-support", - ]); - expect(stepRuns(mainWorkflow.jobs["e2e-support"])).toEqual([ - "npm ci --ignore-scripts", - `if [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then - echo "Skipping changed live E2E parity: main has no prior commit." - exit 0 -fi -npx tsx scripts/checks/e2e-mock-parity.ts --base "$BASE_SHA" --head HEAD -`, - "npm run build:cli", - "npx vitest run --project e2e-support", - ]); - expect(vitestConfig).toContain('name: "package-contract"'); - expect(vitestConfig).toContain('"test/e2e/**"'); - expect(vitestConfig).toContain('"test/install-express-prompt.test.ts"'); - expect(vitestConfig).toContain('"test/install-preflight.test.ts"'); - expect(vitestConfig).toContain('"test/install-openshell-version-check.test.ts"'); }); it("validates CLI shard inputs before using them in shell commands", () => { @@ -1070,50 +904,42 @@ npx tsx scripts/checks/e2e-mock-parity.ts --base "$BASE_SHA" --head HEAD sharedActions.cliCoverageShard, "Validate shard inputs", ); - const shardValidationRun = shardValidationStep.run ?? ""; - const shardRunStep = requiredStep(sharedActions.cliCoverageShard, "Run CLI coverage shard"); const mergeValidationStep = requiredStep( sharedActions.cliCoverageMerge, "Validate shard inputs", ); - const mergeValidationRun = mergeValidationStep.run ?? ""; - const mergeVerifyStep = requiredStep( - sharedActions.cliCoverageMerge, - "Verify CLI shard blob reports", - ); + const temp = mkdtempSync(join(tmpdir(), "nemoclaw-cli-shard-validation-")); + const marker = join(temp, "injected"); + const shellPayload = `$(touch ${marker})`; - expect(shardValidationStep.env).toEqual({ - CLI_SHARD: "${{ inputs.shard }}", - CLI_SHARD_COUNT: "${{ inputs.shard-count }}", - }); - expect(shardValidationRun).toContain("*[!0-9]*"); - expect(shardValidationRun).toContain("Invalid CLI shard"); - expect(shardValidationRun).toContain("Invalid CLI shard count"); - expect(shardValidationRun).toContain("Invalid CLI shard range"); - expect(shardRunStep.env).toEqual({ - CLI_SHARD: "${{ inputs.shard }}", - CLI_SHARD_COUNT: "${{ inputs.shard-count }}", - }); - expect(requiredStepIndex(sharedActions.cliCoverageShard, "Validate shard inputs")).toBeLessThan( - requiredStepIndex(sharedActions.cliCoverageShard, "Run CLI coverage shard"), - ); + try { + const invalidShard = runWorkflowShellStep(shardValidationStep, { + CLI_SHARD: shellPayload, + CLI_SHARD_COUNT: "8", + GITHUB_OUTPUT: join(temp, "github-output"), + }); + const invalidRange = runWorkflowShellStep(shardValidationStep, { + CLI_SHARD: "9", + CLI_SHARD_COUNT: "8", + GITHUB_OUTPUT: join(temp, "github-output"), + }); + const invalidCount = runWorkflowShellStep(mergeValidationStep, { + CLI_SHARD_COUNT: shellPayload, + }); - expect(mergeValidationStep.env).toEqual({ - CLI_SHARD_COUNT: "${{ inputs.shard-count }}", - }); - expect(mergeValidationRun).toContain("*[!0-9]*"); - expect(mergeValidationRun).toContain("Invalid CLI shard count"); - expect(mergeVerifyStep.env).toEqual({ - CLI_SHARD_COUNT: "${{ inputs.shard-count }}", - }); - expect(requiredStepIndex(sharedActions.cliCoverageMerge, "Validate shard inputs")).toBeLessThan( - requiredStepIndex(sharedActions.cliCoverageMerge, "Verify CLI shard blob reports"), - ); - expect(requiredStepIndex(sharedActions.cliCoverageMerge, "Validate shard inputs")).toBeLessThan( - requiredStepIndex(sharedActions.cliCoverageMerge, "Merge CLI coverage"), - ); + expect(invalidShard.status).not.toBe(0); + expect(invalidShard.stdout).toContain("Invalid CLI shard"); + expect(invalidRange.status).not.toBe(0); + expect(invalidRange.stdout).toContain("Invalid CLI shard range"); + expect(invalidCount.status).not.toBe(0); + expect(invalidCount.stdout).toContain("Invalid CLI shard count"); + expect(existsSync(marker)).toBe(false); + } finally { + rmSync(temp, { force: true, recursive: true }); + } }); + // source-shape-contract: security -- Growth-budget changes must inspect trusted GitHub data without fetching PR-authored URLs it("keeps the trusted test-size guard closed around budget policy changes", () => { const growthGuardrails = readYaml( ".github/workflows/codebase-growth-guardrails.yaml", @@ -1128,226 +954,87 @@ npx tsx scripts/checks/e2e-mock-parity.ts --base "$BASE_SHA" --head HEAD expect(guardRun).toContain("has a legacy budget but no matching test file at the PR head"); }); - it("uploads CLI Vitest JSON results for timing analysis", () => { - const uploadStep = requiredStep( - sharedActions.cliCoverageMerge, - "Upload CLI Vitest timing report", - ); - - expect(uploadStep.if).toBe("always()"); - expect(uploadStep.uses).toContain("actions/upload-artifact@"); - expect(uploadStep.with?.name).toBe("cli-vitest-results"); - expect(uploadStep.with?.path).toBe("coverage/cli/vitest-results.json"); - expect(uploadStep.with?.["if-no-files-found"]).toBe("warn"); - expect(uploadStep.with?.["retention-days"]).toBe(14); - }); - - it("uploads same-repository CLI and plugin Cobertura reports (#6692)", () => { + // source-shape-contract: security -- Coverage publication must exclude fork-authored reports and pin the publishing action + it("publishes coverage only from same-repository code (#6692)", () => { const sameRepositoryGuard = "${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}"; const uploadAction = "actions/upload-code-coverage@abb5995db9e0199b0e2bb9dbd136fce4cb1ec4d3"; const reports = [ { action: sharedActions.cliCoverageMerge, - coverageStep: "Merge CLI coverage", uploadStep: "Upload CLI coverage report", - file: "coverage/cli/cobertura-coverage.xml", - label: "code-coverage/cli", }, { action: sharedActions.pluginCoverage, - coverageStep: "Run plugin coverage", uploadStep: "Upload plugin coverage report", - file: "coverage/plugin/cobertura-coverage.xml", - label: "code-coverage/plugin", }, ] as const; for (const report of reports) { - expect(requiredStep(report.action, report.coverageStep).run).toContain( - "--coverage.reporter=cobertura", - ); - const uploadStep = requiredStep(report.action, report.uploadStep); expect(uploadStep.if).toBe(sameRepositoryGuard); expect(uploadStep.uses).toBe(uploadAction); - expect(uploadStep.with).toEqual({ - file: report.file, - language: "TypeScript", - label: report.label, - }); - } - }); - - it("runs CLI coverage in shards and merges coverage before ratcheting", () => { - expect(sharedActions.cliCoverageShard.inputs?.["shard-count"]?.default).toBe(cliShardCount); - expect(sharedActions.cliCoverageMerge.inputs?.["shard-count"]?.default).toBe(cliShardCount); - - const compiledCliUploadStep = requiredStep( - sharedActions.cliCoverageShard, - "Upload compiled CLI artifact", - ); - const shardUploadStep = requiredStep( - sharedActions.cliCoverageShard, - "Upload CLI shard blob report", - ); - const compiledCliDownloadStep = requiredStep( - sharedActions.cliCoverageMerge, - "Download compiled CLI artifact", - ); - const downloadStep = requiredStep( - sharedActions.cliCoverageMerge, - "Download CLI shard blob reports", - ); - const verifyRun = requiredStep( - sharedActions.cliCoverageMerge, - "Verify CLI shard blob reports", - ).run; - - expect(compiledCliUploadStep.if).toBe( - "${{ steps.validate-shard-inputs.outputs.upload_build_artifact == 'true' && success() }}", - ); - expect(compiledCliUploadStep.uses).toContain("actions/upload-artifact@"); - expect(compiledCliUploadStep.with).toEqual({ - name: "cli-build-output", - path: "dist", - "if-no-files-found": "error", - "retention-days": 1, - }); - expect( - requiredStepIndex(sharedActions.cliCoverageShard, "Build CLI for coverage shard"), - ).toBeLessThan( - requiredStepIndex(sharedActions.cliCoverageShard, "Upload compiled CLI artifact"), - ); - expect( - requiredStepIndex(sharedActions.cliCoverageShard, "Upload compiled CLI artifact"), - ).toBeLessThan(requiredStepIndex(sharedActions.cliCoverageShard, "Run CLI coverage shard")); - - expect(shardUploadStep.if).toBe( - "${{ always() && steps.validate-shard-inputs.outcome == 'success' }}", - ); - expect(shardUploadStep.uses).toContain("actions/upload-artifact@"); - expect(shardUploadStep.with?.name).toBe("cli-blob-report-${{ inputs.shard }}"); - expect(shardUploadStep.with?.path).toBe( - ".vitest-reports/blob-${{ inputs.shard }}-${{ inputs.shard-count }}.json", - ); - expect(shardUploadStep.with?.["if-no-files-found"]).toBe("error"); - expect(shardUploadStep.with?.["retention-days"]).toBe(1); - - expect(compiledCliDownloadStep.uses).toContain("actions/download-artifact@"); - expect(compiledCliDownloadStep.with).toEqual({ - name: "cli-build-output", - path: "dist", - }); - expect( - requiredStepIndex(sharedActions.cliCoverageMerge, "Download compiled CLI artifact"), - ).toBeLessThan( - requiredStepIndex(sharedActions.cliCoverageMerge, "Verify compiled CLI artifact"), - ); - expect( - requiredStepIndex(sharedActions.cliCoverageMerge, "Verify compiled CLI artifact"), - ).toBeLessThan( - requiredStepIndex(sharedActions.cliCoverageMerge, "Download CLI shard blob reports"), - ); - - expect(downloadStep.uses).toContain("actions/download-artifact@"); - expect(downloadStep.with?.pattern).toBe("cli-blob-report-*"); - expect(downloadStep.with?.path).toBe(".vitest-reports"); - expect(downloadStep.with?.["merge-multiple"]).toBe(true); - - expect(verifyRun).toContain('seq 1 "$CLI_SHARD_COUNT"'); - expect(verifyRun).toContain('[ ! -s "$blob" ]'); - expect(verifyRun).toContain("Expected ${CLI_SHARD_COUNT} blob reports"); - expect(stepRuns(sharedActions.cliCoverageMerge).join("\n")).toContain( - 'scripts/check-coverage-ratchet.ts coverage/cli/coverage-summary.json ci/coverage-threshold-cli.json "CLI coverage"', - ); - }); - - it("selects an available shard to publish the compiled CLI artifact", () => { - for (const shardCount of [1, 2, 3, cliShardMatrix.length]) { - const expectedProducer = Math.min(4, shardCount); - const producers = Array.from({ length: shardCount }, (_, index) => index + 1).filter( - (shard) => uploadsCompiledCliArtifact(sharedActions.cliCoverageShard, shard, shardCount), - ); - - expect(producers, `${shardCount} total shards`).toEqual([expectedProducer]); } }); - it("keeps final aggregate checks for PR and main workflows", () => { + it("accepts successful aggregate checks and rejects failed required lanes", () => { const prChecks = prWorkflow.jobs.checks; - const prChecksRun = stepRuns(prChecks).join("\n"); const mainChecks = mainWorkflow.jobs.checks; - const mainChecksRun = stepRuns(mainChecks).join("\n"); - - expect(prChecks.if).toBe("always()"); - expect(prChecks.needs).toEqual([ - "changes", - "docs-only-checks", - "static-checks", - "build-typecheck", - "installer-integration", - "cli-tests", - "plugin-tests", - "e2e-support", - "test-e2e-ollama-proxy", - ]); - expect(prWorkflow.jobs["cli-tests"].needs).toEqual(["changes", "cli-test-shards"]); - - for (const jobName of [ - "changes", - "static-checks", - "build-typecheck", - "installer-integration", - "cli-tests", - "plugin-tests", - "e2e-support", - "test-e2e-ollama-proxy", - ]) { - expect(prChecksRun).toContain(`require_success "${jobName}"`); - } - expect(prChecksRun).toContain('require_success "docs-only-checks"'); - - expect(mainChecks.if).toBe("always()"); - expect(mainChecks.needs).toEqual([ - "static-checks", - "build-typecheck", - "installer-integration", - "real-openclaw-dist-harness", - "cli-tests", - "plugin-tests", - "e2e-support", - "test-e2e-ollama-proxy", - ]); - expect(mainWorkflow.jobs["cli-tests"].needs).toBe("cli-test-shards"); - for (const jobName of [ - "static-checks", - "build-typecheck", - "installer-integration", - "real-openclaw-dist-harness", - "cli-tests", - "plugin-tests", - "e2e-support", - "test-e2e-ollama-proxy", - ]) { - expect(mainChecksRun).toContain(`require_success "${jobName}"`); - } - expect(mainWorkflow.jobs["sandbox-images-and-e2e"].needs).toBe("checks"); - }); + const prGate = requiredWorkflowStep(prChecks, "Verify required PR checks"); + const mainGate = requiredWorkflowStep(mainChecks, "Verify required main checks"); + const successfulCode = { + BUILD_TYPECHECK_RESULT: "success", + CHANGES_RESULT: "success", + CLI_TESTS_RESULT: "success", + CODE_CHANGED: "true", + DOCS_ONLY_RESULT: "skipped", + E2E_PROXY_RESULT: "success", + E2E_SUPPORT_RESULT: "success", + INSTALLER_INTEGRATION_RESULT: "success", + PLUGIN_TESTS_RESULT: "success", + STATIC_RESULT: "success", + }; + const successfulMain = { + BUILD_TYPECHECK_RESULT: "success", + CLI_TESTS_RESULT: "success", + E2E_PROXY_RESULT: "success", + E2E_SUPPORT_RESULT: "success", + INSTALLER_INTEGRATION_RESULT: "success", + PLUGIN_TESTS_RESULT: "success", + REAL_OPENCLAW_DIST_HARNESS_RESULT: "success", + STATIC_RESULT: "success", + }; + + const codeSuccess = runWorkflowShellStep(prGate, successfulCode); + const codeFailure = runWorkflowShellStep(prGate, { + ...successfulCode, + STATIC_RESULT: "failure", + }); + const docsOnlySuccess = runWorkflowShellStep(prGate, { + ...successfulCode, + BUILD_TYPECHECK_RESULT: "skipped", + CLI_TESTS_RESULT: "skipped", + CODE_CHANGED: "false", + DOCS_ONLY_RESULT: "success", + E2E_PROXY_RESULT: "skipped", + E2E_SUPPORT_RESULT: "skipped", + INSTALLER_INTEGRATION_RESULT: "skipped", + PLUGIN_TESTS_RESULT: "skipped", + STATIC_RESULT: "skipped", + }); + const mainSuccess = runWorkflowShellStep(mainGate, successfulMain); + const mainFailure = runWorkflowShellStep(mainGate, { + ...successfulMain, + REAL_OPENCLAW_DIST_HARNESS_RESULT: "failure", + }); - it("exports immutable GHCR digests from the Hermes base resolver", () => { - const runs = stepRuns(resolveHermesBaseAction).join("\n"); - - expect(runs).toContain("docker image inspect"); - expect(runs).toContain("${image}@sha256:"); - expect(runs).toContain("mcp_client_imports_ok"); - expect(runs).toContain("Build-time package/import guard only"); - expect(runs).toContain("_MCP_HTTP_AVAILABLE"); - expect(runs).toContain("layout_ok"); - expect(runs).toContain("mapfile -t tracked_refs"); - expect(runs).toContain('candidates=("$tracked_ref")'); - expect(runs).toContain("HERMES_BASE_IMAGE=${digest_ref}"); - expect(runs).toContain("HERMES_BASE_IMAGE=nemoclaw-hermes-base-local"); + expect(codeSuccess.status).toBe(0); + expect(codeFailure.status).not.toBe(0); + expect(codeFailure.stdout).toContain("static-checks failed"); + expect(docsOnlySuccess.status).toBe(0); + expect(mainSuccess.status).toBe(0); + expect(mainFailure.status).not.toBe(0); + expect(mainFailure.stdout).toContain("real-openclaw-dist-harness failed"); }); it("rejects a pulled Hermes base without MCP HTTP imports and falls back locally", () => { @@ -1441,6 +1128,7 @@ npx tsx scripts/checks/e2e-mock-parity.ts --base "$BASE_SHA" --head HEAD } }); + // source-shape-contract: security -- CI dependency installs must never execute package lifecycle scripts from fetched code it("does not run npm lifecycle scripts during CI dependency installs", () => { for (const [actionName, action] of Object.entries(sharedActions)) { const installRuns = stepRuns(action).filter((run) => run.includes("npm install")); @@ -1466,6 +1154,7 @@ npx tsx scripts/checks/e2e-mock-parity.ts --base "$BASE_SHA" --head HEAD expect(installerBootstrapInstall).toContain("cd nemoclaw && npm install --ignore-scripts"); }); + // source-shape-contract: security -- Workflow checkouts must not leave write-capable credentials available to later steps it("does not persist checkout credentials in PR or main jobs", () => { for (const [workflowName, workflow] of [ ["pull_request", prWorkflow], diff --git a/test/regression-e2e-workflow.test.ts b/test/regression-e2e-workflow.test.ts index 20fa490ac80..3885c381fa4 100644 --- a/test/regression-e2e-workflow.test.ts +++ b/test/regression-e2e-workflow.test.ts @@ -43,31 +43,7 @@ function preparedVitestJobs(workflow: RegressionWorkflow) { describe("Regression E2E workflow contract", () => { const workflow = readYaml(".github/workflows/regression-e2e.yaml"); - it.each([ - ["docker-unreachable-gateway-start-e2e", "docker_unreachable_gateway_start"], - ["onboard-inference-smoke-e2e", "onboard_inference_smoke"], - ])("does not advertise or select retired lane %s", (jobName, selectorOutput) => { - const jobsDescription = workflow.on?.workflow_dispatch?.inputs?.jobs?.description ?? ""; - const selectorScript = - workflow.jobs?.select_regression_jobs?.steps?.find((step) => step.id === "select")?.run ?? ""; - - expect(jobsDescription).not.toContain(jobName); - expect(Object.keys(workflow.jobs ?? {})).not.toContain(jobName); - expect(selectorScript).not.toContain(jobName); - expect(selectorScript).not.toContain(selectorOutput); - }); - - it("does not advertise or select the retired strict-tool-call-probe lane", () => { - const jobsDescription = workflow.on?.workflow_dispatch?.inputs?.jobs?.description ?? ""; - const selectorScript = - workflow.jobs?.select_regression_jobs?.steps?.find((step) => step.id === "select")?.run ?? ""; - - expect(jobsDescription).not.toContain("strict-tool-call-probe-e2e"); - expect(Object.keys(workflow.jobs ?? {})).not.toContain("strict-tool-call-probe-e2e"); - expect(selectorScript).not.toContain("strict-tool-call-probe-e2e"); - expect(selectorScript).not.toContain("strict_tool_call_probe"); - }); - + // source-shape-contract: compatibility -- Keeps the executable WhatsApp regression on the supported Vitest live runner it("runs WhatsApp compact QR through Vitest instead of the retired shell script", () => { const job = workflow.jobs?.["whatsapp-qr-compact-e2e"]; const runText = (job?.steps ?? []).map((step) => step.run ?? "").join("\n"); @@ -76,6 +52,7 @@ describe("Regression E2E workflow contract", () => { expect(runText).toContain("npx vitest run --project e2e-live"); }); + // source-shape-contract: security -- Preserves the public NVIDIA credential boundary for Model Router regression execution it("stages the public NVIDIA key for the Model Router's NVIDIA credential", () => { const job = workflow.jobs?.["model-router-provider-routed-inference-e2e"]; const runStep = job?.steps?.find( @@ -85,6 +62,7 @@ describe("Regression E2E workflow contract", () => { expect(runStep?.env?.NVIDIA_INFERENCE_API_KEY).toBeUndefined(); }); + // source-shape-contract: security -- Every discovered non-hermetic Vitest job must use immutable credential-free preparation it("prepares every discovered non-hermetic Vitest job before execution (#6692)", () => { const preparedJobs = preparedVitestJobs(workflow); @@ -128,6 +106,7 @@ describe("Regression E2E workflow contract", () => { } }); + // source-shape-contract: compatibility -- Keeps the gateway drift regression in its stateful integration execution lane it("collects the gateway drift regression from its integration project (#6692)", () => { const job = workflow.jobs?.["gateway-drift-preflight-e2e"]; const runStep = job?.steps?.find( @@ -140,6 +119,7 @@ describe("Regression E2E workflow contract", () => { expect(runStep?.run).not.toContain("vitest run --project cli"); }); + // source-shape-contract: security -- Keeps the custom-plugin EXDEV regression immutable and free of repository secrets it("runs the OpenClaw custom-plugin lifecycle and EXDEV guard in a secret-free lane", () => { const job = workflow.jobs?.["openclaw-plugin-runtime-exdev-e2e"]; const steps = job?.steps ?? []; diff --git a/test/repro-4538-raw-doctor-perms.test.ts b/test/repro-4538-raw-doctor-perms.test.ts index 471548871e4..dd3b4d57956 100644 --- a/test/repro-4538-raw-doctor-perms.test.ts +++ b/test/repro-4538-raw-doctor-perms.test.ts @@ -141,6 +141,7 @@ function writeDoctorFixFake(tmpDir: string): string { describe("raw `openclaw doctor --fix` mutable-perm restore (#4538)", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); + // source-shape-contract: security -- Executes the shipped restore helper to verify hardened mutable ownership modes it("restore helper re-asserts 2770/660 after the tree is tightened to 700/600", () => { const { tmpDir, configDir, configFile } = seedTightenedConfigTree(); const nestedDir = path.join(configDir, "agents", "main"); @@ -209,6 +210,7 @@ describe("raw `openclaw doctor --fix` mutable-perm restore (#4538)", () => { } }); + // source-shape-contract: security -- Executes the shipped guard to preserve permissions after a failing doctor repair it("emitted openclaw() guard restores the contract AND preserves a nonzero exit", () => { const { tmpDir, configDir, configFile } = seedTightenedConfigTree(); // Start from the intact contract so the simulated doctor run is what @@ -252,6 +254,7 @@ describe("raw `openclaw doctor --fix` mutable-perm restore (#4538)", () => { } }); + // source-shape-contract: security -- Executes the shipped guard under errexit to prove restoration remains fail safe it("emitted openclaw() guard restores the contract even under an inherited `set -e`", () => { // Regression for the errexit gap: when the guard is sourced into a shell // with errexit on, a nonzero `doctor --fix` must not abort openclaw() diff --git a/test/repro-5978-policy-denial-hint.test.ts b/test/repro-5978-policy-denial-hint.test.ts index e7dbeeb0bfd..7a54b3bbef0 100644 --- a/test/repro-5978-policy-denial-hint.test.ts +++ b/test/repro-5978-policy-denial-hint.test.ts @@ -306,6 +306,7 @@ describe("sandbox policy-denial logs breadcrumb (#5978)", () => { expect(stdout).not.toContain("logs --tail 50"); }); + // source-shape-contract: compatibility -- Executing the emitted shell hook twice protects login profile and bashrc coexistence it("prints only once when the file is sourced twice in one login shell", () => { // A login shell sources both the system profile and bashrc hooks, each of // which sources this file and runs its trailing auto-invocation — the diff --git a/test/runner.test.ts b/test/runner.test.ts index 1f539db5bb9..992ef6f22cb 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -669,6 +669,7 @@ describe("regression guards", () => { }); describe("credential exposure guards (#429)", () => { + // source-shape-contract: security -- Executable walkthrough commands must never materialize the NVIDIA inference credential in child arguments it("walkthrough.sh does not embed NVIDIA_INFERENCE_API_KEY in tmux or sandbox commands", () => { const fs = require("fs"); const src = fs.readFileSync( diff --git a/test/source-shape-scanner.test.ts b/test/source-shape-scanner.test.ts index fb47de77bd5..410b0893ea8 100644 --- a/test/source-shape-scanner.test.ts +++ b/test/source-shape-scanner.test.ts @@ -3,7 +3,12 @@ import { describe, expect, it } from "vitest"; -import { scanTextForTest } from "../scripts/find-source-shape-tests"; +import { + contractExceptionAllowlistErrors, + scanTextForTest, + scanTextForTestReport, + sourceShapeSummary, +} from "../scripts/find-source-shape-tests"; function detectedCaseNames(source: string): string[] { return scanTextForTest("test/virtual-source-shape.test.ts", source).map((entry) => entry.name); @@ -110,4 +115,554 @@ describe("source-shape scanner", () => { expect(cases).toEqual([]); }); + + it("detects direct assertions on shipped declarative files", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import YAML from "yaml"; + import { expect, it } from "vitest"; + + it("mirrors blueprint keys", () => { + const config = YAML.parse(readFileSync("nemoclaw-blueprint/blueprint.yaml", "utf8")); expect(config.components).toHaveProperty("sandbox"); + }); + + it("mirrors an E2E manifest", () => { + const target = JSON.parse(readFileSync("test/e2e/targets/cloud.json", "utf8")); expect(target.requiredSecrets).toEqual(["NVIDIA_API_KEY"]); + }); + `); + + expect(cases).toEqual(["mirrors blueprint keys", "mirrors an E2E manifest"]); + }); + + it("detects Node assertions and source-derived expected arguments", () => { + const cases = detectedCaseNames(` + import assert from "node:assert/strict"; + import nodeAssert, { + deepStrictEqual as same, + strict as strictAssert, + } from "node:assert"; + import { readFileSync } from "node:fs"; + import { spawnSync } from "node:child_process"; + import { it } from "vitest"; + import { validateBlueprint } from "../src/lib/config-validator"; + + const cjsSame = require("node:assert").deepEqual; + const esmSame = nodeAssert.deepStrictEqual; + const { deepEqual: destructuredSame } = nodeAssert; + + it("uses deep equality", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + assert.deepStrictEqual(raw.scripts, { test: "vitest" }); + }); + + it("uses callable assert", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + assert(raw.private); + }); + + it("puts source data in the expected position", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + assert.equal("nemoclaw", raw.name); + }); + + it("uses an aliased default assertion", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + nodeAssert.deepEqual({}, raw.scripts); + }); + + it("uses an aliased named assertion", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + same({}, raw.scripts); + }); + + it("uses a strict namespace assertion", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + strictAssert.partialDeepStrictEqual({}, raw.scripts); + }); + + it("uses a nested strict assertion", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + nodeAssert.strict.deepStrictEqual({}, raw.scripts); + }); + + it("uses a property-extracted CommonJS assertion", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + cjsSame({}, raw.scripts); + }); + + it("uses a property-extracted ESM assertion", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + esmSame({}, raw.scripts); + }); + + it("uses a destructured assertion alias", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + destructuredSame({}, raw.scripts); + }); + + it("ignores a source-derived diagnostic message", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + assert.ok(true, \`package \${raw.name}\`); + }); + + it("asserts an execution result", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + const result = spawnSync("tool", [raw.name]); + assert.equal(result.stdout, raw.name); + }); + + it("asserts production consumer behavior", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + assert.equal(validateBlueprint(raw).ok, raw.private); + }); + `); + + expect(cases).toEqual([ + "uses deep equality", + "uses callable assert", + "puts source data in the expected position", + "uses an aliased default assertion", + "uses an aliased named assertion", + "uses a strict namespace assertion", + "uses a nested strict assertion", + "uses a property-extracted CommonJS assertion", + "uses a property-extracted ESM assertion", + "uses a destructured assertion alias", + ]); + }); + + it("detects repeated parent traversal from co-located tests", () => { + const cases = scanTextForTest( + "src/lib/actions/virtual-source-shape.test.ts", + ` + import { readFileSync } from "node:fs"; + import { expect, it } from "vitest"; + + it("reads a deep config path", () => { + const raw = readFileSync("../../../nemoclaw-blueprint/blueprint.yaml", "utf8"); + expect(raw).toContain("version:"); + }); + + it("reads a deep root file", () => { + const raw = JSON.parse(readFileSync("../../../package.json", "utf8")); + expect(raw.scripts).toHaveProperty("test"); + }); + `, + ).map((entry) => entry.name); + + expect(cases).toEqual(["reads a deep config path", "reads a deep root file"]); + }); + + it("tracks source assignments from applicable setup hooks only", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import { beforeAll as setupAll, beforeEach, describe, expect, it } from "vitest"; + + let globalRaw; + function loadGlobalConfig() { + globalRaw = JSON.parse(readFileSync("package.json", "utf8")); + } + const aliasedLoadGlobalConfig = loadGlobalConfig; + setupAll(aliasedLoadGlobalConfig); + + it("uses top-level setup data", () => expect(globalRaw.name).toBe("nemoclaw")); + + describe("one suite", () => { + let suiteRaw; + beforeEach(() => { + suiteRaw = JSON.parse(readFileSync("package.json", "utf8")); + }); + it("uses suite setup data", () => expect(suiteRaw.private).toBe(true)); + }); + + describe("sibling suite", () => { + it("does not inherit a sibling hook", () => { + const suiteRaw = { private: true }; + expect(suiteRaw.private).toBe(true); + }); + }); + `); + + expect(cases).toEqual(["uses top-level setup data", "uses suite setup data"]); + }); + + it("detects CommonJS and dynamic declarative imports", () => { + const cases = detectedCaseNames(` + import { expect, it } from "vitest"; + + it("requires package metadata", () => { + const raw = require("../package.json"); + expect(raw.scripts).toHaveProperty("test"); + }); + + it("dynamically imports package metadata", async () => { + const raw = (await import("../package.json", { with: { type: "json" } })).default; + expect(raw.name).toBe("nemoclaw"); + }); + + it("chains a dynamic package import", async () => { + const raw = await import("../package.json").then((module) => module.default); + expect(raw.name).toBe("nemoclaw"); + }); + + it("directly requires package metadata", () => { + expect(require("../package.json").scripts).toHaveProperty("test"); + }); + + it("directly imports package metadata", async () => { + expect((await import("../package.json")).default.private).toBe(true); + }); + + it("ignores a required fixture", () => { + expect(require("./fixtures/package.json").name).toBe("fixture"); + }); + `); + + expect(cases).toEqual([ + "requires package metadata", + "dynamically imports package metadata", + "chains a dynamic package import", + "directly requires package metadata", + "directly imports package metadata", + ]); + }); + + it("preserves source references inside template interpolation", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import { expect, it } from "vitest"; + + it("formats a raw config field", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + expect(\`package=\${raw.name}\`).toBe("package=nemoclaw"); + }); + + it("keeps interpolation after comment and regex braces", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + expect(\`package=\${/* } */ /}/.test("}") ? raw.name : ""}\`).toBe("package=nemoclaw"); + }); + + it("ignores a variable name in template text", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + expect(\`the word raw is not an interpolation\`).toBe("the word raw is not an interpolation"); + }); + `); + + expect(cases).toEqual([ + "formats a raw config field", + "keeps interpolation after comment and regex braces", + ]); + }); + + it("detects declarative imports including Vitest config", () => { + const cases = detectedCaseNames(` + import target from "../test/e2e/targets/cloud.json"; + import vitestConfig from "../vitest.config"; + import { expect, it } from "vitest"; + + it("mirrors imported target keys", () => expect(target.requiredSecrets).toEqual(["NVIDIA_API_KEY"])); + it("mirrors project names", () => expect(vitestConfig.test.projects).toEqual(["cli", "integration"])); + `); + + expect(cases).toEqual(["mirrors imported target keys", "mirrors project names"]); + }); + + it("keeps true fixtures and E2E executable source out of declarative findings", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import YAML from "yaml"; + import { expect, it } from "vitest"; + + it("checks a fixture", () => { + const config = YAML.parse(readFileSync("test/fixtures/config.yaml", "utf8")); expect(config.mode).toBe("fixture"); + }); + + it("checks E2E source", () => { + const source = readFileSync("test/e2e/live/example.test.ts", "utf8"); expect(source).toContain("runLiveProbe"); + }); + `); + + expect(cases).toEqual([]); + }); + + it("exempts only values rooted in production consumers", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import YAML from "yaml"; + import { expect, it } from "vitest"; + import { validateBlueprint } from "../src/lib/config-validator"; + + it("asserts validator behavior", () => { + const raw = YAML.parse(readFileSync("nemoclaw-blueprint/blueprint.yaml", "utf8")); + const result = validateBlueprint(raw); + expect(result.ok).toBe(true); + expect(validateBlueprint(raw).errors).toEqual([]); + }); + + it("still catches nested raw shape", () => { + const raw = YAML.parse(readFileSync("nemoclaw-blueprint/blueprint.yaml", "utf8")); + const combined = { raw, checked: validateBlueprint(raw) }; + expect(combined.raw.components).toHaveProperty("sandbox"); + }); + `); + + expect(cases).toEqual(["still catches nested raw shape"]); + }); + + it("detects explicit raw-config accessors and local selectors", () => { + const cases = detectedCaseNames(` + import { expect, it } from "vitest"; + import { readWorkflow } from "./helpers/e2e-workflow-contract"; + import { listTargets } from "./e2e/registry/registry"; + + it("mirrors workflow jobs through a selector", () => { + const workflow = readWorkflow(); + function sortedJobNames() { return jobNames().sort(); } + function jobNames() { return Object.keys(workflow.jobs); } + expect(sortedJobNames()).toEqual(["build", "test"]); + }); + + it("mirrors registry targets directly", () => + expect(listTargets().map((target) => target.id)).toEqual(["local", "cloud"])); + + it("detects a selector that wraps an accessor", () => { + function targetIds() { return listTargets().map((target) => target.id); } + expect(targetIds()).toEqual(["local", "cloud"]); + }); + + it("detects an accessor element alias", () => { + const first = listTargets()[0]; expect(first.id).toBe("local"); + }); + + it("detects a nested accessor initializer", () => { + const jobs = Object.keys(readWorkflow().jobs); expect(jobs).toEqual(["test", "build"]); + }); + + it("detects destructured raw config", () => { + const { jobs } = readWorkflow(); expect(Object.keys(jobs)).toEqual(["test", "build"]); + }); + + it("detects an accumulator fed by registry entries", () => { + const ids = []; + for (const target of listTargets()) ids.push(target.id); + expect(ids).toEqual(["local", "cloud"]); + }); + + it("detects an assignment accumulator", () => { + let selected; selected = listTargets()[0]; expect(selected.id).toBe("local"); + }); + + it("only checks the helper binding", () => expect(readWorkflow).toBeTypeOf("function")); + `); + + expect(cases).toEqual([ + "mirrors workflow jobs through a selector", + "mirrors registry targets directly", + "detects a selector that wraps an accessor", + "detects an accessor element alias", + "detects a nested accessor initializer", + "detects destructured raw config", + "detects an accumulator fed by registry entries", + "detects an assignment accumulator", + ]); + }); + + it("tracks namespace accessors but not derived registry and manifest helpers", () => { + const cases = detectedCaseNames(` + import { expect, it } from "vitest"; + import * as workflows from "./helpers/e2e-workflow-contract"; + import * as registry from "./e2e/registry/registry"; + import { probesForState } from "./e2e/registry/expected-states"; + import { loadManifest, loadManifestsFromDir } from "./e2e/registry/manifests"; + + it("mirrors a namespace-loaded workflow", () => { + expect(Object.keys(workflows.readWorkflow().jobs)).toEqual(["test", "build"]); + expect(registry.listTargets().map((target) => target.id)).toEqual(["local", "cloud"]); + }); + + it("asserts derived helper behavior", () => { + const fixture = workflows.readYaml("test/fixtures/workflow.yaml"); expect(fixture.name).toBe("fixture"); + expect(probesForState({ probes: ["ready"] })).toEqual(["ready"]); + expect(loadManifest("test/fixtures/target.yaml").valid).toBe(true); + expect(loadManifestsFromDir("test/fixtures")).toHaveLength(1); + }); + `); + + expect(cases).toEqual(["mirrors a namespace-loaded workflow"]); + }); + + it("does not taint spawn results when raw config feeds the command", () => { + const cases = detectedCaseNames(` + import { spawnSync } from "node:child_process"; + import { expect, it } from "vitest"; + import { readWorkflow } from "./helpers/e2e-workflow-contract"; + + it("asserts executed behavior", () => { + const workflow = readWorkflow(); + const result = spawnSync("workflow-check", [workflow.name], { encoding: "utf8" }); + expect(result.stdout).toContain(workflow.name); + }); + `); + + expect(cases).toEqual([]); + }); + + it("does not taint execution of a program extracted from config", () => { + const cases = detectedCaseNames(` + import { expect, it } from "vitest"; + import { readWorkflow } from "./helpers/e2e-workflow-contract"; + + const DynamicFunction = Object.getPrototypeOf(async () => undefined).constructor; + const workflow = readWorkflow(); + const script = workflow.jobs.test.steps[0].with.script; + async function runScript() { + await new DynamicFunction("input", script)("fixture"); + } + + it("asserts executed workflow behavior", async () => { + await expect(runScript()).resolves.toBeUndefined(); + }); + `); + + expect(cases).toEqual([]); + }); + + it("does not treat an AsyncFunction property name as execution", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import { expect, it } from "vitest"; + + it("keeps raw data tainted", () => { + const raw = JSON.parse(readFileSync("package.json", "utf8")); + const shape = { AsyncFunction: true, scripts: raw.scripts }; + expect(shape.scripts).toHaveProperty("test"); + }); + `); + + expect(cases).toEqual(["keeps raw data tainted"]); + }); + + it("scopes a valid contract exception to the immediately following finding", () => { + const report = scanTextForTestReport( + "test/virtual-source-shape.test.ts", + ` + import { readFileSync } from "node:fs"; + import YAML from "yaml"; + import { expect, it } from "vitest"; + + // source-shape-contract: security -- Cross-field digest equality protects the shipped trust anchor + it("protects an integrity anchor", () => { + const config = YAML.parse(readFileSync("nemoclaw-blueprint/blueprint.yaml", "utf8")); + expect(config.digest).toBe(config.components.sandbox.digest); + }); + + it("still detects the next mirror", () => { + const config = YAML.parse(readFileSync("nemoclaw-blueprint/blueprint.yaml", "utf8")); + expect(config.components).toHaveProperty("sandbox"); + }); + `, + ); + + expect(report.contractExceptions.map((entry) => entry.name)).toEqual([ + "protects an integrity anchor", + ]); + expect(report.cases.map((entry) => entry.name)).toEqual(["still detects the next mirror"]); + expect(report.invalidContractExceptions).toEqual([]); + expect(sourceShapeSummary(report)).toMatchObject({ + source_shape_contract_exceptions: 1, + source_shape_invalid_contract_exceptions: 0, + }); + }); + + it("recognizes contract annotations after template literals", () => { + const report = scanTextForTestReport( + "test/virtual-source-shape.test.ts", + ` + import { readFileSync } from "node:fs"; + import YAML from "yaml"; + import { expect, it } from "vitest"; + + const fixture = \`template literal before the annotation\`; + + // source-shape-contract: compatibility -- Exact legacy keys preserve the supported serialized interface + it("protects a serialized compatibility contract", () => { + const config = YAML.parse(readFileSync("nemoclaw-blueprint/blueprint.yaml", "utf8")); + expect(config.legacyKey).toBe(fixture); + }); + `, + ); + + expect(report.contractExceptions.map((entry) => entry.name)).toEqual([ + "protects a serialized compatibility contract", + ]); + expect(report.cases).toEqual([]); + expect(report.invalidContractExceptions).toEqual([]); + }); + + it("rejects unsupported, short, and misplaced contract exceptions", () => { + const source = (annotation: string, separator = "") => ` + import { readFileSync } from "node:fs"; + import { expect, it } from "vitest"; + + ${annotation} + ${separator} + it("mirrors a workflow", () => { + const workflow = readFileSync(".github/workflows/ci.yml", "utf8"); + expect(workflow).toContain("pull_request:"); + }); + `; + + for (const [annotation, separator, reason] of [ + [ + "// source-shape-contract: snapshot -- This reason is sufficiently detailed", + "", + "unsupported category", + ], + ["// source-shape-contract: security -- Trust anchor", "", "reason is too short"], + [ + "// source-shape-contract: security -- This reason is sufficiently detailed", + "// not adjacent", + "immediately above", + ], + ]) { + const report = scanTextForTestReport( + "test/virtual-source-shape.test.ts", + source(annotation ?? "", separator), + ); + expect(report.invalidContractExceptions[0]?.reason).toContain(reason); + expect(report.cases).toHaveLength(1); + expect(sourceShapeSummary(report).source_shape_invalid_contract_exceptions).toBe(1); + } + }); + + it("rejects replacing an allowlisted exception with a same-count exception", () => { + const allowed = [ + { file: "test/integrity.test.ts", test: "protects digest", category: "security" as const }, + ]; + expect( + contractExceptionAllowlistErrors( + [{ file: "test/integrity.test.ts", name: "protects digest", category: "security" }], + allowed, + ), + ).toEqual([]); + + const errors = contractExceptionAllowlistErrors( + [{ file: "test/integrity.test.ts", name: "mirrors keys", category: "security" }], + allowed, + ); + expect(errors).toEqual([ + expect.stringContaining("unapproved source-shape exception"), + expect.stringContaining("unused source-shape exception allowance"), + ]); + + expect( + contractExceptionAllowlistErrors( + [ + { file: "test/integrity.test.ts", name: "protects digest", category: "security" }, + { file: "test/integrity.test.ts", name: "protects digest", category: "security" }, + ], + allowed, + ), + ).toEqual([expect.stringContaining("duplicate source-shape exception identity")]); + }); }); diff --git a/test/test-boundary-guards.test.ts b/test/test-boundary-guards.test.ts index f78e17f8e38..3237fbf5b9d 100644 --- a/test/test-boundary-guards.test.ts +++ b/test/test-boundary-guards.test.ts @@ -691,6 +691,7 @@ describe("Vitest project membership boundary", () => { ["test/vitest-coverage-thresholds.test.ts", "integration"], ["test/example.test.js", "integration"], ["test/install-build-dependency-preflight.test.ts", "installer-integration"], + ["test/install-clone-ref.test.ts", "installer-integration"], ["test/install-express-prompt.test.ts", "installer-integration"], ["test/install-openshell-version-check.test.ts", "installer-integration"], ["test/install-preflight-docker-bootstrap.test.ts", "installer-integration"], diff --git a/test/validate-blueprint.test.ts b/test/validate-blueprint.test.ts index 5f0844bf4a8..d8420dd3670 100644 --- a/test/validate-blueprint.test.ts +++ b/test/validate-blueprint.test.ts @@ -2,57 +2,31 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Protect security and routing semantics declared by the shipping blueprint, - * provider profiles, and policy files. Structural validation belongs to + * Protect the blueprint image trust anchor and the effective sandbox policies + * that NemoClaw submits after its production create/merge path consumes the + * checked-in policy sources. Structural validation belongs to * scripts/validate-configs.ts. */ import { readFileSync } from "node:fs"; + import { describe, expect, it } from "vitest"; import YAML from "yaml"; +import { prepareInitialSandboxCreatePolicy } from "../src/lib/onboard/initial-policy"; +import * as policies from "../src/lib/policy"; + const BLUEPRINT_PATH = new URL("../nemoclaw-blueprint/blueprint.yaml", import.meta.url); -const ROUTER_POOL_CONFIG_PATH = new URL( - "../nemoclaw-blueprint/router/pool-config.yaml", - import.meta.url, -); const BASE_POLICY_PATH = new URL( "../nemoclaw-blueprint/policies/openclaw-sandbox.yaml", import.meta.url, ); -const BRAVE_PROVIDER_PROFILE_PATH = new URL( - "../nemoclaw-blueprint/provider-profiles/brave.yaml", - import.meta.url, -); -const TAVILY_PROVIDER_PROFILE_PATH = new URL( - "../nemoclaw-blueprint/provider-profiles/tavily.yaml", - import.meta.url, -); -const TAVILY_PROVIDER_PROFILE_FOR_HERMES_PATH = new URL( - "../nemoclaw-blueprint/provider-profiles/tavily-hermes-v1.yaml", - import.meta.url, -); -const TAVILY_POLICY_PRESET_PATH = new URL( - "../nemoclaw-blueprint/policies/presets/tavily.yaml", - import.meta.url, -); -const DEEPAGENTS_POLICY_PATH = new URL( - "../agents/langchain-deepagents-code/policy-additions.yaml", - import.meta.url, -); const PERMISSIVE_POLICY_PATH = new URL( "../nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml", import.meta.url, ); const HERMES_POLICY_PATH = new URL("../agents/hermes/policy-additions.yaml", import.meta.url); -const hermesPermissivePolicyPath = new URL( - "../agents/hermes/policy-permissive.yaml", - import.meta.url, -); -const OPENCLAW_PERMISSIVE_POLICY_PATH = new URL( - "../agents/openclaw/policy-permissive.yaml", - import.meta.url, -); + type Blueprint = { digest?: string; components?: { @@ -60,16 +34,6 @@ type Blueprint = { }; }; -type RouterPoolModel = { - name?: string; - litellm_model?: string; - api_base?: string; -}; - -type RouterPoolConfig = { - models?: RouterPoolModel[]; -}; - type Rule = { allow?: { method?: string; path?: string } }; type Endpoint = { host?: string; @@ -79,800 +43,267 @@ type Endpoint = { access?: string; tls?: string; allow_encoded_slash?: boolean; - websocket_credential_rewrite?: boolean; - request_body_credential_rewrite?: boolean; rules?: Rule[]; - binaries?: Array<{ path: string }>; }; - type PolicyEntry = { - name?: string; endpoints?: Endpoint[]; - binaries?: Array<{ path: string }>; + binaries?: Array<{ path?: string }>; }; - type SandboxPolicy = { - version?: number; - filesystem_policy?: { read_only?: string[] }; network_policies?: Record; }; -type PolicyPreset = { - preset?: { name?: string; description?: string }; - network_policies?: Record; -}; +function loadYaml(path: URL): T { + return YAML.parse(readFileSync(path, "utf-8")); +} -type ProviderProfileCredential = { - env_vars?: string[]; - auth_style?: string; - header_name?: string; -}; +function parseEffectivePolicy(policy: string): SandboxPolicy { + return YAML.parse(policy) as SandboxPolicy; +} -type ProviderProfileEndpoint = { - host?: string; - port?: number; - protocol?: string; - access?: string; - enforcement?: string; - request_body_credential_rewrite?: boolean; - rules?: Rule[]; -}; +function endpoint(policy: SandboxPolicy, policyName: string, host: string): Endpoint { + const candidate = policy.network_policies?.[policyName]?.endpoints?.find( + (entry) => entry.host === host, + ); + expect(candidate, `${policyName} must allow ${host}`).toBeDefined(); + return candidate ?? {}; +} -type ProviderProfile = { - id?: string; - credentials?: ProviderProfileCredential[]; - endpoints?: ProviderProfileEndpoint[]; - binaries?: string[]; -}; +function methods(candidate: Endpoint): string[] { + return (candidate.rules ?? []) + .map((rule) => rule.allow?.method) + .filter((method): method is string => typeof method === "string") + .sort(); +} -function loadYaml(path: URL): T { - return YAML.parse(readFileSync(path, "utf-8")); +function binaries(policy: SandboxPolicy, policyName: string): string[] { + return (policy.network_policies?.[policyName]?.binaries ?? []) + .map((binary) => binary.path) + .filter((binary): binary is string => typeof binary === "string") + .sort(); +} + +function allEndpoints(policy: SandboxPolicy): Endpoint[] { + return Object.values(policy.network_policies ?? {}).flatMap((entry) => entry.endpoints ?? []); } const bp = loadYaml(BLUEPRINT_PATH); -describe("blueprint.yaml", () => { +describe("blueprint image trust anchor", () => { + // source-shape-contract: security -- The immutable sandbox image digest is the executable supply-chain trust anchor it("pins the sandbox image by digest instead of a mutable tag (#1438)", () => { - // The blueprint MUST NOT pull a sandbox image by a mutable tag like - // ":latest" — a registry compromise or accidental force-push could - // silently swap the image. Pin via @sha256:... so the image cannot - // change without a corresponding blueprint update. const sandbox = bp.components?.sandbox; const image = typeof sandbox?.image === "string" ? sandbox.image : ""; + expect(image.length).toBeGreaterThan(0); expect(image).toContain("@sha256:"); - // Belt and braces: explicitly forbid the ":latest" tag form even if the - // image string has been rearranged. expect(image).not.toMatch(/:latest$/); expect(image).not.toMatch(/:latest@/); - // The digest itself must be a 64-hex sha256. - const digestMatch = image.match(/@sha256:([0-9a-f]{64})$/); - expect(digestMatch).not.toBeNull(); + expect(image.match(/@sha256:([0-9a-f]{64})$/)).not.toBeNull(); }); + // source-shape-contract: security -- Cross-field digest equality prevents the shipped sandbox trust anchor from drifting it("populates the top-level digest field with the image digest (#1438)", () => { - // The top-level `digest:` field at the top of blueprint.yaml is - // documented as "Computed at release time" and was empty on main, - // which left blueprint-level integrity unverifiable. Mirror the - // sandbox image manifest digest into the top-level field so any - // consumer can read a single field to know what's pinned, and so - // a future contributor can't bump one without bumping the other. const topLevelDigest = typeof bp.digest === "string" ? bp.digest : ""; - expect(topLevelDigest.length).toBeGreaterThan(0); - // Must be a sha256:<64-hex> string. - expect(topLevelDigest).toMatch(/^sha256:[0-9a-f]{64}$/); - - const sandbox = bp.components?.sandbox; - const image = typeof sandbox?.image === "string" ? sandbox.image : ""; + const image = + typeof bp.components?.sandbox?.image === "string" ? bp.components.sandbox.image : ""; const imageDigestMatch = image.match(/@sha256:([0-9a-f]{64})$/); - expect(imageDigestMatch).not.toBeNull(); - const imageDigest = `sha256:${imageDigestMatch?.[1] ?? ""}`; - - // The two digests must agree. If a future bump touches one but not - // the other, this assertion catches it before merge. - expect(topLevelDigest).toBe(imageDigest); - }); -}); - -describe("Model Router pool config", () => { - const pool = loadYaml(ROUTER_POOL_CONFIG_PATH); - it("routes NVIDIA API keys to the public NVIDIA inference endpoint (#3255)", () => { - const apiBases = new Set((pool.models ?? []).map((model) => model.api_base)); - expect(apiBases).toEqual(new Set(["https://integrate.api.nvidia.com/v1"])); - }); - - it("uses valid LiteLLM NVIDIA model identifiers (#3255)", () => { - const modelsByName = new Map( - (pool.models ?? []).map((model) => [model.name, model.litellm_model]), - ); - expect(modelsByName.get("nemotron-3-nano-reasoning")).toBe( - "openai/nvidia/nemotron-3-nano-30b-a3b", - ); - expect(modelsByName.get("nemotron-3-super")).toBe("openai/nvidia/nemotron-3-super-120b-a12b"); - for (const litellmModel of modelsByName.values()) { - expect(litellmModel).not.toMatch(/nvidia\/nvidia\//); - expect(litellmModel).not.toContain("Nemotron-3-Nano-30B-A3B"); - expect(litellmModel).not.toContain("nemotron-3-super-v3"); - } + expect(topLevelDigest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(imageDigestMatch).not.toBeNull(); + expect(topLevelDigest).toBe(`sha256:${imageDigestMatch?.[1] ?? ""}`); }); }); -describe("base sandbox policy", () => { - const policy = loadYaml(BASE_POLICY_PATH); - - it("no endpoint rule uses wildcard method", () => { - const np = policy.network_policies ?? {}; - const violations: string[] = []; - for (const [policyName, cfg] of Object.entries(np)) { - const endpoints = cfg.endpoints; - if (!endpoints) continue; - for (const ep of endpoints) { - const rules = ep.rules; - if (!rules) continue; - for (const rule of rules) { - const method = rule.allow?.method; - if (method === "*") { - violations.push(`${policyName} → ${ep.host}: method "*"`); +describe("effective sandbox policy behavior", () => { + it("keeps default OpenClaw egress least-privilege after create-policy preparation", () => { + const prepared = prepareInitialSandboxCreatePolicy(BASE_POLICY_PATH.pathname, [], { + agentName: "openclaw", + }); + try { + const consumed = policies.mergePresetNamesIntoPolicy( + readFileSync(prepared.policyPath, "utf-8"), + [], + { agent: "openclaw" }, + ); + const policy = parseEffectivePolicy(consumed.policy); + const networkPolicies = policy.network_policies ?? {}; + + expect(consumed.missingPresets).toEqual([]); + + for (const [policyName, entry] of Object.entries(networkPolicies)) { + for (const candidate of entry.endpoints ?? []) { + expect(methods(candidate), `${policyName}:${candidate.host}`).not.toContain("*"); + if ((candidate.rules ?? []).length > 0) { + expect(candidate, `${policyName}:${candidate.host}`).toMatchObject({ + protocol: "rest", + enforcement: "enforce", + }); } } } - } - expect(violations).toEqual([]); - }); - it("every endpoint with rules has protocol: rest and enforcement: enforce", () => { - const np = policy.network_policies ?? {}; - const violations: string[] = []; - for (const [policyName, cfg] of Object.entries(np)) { - const endpoints = cfg.endpoints; - if (!endpoints) continue; - for (const ep of endpoints) { - if (!ep.rules) continue; - if (ep.protocol !== "rest") { - violations.push(`${policyName} → ${ep.host}: missing protocol: rest`); - } - if (ep.enforcement !== "enforce") { - violations.push(`${policyName} → ${ep.host}: missing enforcement: enforce`); - } - } - } - expect(violations).toEqual([]); - }); - - it("allows NVIDIA embeddings on the NVIDIA inference host", () => { - const np = policy.network_policies ?? {}; - const endpoints = np.nvidia?.endpoints; - const missingHosts: string[] = []; - const host = "integrate.api.nvidia.com"; - const endpoint = endpoints?.find((entry) => entry.host === host); - const hasEmbeddingsRule = endpoint?.rules?.some( - (rule) => rule.allow?.method === "POST" && rule.allow?.path === "/v1/embeddings", - ); - if (!hasEmbeddingsRule) { - missingHosts.push(host); - } - expect(missingHosts).toEqual([]); - }); - - // Walk every endpoint in every network_policies entry and return the - // entries whose host matches `hostMatcher`. Used by the regressions below. - function findEndpoints(hostMatcher: (h: string) => boolean): Endpoint[] { - const out: Endpoint[] = []; - const np = policy.network_policies; - if (!np) return out; - for (const value of Object.values(np)) { - const endpoints = value.endpoints; - if (!Array.isArray(endpoints)) continue; - for (const ep of endpoints) { - if (typeof ep.host === "string" && hostMatcher(ep.host)) { - out.push(ep); - } - } - } - return out; - } - - it("does not expose sentry.io in the base policy by default (#1437)", () => { - const sentryEndpoints = findEndpoints((h) => h === "sentry.io"); - expect(sentryEndpoints).toEqual([]); - }); - - it("does not silently grant GitHub access in the base policy (#1583)", () => { - // Until #1583, github.com / api.github.com plus the git/gh - // binaries lived in network_policies and were therefore included - // in every sandbox regardless of user opt-in. The fix moves the - // entry into a discoverable preset (`presets/github.yaml`). This - // assertion blocks the regression where someone re-adds a github - // entry to the base policy and silently re-grants every sandbox - // unscoped GitHub access. - const np = policy.network_policies; - expect(np && "github" in np).toBe(false); - - // Belt and braces: also assert no endpoint in any base-policy - // entry references github.com or api.github.com, so the - // regression can't be smuggled in under a renamed key. - const githubHosts = findEndpoints((h) => h === "github.com" || h === "api.github.com"); - expect(githubHosts).toEqual([]); - }); - - it("allows inference.local:443 GET and POST in the managed_inference policy (#2663)", () => { - // inference.local is the OpenShell gateway's managed inference virtual - // hostname — the gateway proxies it to the configured provider (OpenAI, - // NVIDIA, etc.). Every sandbox uses this route regardless of provider. - // Without this entry the OpenShell proxy blocks url-fetch calls to - // https://inference.local/v1/... with "Blocked hostname or - // private/internal/special-use IP address", breaking all inference. - const np = policy.network_policies ?? {}; - expect(np.managed_inference).toBeDefined(); - const endpoints = np.managed_inference?.endpoints ?? []; - const inferenceEp = endpoints.find((ep) => ep.host === "inference.local"); - expect(inferenceEp).toBeDefined(); - expect(inferenceEp?.port).toBe(443); - const rules = inferenceEp?.rules ?? []; - const hasGet = rules.some( - (r) => r.allow?.method?.toUpperCase() === "GET" && r.allow?.path === "/**", - ); - const hasPost = rules.some( - (r) => r.allow?.method?.toUpperCase() === "POST" && r.allow?.path === "/**", - ); - expect(hasGet).toBe(true); - expect(hasPost).toBe(true); - }); - - it("allows openclaw and tool binaries in the managed_inference policy (#2663)", () => { - const np = policy.network_policies ?? {}; - const binaries = (np.managed_inference?.binaries ?? []).map((b) => b.path).sort(); - expect(binaries).toEqual([ - "/usr/bin/curl", - "/usr/bin/node", - "/usr/bin/python3", - "/usr/local/bin/node", - "/usr/local/bin/openclaw", - ]); - }); - - it("allows encoded scoped-package paths only on the ClawHub endpoint (#4104)", () => { - const np = policy.network_policies ?? {}; - const clawhubEndpoints = np.clawhub?.endpoints ?? []; - expect(clawhubEndpoints).toHaveLength(1); - expect(clawhubEndpoints[0]).toMatchObject({ - host: "clawhub.ai", - protocol: "rest", - enforcement: "enforce", - allow_encoded_slash: true, - }); - - const encodedSlashHosts = Object.values(np).flatMap((entry) => - (entry.endpoints ?? []) - .filter((endpoint) => endpoint.allow_encoded_slash === true) - .map((endpoint) => endpoint.host), - ); - expect(encodedSlashHosts).toEqual(["clawhub.ai"]); - }); + const nvidia = endpoint(policy, "nvidia", "integrate.api.nvidia.com"); + expect(nvidia.rules).toContainEqual({ allow: { method: "POST", path: "/v1/embeddings" } }); - it("does not reference the absent Claude CLI binary", () => { - const serialized = JSON.stringify(policy.network_policies ?? {}); - expect(serialized).not.toContain("/usr/local/bin/claude"); - }); - - it("does not silently grant Telegram access in the base policy (#2180)", () => { - // Until #1705 (later regressed by #1700 and re-surfaced in #2180), - // `api.telegram.org` plus a /usr/local/bin/node binary lived in the - // base network_policies, so every sandbox could call the Telegram - // Bot API regardless of whether the user selected the telegram - // messaging channel or policy preset. The fix keeps Telegram access - // inside `presets/telegram.yaml`. This assertion blocks a regression - // where someone re-adds a telegram entry to the base policy and - // silently re-grants every sandbox unscoped Telegram access. - const np = policy.network_policies as Record | undefined; - expect(np && typeof np === "object" && "telegram" in np).toBe(false); - - const telegramHosts = findEndpoints((h) => h === "api.telegram.org"); - expect(telegramHosts).toEqual([]); - }); - - it("does not silently grant Discord access in the base policy (#2180)", () => { - // Parallel to the Telegram regression above. Discord (discord.com, - // gateway.discord.gg, cdn.discordapp.com, media.discordapp.net) is - // the opt-in preset path, not baseline. Re-adding these endpoints - // to the base policy lets any sandbox reach Discord without the - // user having selected the discord messaging channel or preset. - const np = policy.network_policies as Record | undefined; - expect(np && typeof np === "object" && "discord" in np).toBe(false); - - const discordHosts = findEndpoints( - (h) => - h === "discord.com" || - h === "gateway.discord.gg" || - h === "*.discord.gg" || - h === "cdn.discordapp.com" || - h === "media.discordapp.net", - ); - expect(discordHosts).toEqual([]); - }); - - it("does not silently grant Slack access in the base policy (#2180)", () => { - // Slack was never in the baseline, but guard against it being added - // in the same merge-conflict-resolution pattern that re-added - // Telegram and Discord after #1705. Slack access is in - // presets/slack.yaml only. - const np = policy.network_policies as Record | undefined; - expect(np && typeof np === "object" && "slack" in np).toBe(false); - - const slackHosts = findEndpoints( - (h) => - h === "slack.com" || - h.endsWith(".slack.com") || - h === "wss-primary.slack.com" || - h === "wss-backup.slack.com", - ); - expect(slackHosts).toEqual([]); - }); - - it("omits npm and node binaries from the baseline npm_registry policy (#1458)", () => { - const np = policy.network_policies ?? {}; - const npmRegistry = np.npm_registry; - expect(npmRegistry).toBeDefined(); - const binaries = npmRegistry?.binaries; - expect(Array.isArray(binaries)).toBe(true); - const paths = (binaries ?? []).map((b) => b.path).sort(); - // Only openclaw CLI should reach the npm registry by default. - // npm/node being in this list lets the agent bypass 'none' policy preset. - // Exact allowlist — adding any binary here requires a deliberate review. - expect(paths).toEqual(["/usr/local/bin/openclaw"]); - }); -}); - -describe("Brave Search provider profile", () => { - const profile = loadYaml(BRAVE_PROVIDER_PROFILE_PATH); - - it("routes BRAVE_API_KEY through Brave's subscription-token header", () => { - expect(profile.id).toBe("brave"); - expect(profile.credentials).toEqual([ - expect.objectContaining({ - env_vars: ["BRAVE_API_KEY"], - auth_style: "header", - header_name: "x-subscription-token", - }), - ]); - }); - - it("matches the Brave Search API endpoint used by the policy preset", () => { - expect(profile.endpoints).toEqual([ - expect.objectContaining({ - host: "api.search.brave.com", + const managedInference = endpoint(policy, "managed_inference", "inference.local"); + expect(managedInference).toMatchObject({ port: 443, protocol: "rest", - access: "read-write", enforcement: "enforce", - }), - ]); - }); -}); - -describe("Tavily Search provider profile", () => { - const profile = loadYaml(TAVILY_PROVIDER_PROFILE_PATH); - const hermesProfile = loadYaml(TAVILY_PROVIDER_PROFILE_FOR_HERMES_PATH); - const preset = loadYaml(TAVILY_POLICY_PRESET_PATH); - const deepAgentsPolicy = loadYaml(DEEPAGENTS_POLICY_PATH); - const defaultOpenClawPermissivePolicy = loadYaml(PERMISSIVE_POLICY_PATH); - const hermesPermissivePolicy = loadYaml(hermesPermissivePolicyPath); - const openClawPermissivePolicy = loadYaml(OPENCLAW_PERMISSIVE_POLICY_PATH); - - it("routes TAVILY_API_KEY through a bearer authorization header", () => { - expect(profile.id).toBe("tavily"); - expect(profile.credentials).toEqual([ - expect.objectContaining({ - env_vars: ["TAVILY_API_KEY"], - auth_style: "bearer", - header_name: "authorization", - }), - ]); - }); + }); + expect(methods(managedInference)).toEqual(["GET", "POST"]); + expect(binaries(policy, "managed_inference")).toEqual( + [ + "/usr/bin/curl", + "/usr/bin/node", + "/usr/bin/python3", + "/usr/local/bin/node", + "/usr/local/bin/openclaw", + ].sort(), + ); - it("keeps both provider policy layers aligned with the least-privilege preset", () => { - const presetEndpoint = preset.network_policies?.tavily?.endpoints?.[0]; - const expectedRules = [ - { allow: { method: "POST", path: "/search" } }, - { allow: { method: "POST", path: "/extract" } }, - ]; - - expect(presetEndpoint?.rules).toEqual(expectedRules); - for (const candidate of [profile, hermesProfile]) { - expect(candidate.endpoints).toEqual([ - { - host: "api.tavily.com", - port: 443, - protocol: "rest", - enforcement: "enforce", - request_body_credential_rewrite: true, - rules: expectedRules, - }, - ]); - expect(candidate.endpoints?.[0]).not.toHaveProperty("access"); + const clawhub = endpoint(policy, "clawhub", "clawhub.ai"); + expect(clawhub).toMatchObject({ allow_encoded_slash: true }); + expect( + allEndpoints(policy) + .filter((candidate) => candidate.allow_encoded_slash === true) + .map((candidate) => candidate.host), + ).toEqual(["clawhub.ai"]); + + expect(binaries(policy, "npm_registry")).toEqual(["/usr/local/bin/openclaw"]); + expect(JSON.stringify(networkPolicies)).not.toContain("/usr/local/bin/claude"); + + const defaultHosts = new Set(allEndpoints(policy).map((candidate) => candidate.host)); + for (const optInHost of [ + "github.com", + "api.github.com", + "sentry.io", + "api.telegram.org", + "discord.com", + "gateway.discord.gg", + "slack.com", + ]) { + expect(defaultHosts, optInHost).not.toContain(optInHost); + } + } finally { + prepared.cleanup?.(); } }); - it("limits the binary allowlist to runtimes the Tavily client actually uses", () => { - expect(profile.binaries).toEqual([ - "/opt/venv/bin/python3*", - "/usr/local/bin/node", - "/usr/bin/node", - "/usr/local/bin/curl", - "/usr/bin/curl", - ]); - }); - - it("keeps its binary allowlist aligned with the Tavily policy preset", () => { - const presetBinaries = preset.network_policies?.tavily?.binaries?.map(({ path }) => path); - for (const binary of profile.binaries ?? []) expect(presetBinaries).toContain(binary); - }); - - it("anchors managed Python access to Deep Agents Code's read-only venv", () => { - const managedPython = "/opt/venv/bin/python3*"; - const managedInferenceBinaries = deepAgentsPolicy.network_policies?.managed_inference?.binaries; - - expect(deepAgentsPolicy.filesystem_policy?.read_only).toContain("/opt/venv"); - expect(managedInferenceBinaries).toContainEqual({ path: managedPython }); - expect(profile.binaries).toContain(managedPython); - }); - - it("supports Hermes' exact managed Python path and JSON credential rewrite", () => { - const endpoint = preset.network_policies?.tavily?.endpoints?.find( - (candidate) => candidate.host === "api.tavily.com", - ); - - expect(hermesProfile).toMatchObject({ - id: "tavily-hermes-v1", - credentials: [ - expect.objectContaining({ - env_vars: ["TAVILY_API_KEY"], - auth_style: "bearer", - header_name: "authorization", - }), - ], - endpoints: [expect.objectContaining({ host: "api.tavily.com", port: 443 })], - binaries: ["/opt/hermes/.venv/bin/python", "/usr/local/bin/curl", "/usr/bin/curl"], - }); - expect(endpoint).toMatchObject({ - protocol: "rest", - enforcement: "enforce", - request_body_credential_rewrite: true, - rules: [ - { allow: { method: "POST", path: "/search" } }, - { allow: { method: "POST", path: "/extract" } }, - ], + it("keeps permissive OpenClaw compatibility routes after create-policy preparation", () => { + const prepared = prepareInitialSandboxCreatePolicy(PERMISSIVE_POLICY_PATH.pathname, [], { + agentName: "openclaw", }); - expect(endpoint).not.toHaveProperty("access"); - }); - - it("preserves Tavily credential rewriting when agent shields are down", () => { - for (const policy of [ - defaultOpenClawPermissivePolicy, - openClawPermissivePolicy, - hermesPermissivePolicy, - ]) { - const endpoint = policy.network_policies?.tavily?.endpoints?.find( - (candidate) => candidate.host === "api.tavily.com", + try { + const consumed = policies.mergePresetNamesIntoPolicy( + readFileSync(prepared.policyPath, "utf-8"), + [], + { agent: "openclaw" }, ); + const policy = parseEffectivePolicy(consumed.policy); + const managedInference = endpoint(policy, "managed_inference", "inference.local"); - expect(endpoint).toMatchObject({ + expect(managedInference).toMatchObject({ + port: 443, protocol: "rest", enforcement: "enforce", access: "full", - request_body_credential_rewrite: true, }); - expect(endpoint?.rules).toBeUndefined(); - expect(policy.network_policies?.tavily?.binaries).toEqual([{ path: "/**" }]); - } - }); -}); - -describe("permissive sandbox policy", () => { - // openclaw-sandbox-permissive.yaml is applied by `shields down --policy - // permissive`. It must carry forward the gateway-managed inference route - // so the mental model stays consistent with the base policy and so we - // don't silently depend on OpenShell's implicit allow for - // gateway-bound virtual hostnames. - // Ref: https://github.com/NVIDIA/NemoClaw/issues/2513, #2663 - const policy = loadYaml(PERMISSIVE_POLICY_PATH); - const agentPolicy = loadYaml(OPENCLAW_PERMISSIVE_POLICY_PATH); - - it("allows inference.local:443 in the managed_inference block (#2513)", () => { - const np = policy.network_policies ?? {}; - expect(np.managed_inference).toBeDefined(); - const endpoints = np.managed_inference?.endpoints ?? []; - const inferenceEp = endpoints.find((ep) => ep.host === "inference.local"); - expect(inferenceEp).toBeDefined(); - expect(inferenceEp?.port).toBe(443); - // Permissive policy uses the `access: full` convention (any method, any - // path) rather than explicit per-method rules. That is consistent with - // every other host in this file. - expect(inferenceEp?.access).toBe("full"); - expect(inferenceEp?.enforcement).toBe("enforce"); - }); + expect(binaries(policy, "managed_inference")).toEqual(["/**"]); - it("uses a permissive '/**' binary allowlist for managed_inference (#2513)", () => { - const np = policy.network_policies ?? {}; - const binaries = (np.managed_inference?.binaries ?? []).map((b) => b.path); - // Matches the permissive-file convention used by every other block - // (e.g. `nvidia`, `github`, `huggingface`, etc.). - expect(binaries).toEqual(["/**"]); - }); - - it("preserves ClawHub encoded scoped-package paths in permissive mode (#4104)", () => { - for (const candidate of [policy, agentPolicy]) { - const endpoints = candidate.network_policies?.clawhub?.endpoints ?? []; - expect(endpoints).toHaveLength(1); - expect(endpoints[0]).toMatchObject({ - host: "clawhub.ai", + const clawhub = endpoint(policy, "clawhub", "clawhub.ai"); + expect(clawhub).toMatchObject({ protocol: "rest", enforcement: "enforce", - allow_encoded_slash: true, access: "full", + allow_encoded_slash: true, }); + } finally { + prepared.cleanup?.(); } }); -}); -describe("Hermes sandbox policy", () => { - const policy = loadYaml(HERMES_POLICY_PATH); - - function expectManagedInferenceSecurityShape(): void { - const np = policy.network_policies ?? {}; - const managedInference = np.managed_inference; - expect(managedInference?.name).toBe("managed_inference"); - expect(managedInference?.binaries?.map((b) => b.path)).toEqual([ - "/usr/local/bin/hermes", - "/usr/bin/python3.11", - "/opt/hermes/.venv/bin/python", - ]); - - const endpoints = managedInference?.endpoints ?? []; - expect(endpoints).toHaveLength(1); - expect(endpoints[0]).toMatchObject({ - host: "inference.local", - port: 443, - protocol: "rest", - enforcement: "enforce", + it("keeps Hermes inference and package access narrow after create-policy preparation", () => { + const prepared = prepareInitialSandboxCreatePolicy(HERMES_POLICY_PATH.pathname, [], { + agentName: "hermes", }); - expect(endpoints[0].access).toBeUndefined(); - expect(endpoints[0].rules).toEqual([ - { allow: { method: "POST", path: "/v1/chat/completions" } }, - { allow: { method: "POST", path: "/v1/messages" } }, - { allow: { method: "POST", path: "/v1/responses" } }, - { allow: { method: "POST", path: "/v1/completions" } }, - { allow: { method: "POST", path: "/v1/embeddings" } }, - { allow: { method: "GET", path: "/v1/models" } }, - { allow: { method: "GET", path: "/v1/models/**" } }, - ]); - } - - it("keeps a narrow inference API allowlist for managed_inference (#4230)", () => { - expectManagedInferenceSecurityShape(); - }); - - function expectGithubBaselineAbsent(): void { - const np = policy.network_policies ?? {}; - expect("github" in np).toBe(false); - const hosts = Object.values(np).flatMap((entry) => (entry.endpoints ?? []).map((e) => e.host)); - expect(hosts).not.toContain("github.com"); - expect(hosts).not.toContain("api.github.com"); - } - - it("base policy does not silently grant GitHub access; only the opt-in preset does", () => { - expectGithubBaselineAbsent(); - }); -}); - -describe("github preset", () => { - const PRESET_PATH = new URL( - "../nemoclaw-blueprint/policies/presets/github.yaml", - import.meta.url, - ); + try { + const consumed = policies.mergePresetNamesIntoPolicy( + readFileSync(prepared.policyPath, "utf-8"), + [], + { agent: "hermes" }, + ); + const policy = parseEffectivePolicy(consumed.policy); + const managedInference = endpoint(policy, "managed_inference", "inference.local"); - it("only advertises the installed git binary in the github preset (#2179)", () => { - const parsed = loadYaml(PRESET_PATH); - const meta = parsed.preset; - expect(meta?.description).toBe("GitHub.com and GitHub API access (git)"); - expect(meta?.description ?? "").not.toMatch(/\bgh\b/); + expect(managedInference).toMatchObject({ + port: 443, + protocol: "rest", + enforcement: "enforce", + }); + expect(managedInference).not.toHaveProperty("access"); + expect(managedInference.rules).toEqual([ + { allow: { method: "POST", path: "/v1/chat/completions" } }, + { allow: { method: "POST", path: "/v1/messages" } }, + { allow: { method: "POST", path: "/v1/responses" } }, + { allow: { method: "POST", path: "/v1/completions" } }, + { allow: { method: "POST", path: "/v1/embeddings" } }, + { allow: { method: "GET", path: "/v1/models" } }, + { allow: { method: "GET", path: "/v1/models/**" } }, + ]); + expect(binaries(policy, "managed_inference")).toEqual( + ["/opt/hermes/.venv/bin/python", "/usr/bin/python3.11", "/usr/local/bin/hermes"].sort(), + ); - const binaries = (parsed.network_policies?.github?.binaries ?? []) - .map((binary) => binary.path) - .sort(); - expect(binaries).toEqual(["/usr/bin/git"]); - }); -}); + const hosts = new Set(allEndpoints(policy).map((candidate) => candidate.host)); + expect(hosts).not.toContain("github.com"); + expect(hosts).not.toContain("api.github.com"); -describe("huggingface preset", () => { - // The huggingface preset used to allow POST /** on huggingface.co, - // which let an agent that found an HF token in the environment - // publish models, datasets, and create repositories via - // /api/repos/create and friends. Inference Provider traffic flows - // through router.huggingface.co, not huggingface.co, so the POST - // rule was never required for read-only `from_pretrained` flows. - // The fix removes the POST rule from huggingface.co (download-only). - // These tests block a regression where someone re-adds it. - // See #1432. - const HUGGINGFACE_PRESET_PATH = new URL( - "../nemoclaw-blueprint/policies/presets/huggingface.yaml", - import.meta.url, - ); - const huggingfacePreset = loadYaml(HUGGINGFACE_PRESET_PATH); - - function presetEndpoints(): Endpoint[] { - const np = huggingfacePreset.network_policies; - if (!np) return []; - const hf = np.huggingface; - return Array.isArray(hf?.endpoints) ? hf.endpoints : []; - } - - it("omits POST allow rules for huggingface.co (#1432)", () => { - const endpoints = presetEndpoints().filter((ep) => ep.host === "huggingface.co"); - expect(endpoints.length).toBeGreaterThan(0); - for (const ep of endpoints) { - const rules = Array.isArray(ep.rules) ? ep.rules : []; - const hasPost = rules.some( - (r) => - r && - r.allow && - typeof r.allow.method === "string" && - r.allow.method.toUpperCase() === "POST", + const pypi = policy.network_policies?.pypi; + for (const candidate of pypi?.endpoints ?? []) { + expect(methods(candidate)).toEqual(["GET"]); + } + expect(binaries(policy, "pypi")).toEqual( + expect.arrayContaining([ + "/opt/hermes/.venv/bin/python", + "/usr/bin/curl", + "/usr/bin/python3*", + "/usr/local/bin/curl", + "/usr/local/bin/pip3", + ]), ); - expect(hasPost).toBe(false); + } finally { + prepared.cleanup?.(); } }); - it("retains GET for huggingface.co so downloads still work (#1432)", () => { - const endpoints = presetEndpoints().filter((ep) => ep.host === "huggingface.co"); - for (const ep of endpoints) { - const rules = Array.isArray(ep.rules) ? ep.rules : []; - const hasGet = rules.some( - (r) => - r && - r.allow && - typeof r.allow.method === "string" && - r.allow.method.toUpperCase() === "GET", + it("applies optional source-control and package presets through the production merge path", () => { + const prepared = prepareInitialSandboxCreatePolicy(BASE_POLICY_PATH.pathname, [], { + agentName: "openclaw", + additionalPresets: ["github", "huggingface", "jira"], + }); + try { + const consumed = policies.mergePresetNamesIntoPolicy( + readFileSync(prepared.policyPath, "utf-8"), + [], + { agent: "openclaw" }, ); - expect(hasGet).toBe(true); - } - }); -}); + const policy = parseEffectivePolicy(consumed.policy); -describe("jira preset", () => { - const JIRA_PRESET_PATH = new URL( - "../nemoclaw-blueprint/policies/presets/jira.yaml", - import.meta.url, - ); - const jiraPreset = loadYaml(JIRA_PRESET_PATH); + expect(prepared.appliedPresets).toEqual(["github", "huggingface", "jira"]); + expect(consumed.missingPresets).toEqual([]); + expect(binaries(policy, "github")).toEqual(["/usr/bin/git"]); - it("allows Node but not curl for Jira (#3758)", () => { - const binaries = (jiraPreset.network_policies?.atlassian?.binaries ?? []) - .map((binary) => binary.path) - .sort(); + const huggingface = endpoint(policy, "huggingface", "huggingface.co"); + expect(methods(huggingface)).toContain("GET"); + expect(methods(huggingface)).not.toContain("POST"); - expect(binaries).toEqual(["/usr/bin/node", "/usr/local/bin/node"]); - expect(binaries).not.toContain("/usr/bin/curl"); - expect(binaries).not.toContain("/usr/local/bin/curl"); + expect(binaries(policy, "atlassian")).toEqual(["/usr/bin/node", "/usr/local/bin/node"]); + expect(binaries(policy, "atlassian")).not.toContain("/usr/bin/curl"); + expect(binaries(policy, "atlassian")).not.toContain("/usr/local/bin/curl"); + } finally { + prepared.cleanup?.(); + } }); }); - -describe("messaging WebSocket presets", () => { - const DISCORD_PRESET_PATH = new URL( - "../src/lib/messaging/channels/discord/policy/openclaw.yaml", - import.meta.url, - ); - const SLACK_PRESET_PATH = new URL( - "../src/lib/messaging/channels/slack/policy/openclaw.yaml", - import.meta.url, - ); - - const presets = [ - { - name: "discord", - policyKey: "discord", - host: "gateway.discord.gg", - credentialRewrite: true, - data: loadYaml(DISCORD_PRESET_PATH), - }, - { - name: "discord", - policyKey: "discord", - host: "*.discord.gg", - credentialRewrite: true, - data: loadYaml(DISCORD_PRESET_PATH), - }, - { - name: "slack", - policyKey: "slack", - host: "wss-primary.slack.com", - credentialRewrite: true, - data: loadYaml(SLACK_PRESET_PATH), - }, - { - name: "slack", - policyKey: "slack", - host: "wss-backup.slack.com", - credentialRewrite: true, - data: loadYaml(SLACK_PRESET_PATH), - }, - ]; - - for (const preset of presets) { - it(`${preset.name} ${preset.host} uses native WebSocket inspection`, () => { - const endpoints = preset.data.network_policies?.[preset.policyKey]?.endpoints ?? []; - const endpoint = endpoints.find((candidate) => candidate.host === preset.host); - expect(endpoint).toBeDefined(); - expect(endpoint).toMatchObject({ protocol: "websocket", enforcement: "enforce" }); - expect(endpoint).not.toHaveProperty("access"); - expect(endpoint).not.toHaveProperty("tls"); - expect(endpoint?.websocket_credential_rewrite === true).toBe(preset.credentialRewrite); - expect(endpoint?.rules).toEqual( - expect.arrayContaining([ - { allow: { method: "GET", path: "/**" } }, - { allow: { method: "WEBSOCKET_TEXT", path: "/**" } }, - ]), - ); - }); - } -}); - -describe("Slack REST credential rewrite", () => { - const SLACK_PRESET_PATH = new URL( - "../src/lib/messaging/channels/slack/policy/openclaw.yaml", - import.meta.url, - ); - const data = loadYaml(SLACK_PRESET_PATH); - const slackRestHosts = ["slack.com", "api.slack.com", "hooks.slack.com"]; - - for (const host of slackRestHosts) { - it(`${host} enables request-body credential rewrite`, () => { - const endpoints = data.network_policies?.slack?.endpoints ?? []; - const endpoint = endpoints.find((candidate) => candidate.host === host); - expect(endpoint).toBeDefined(); - expect(endpoint).toMatchObject({ - protocol: "rest", - enforcement: "enforce", - request_body_credential_rewrite: true, - }); - }); - } -}); - -describe("npm preset", () => { - // Regression #2767: npm/Yarn registry endpoints used `protocol: rest` - // with only GET allowed. Node 22 undici issues HTTP CONNECT through - // HTTPS_PROXY for TLS tunneling; the L7 proxy rejects parallel CONNECT - // tunnels, causing NET:FAIL and ECONNRESET on tarball downloads. - // The fix switches to L4 tunnel mode. - const NPM_PRESET_PATH = new URL( - "../nemoclaw-blueprint/policies/presets/npm.yaml", - import.meta.url, - ); - const npmPreset = loadYaml(NPM_PRESET_PATH); - - function npmEndpoints(): Endpoint[] { - const np = npmPreset.network_policies; - if (!np) return []; - const entry = np.npm_yarn; - return Array.isArray(entry?.endpoints) ? entry.endpoints : []; - } - - const REGISTRY_HOSTS = ["registry.npmjs.org", "registry.yarnpkg.com"]; - - for (const host of REGISTRY_HOSTS) { - it(`uses an L4 tunnel for CONNECT compatibility on ${host} (access: full, tls: skip) (#2767)`, () => { - const endpoints = npmEndpoints().filter((ep) => ep.host === host); - expect(endpoints.length).toBeGreaterThan(0); - for (const ep of endpoints) { - expect(ep.access).toBe("full"); - expect(ep).toHaveProperty("tls", "skip"); - // Must NOT use protocol: rest — that triggers L7 method inspection - // which rejects CONNECT tunnels from Node 22 undici. - expect(ep).not.toHaveProperty("protocol"); - expect(ep).not.toHaveProperty("rules"); - } - }); - } -}); diff --git a/test/vitest-coverage-thresholds.test.ts b/test/vitest-coverage-thresholds.test.ts index adfdd728b9f..3c84e1bf0bd 100644 --- a/test/vitest-coverage-thresholds.test.ts +++ b/test/vitest-coverage-thresholds.test.ts @@ -16,6 +16,7 @@ type RootTestOptions = { }; describe("Vitest security coverage thresholds", () => { + // source-shape-contract: security -- Exact per-file floors are executable security regression gates for merged coverage it("enforces the exact per-file security floors for full and merged coverage (#6692)", () => { expect(resolveVitestCoverageThresholds([])).toEqual({ perFile: true, diff --git a/test/vitest-developer-feedback.test.ts b/test/vitest-developer-feedback.test.ts index ff2378b78f3..ac65947e880 100644 --- a/test/vitest-developer-feedback.test.ts +++ b/test/vitest-developer-feedback.test.ts @@ -16,6 +16,7 @@ type RootTestOptions = { const focusedProjects = "--project cli --project plugin --project e2e-support"; describe("Vitest developer feedback", () => { + // source-shape-contract: compatibility -- Root Vitest feedback must preserve CI failure logs while deferring reporter selection it("lets Vitest select reporters and preserves failed-test logs in CI (#6692)", () => { const testOptions = rootVitestConfig.test as RootTestOptions; diff --git a/test/vitest-state-isolation.test.ts b/test/vitest-state-isolation.test.ts index 4fa23042dcf..89eb8eeee6a 100644 --- a/test/vitest-state-isolation.test.ts +++ b/test/vitest-state-isolation.test.ts @@ -50,6 +50,7 @@ describe("Vitest state isolation", () => { } }); + // source-shape-contract: security -- Live and root runners must not enable unvalidated automatic state cleanup it("keeps root and live projects free of unvalidated automatic cleanup", () => { const projects = projectTestOptions(); const rootOptions = rootVitestConfig.test as ProjectTestOptions; @@ -67,6 +68,7 @@ describe("Vitest state isolation", () => { } }); + // source-shape-contract: compatibility -- Standalone plugin execution must share deterministic cleanup without mock reset it("keeps standalone plugin runs aligned without enabling mockReset", () => { expect(pluginVitestProjectOptions.test).toMatchObject(vitestStateIsolation); expect(standalonePluginVitestConfig.test).toMatchObject(vitestStateIsolation); diff --git a/test/vitest-temp-root.test.ts b/test/vitest-temp-root.test.ts index 7e5e8184b8b..bfeafce5e40 100644 --- a/test/vitest-temp-root.test.ts +++ b/test/vitest-temp-root.test.ts @@ -202,6 +202,7 @@ describe("Vitest temp root", () => { } }); + // source-shape-contract: compatibility -- Root and standalone runners must install the shared temporary-root cleanup boundary it("wires cleanup into root and standalone plugin test runs", () => { expect(rootVitestConfig.test?.globalSetup).toBe(ROOT_SETUP); expect(pluginVitestConfig.test?.globalSetup).toBe( diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index ff62c35ff91..86ebe80392c 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -62,6 +62,7 @@ function triggeredBy(relativePath: string): string[] { } describe("Vitest opaque-input watch triggers", () => { + // source-shape-contract: compatibility -- Root watch mode must install the canonical opaque-input trigger resolver it("registers the focused mappings at the root configuration boundary (#6692)", () => { expect(rootVitestConfig.test?.watchTriggerPatterns).toBe(vitestWatchTriggerPatterns); }); diff --git a/vitest.config.ts b/vitest.config.ts index da200ccb713..bd368fa6b0a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -121,6 +121,7 @@ export default defineConfig({ "test/package-contract/**", "test/install-express-prompt.test.ts", "test/install-build-dependency-preflight.test.ts", + "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-openshell-version-check.test.ts", @@ -138,6 +139,7 @@ export default defineConfig({ include: [ "test/install-express-prompt.test.ts", "test/install-build-dependency-preflight.test.ts", + "test/install-clone-ref.test.ts", "test/install-preflight.test.ts", "test/install-preflight-docker-bootstrap.test.ts", "test/install-openshell-version-check.test.ts",