diff --git a/schemas/policy-preset.schema.json b/schemas/policy-preset.schema.json index 82a8e47c3c3..bcf674942ff 100644 --- a/schemas/policy-preset.schema.json +++ b/schemas/policy-preset.schema.json @@ -27,7 +27,7 @@ "$defs": { "networkPolicyEntry": { "type": "object", - "required": ["name", "endpoints"], + "required": ["name", "endpoints", "binaries"], "properties": { "name": { "type": "string" }, "endpoints": { @@ -37,7 +37,8 @@ }, "binaries": { "type": "array", - "items": { "$ref": "#/$defs/binary" } + "items": { "$ref": "#/$defs/binary" }, + "minItems": 1 } } }, diff --git a/schemas/router-pool-config.schema.json b/schemas/router-pool-config.schema.json new file mode 100644 index 00000000000..caa026b66d0 --- /dev/null +++ b/schemas/router-pool-config.schema.json @@ -0,0 +1,48 @@ +{ + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/NVIDIA/NemoClaw/schemas/router-pool-config.schema.json", + "title": "NemoClaw Model Router Pool Config", + "description": "Schema for nemoclaw-blueprint/router/pool-config.yaml.", + "type": "object", + "required": ["routing", "models"], + "additionalProperties": false, + "properties": { + "routing": { + "type": "object", + "required": ["method", "checkpoint", "tolerance", "encoder", "encoder_backend"], + "additionalProperties": false, + "properties": { + "method": { "type": "string", "minLength": 1 }, + "checkpoint": { "type": "string", "minLength": 1 }, + "tolerance": { "type": "number", "minimum": 0, "maximum": 1 }, + "encoder": { "type": "string", "minLength": 1 }, + "encoder_backend": { "type": "string", "minLength": 1 } + } + }, + "models": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "name", + "display_name", + "litellm_model", + "cost_per_m_input_tokens", + "cost_per_m_output_tokens", + "api_base" + ], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "display_name": { "type": "string", "minLength": 1 }, + "litellm_model": { "type": "string", "minLength": 1 }, + "cost_per_m_input_tokens": { "type": "number", "minimum": 0 }, + "cost_per_m_output_tokens": { "type": "number", "minimum": 0 }, + "api_base": { "type": "string", "pattern": "^https://" } + } + } + } + } +} diff --git a/schemas/sandbox-policy.schema.json b/schemas/sandbox-policy.schema.json index 7c6fec34e56..dcd79ea8169 100644 --- a/schemas/sandbox-policy.schema.json +++ b/schemas/sandbox-policy.schema.json @@ -52,7 +52,7 @@ "$defs": { "networkPolicyEntry": { "type": "object", - "required": ["name", "endpoints"], + "required": ["name", "endpoints", "binaries"], "properties": { "name": { "type": "string" }, "endpoints": { @@ -62,7 +62,8 @@ }, "binaries": { "type": "array", - "items": { "$ref": "#/$defs/binary" } + "items": { "$ref": "#/$defs/binary" }, + "minItems": 1 } } }, diff --git a/scripts/checks/layer-import-boundaries.ts b/scripts/checks/layer-import-boundaries.ts index c06062bc149..4660e50c45f 100644 --- a/scripts/checks/layer-import-boundaries.ts +++ b/scripts/checks/layer-import-boundaries.ts @@ -122,6 +122,10 @@ function isCommandFile(repoPath: string): boolean { return repoPath.startsWith("src/commands/"); } +function isMessagingManifestFile(repoPath: string): boolean { + return repoPath.startsWith("src/lib/messaging/manifest/"); +} + function isActionFile(repoPath: string): boolean { if (repoPath.startsWith("src/lib/actions/")) return true; return /(^|\/)[^/]+-actions?\.ts$/.test(repoPath); @@ -262,6 +266,51 @@ function checkNoBinLibShimImport(absPath: string, repoPath: string, violations: } } +function checkMessagingManifestFile( + absPath: string, + repoPath: string, + violations: Violation[], +): void { + const forbiddenFragments = [ + "gateway", + "state/registry", + "credentials", + "node:fs", + "node:child_process", + "child_process", + "adapters/openshell", + "src/commands", + "lib/actions", + ]; + + for (const ref of collectImportRefs(absPath)) { + if (ref.specifier === "fs" || ref.specifier.startsWith("fs/")) { + addViolation( + violations, + repoPath, + ref.line, + ref.column, + "messaging-manifest-purity", + "messaging manifest modules must not import fs", + ); + continue; + } + const target = resolveInternalImport(absPath, ref.specifier); + const haystack = `${ref.specifier}\n${target ?? ""}`; + const fragment = forbiddenFragments.find((candidate) => haystack.includes(candidate)); + if (fragment) { + addViolation( + violations, + repoPath, + ref.line, + ref.column, + "messaging-manifest-purity", + `messaging manifest modules must not import ${fragment}`, + ); + } + } +} + function checkCommandFile(absPath: string, repoPath: string, violations: Violation[]): void { const sourceFile = sourceFileFor(absPath); let commandClassCount = 0; @@ -305,6 +354,9 @@ export function findLayerImportBoundaryViolations(root = SRC_ROOT): Violation[] if (isDomainFile(repoPath)) checkDomainFile(absPath, repoPath, violations); if (isActionFile(repoPath)) checkActionFile(absPath, repoPath, violations); if (isAdapterFile(repoPath)) checkAdapterFile(absPath, repoPath, violations); + if (isMessagingManifestFile(repoPath)) { + checkMessagingManifestFile(absPath, repoPath, violations); + } if (isCommandFile(repoPath)) checkCommandFile(absPath, repoPath, violations); } return violations; diff --git a/scripts/find-source-shape-tests.ts b/scripts/find-source-shape-tests.ts index 56021a04776..2197ac309dd 100755 --- a/scripts/find-source-shape-tests.ts +++ b/scripts/find-source-shape-tests.ts @@ -5,7 +5,7 @@ // Finds tests that read production source text and assert on its shape. These // tests tend to couple coverage to implementation strings instead of behavior. -import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import { basename, dirname, join, relative } from "node:path"; import { fileURLToPath } from "node:url"; import ts from "typescript"; @@ -52,6 +52,8 @@ type VariableDecl = { type SourceFunction = { readonly name: string; readonly sourceRead: SourceRead; + readonly parameterNames: readonly string[]; + readonly parameterizedPathRead: boolean; }; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); @@ -113,17 +115,33 @@ function looksLikeTestFixturePath(text: string): boolean { const normalized = normalizePathText(text); return ( /Dockerfile\.sandbox/.test(normalized) || - /["'`]test["'`]/.test(normalized) || + /(?:^|\/)fixtures?\//.test(normalized) || /\.agents\/skills/.test(normalized) ); } +function looksLikeDeclarativeConfigPath(text: string): boolean { + const normalized = normalizePathText(text); + // Declarative configs below have dedicated schema/resolver validation. Keep + // this scanner focused on source-code shape assertions rather than treating + // every schema-backed config invariant as source-text coupling. + return ( + /nemoclaw-blueprint\/blueprint\.yaml/.test(normalized) || + /nemoclaw-blueprint\/policies\//.test(normalized) || + /nemoclaw-blueprint\/router\/pool-config\.yaml/.test(normalized) || + /nemoclaw-blueprint\/model-specific-setup\//.test(normalized) || + /agents\/[^/]+\/policy-(?:additions|permissive)\.yaml/.test(normalized) + ); +} + function isProductionPathExpression( text: string, productionPathVars: ReadonlySet, ): boolean { const normalized = normalizePathText(text); - if (looksLikeTestFixturePath(normalized)) return false; + if (looksLikeTestFixturePath(normalized) || looksLikeDeclarativeConfigPath(normalized)) { + return false; + } if ([...productionPathVars].some((name) => textContainsIdentifier(normalized, name))) return true; return hasDirectProductionPathHint(normalized); @@ -131,6 +149,10 @@ function isProductionPathExpression( function hasDirectProductionPathHint(text: string): boolean { return ( + /["'`](?:\.\.\/)?(?:\.github|agents|bin|dist|nemoclaw|nemoclaw-blueprint|scripts|src|test\/e2e)\//.test( + text, + ) || + /["'`](?:package\.json|install\.sh|\.pre-commit-config\.yaml)["'`]/.test(text) || /["'`]\.\.\/Dockerfile(?:\.base)?["'`]/.test(text) || /["'`]\.\.\/bin\//.test(text) || /["'`]\.\.\/agents\//.test(text) || @@ -146,6 +168,9 @@ function hasDirectProductionPathHint(text: string): boolean { /join\(\s*["'`]\.\.["'`]\s*,\s*["'`](?:\.github|agents|bin|dist|nemoclaw|nemoclaw-blueprint|scripts|src|Dockerfile(?:\.base)?|install\.sh|package\.json)["'`]\s*\)/.test( text, ) || + /path\.join\(\s*process\.cwd\(\)\s*,\s*["'`](?:\.github|agents|bin|dist|nemoclaw|nemoclaw-blueprint|scripts|src|Dockerfile(?:\.base)?|install\.sh|package\.json)["'`]/.test( + text, + ) || /(import\.meta\.dirname|import\.meta\.url)[\s\S]*["'`](?![\w.-]+\.test\.ts["'`])[\w.-]+\.ts["'`]/.test( text, ) || @@ -160,10 +185,14 @@ function hasDirectProductionPathHint(text: string): boolean { function isPathLikeVariableName(name: string): boolean { return ( /^(REPO_ROOT|ROOT)$/.test(name) || - /(path|file|script|source|src|dockerfile|payload|installer)/i.test(name) + /(root|dir|path|file|files|script|source|src|dockerfile|payload|installer)/i.test(name) ); } +function isReadFileExpressionText(text: string): boolean { + return /\b(?:readFileSync|readFile)\s*\(/.test(text); +} + function isReadFileCall(node: ts.CallExpression): boolean { const expression = node.expression; if (ts.isIdentifier(expression)) { @@ -182,11 +211,25 @@ function isSourceTextLikeName(name: string): boolean { } function isTextDerivation(initText: string): boolean { - return /(\.indexOf\b|\.search\b|\.includes\b|\.match(All)?\b|\.slice\b|\.split\b|\.replace(All)?\b|\.trim(End)?\b|\.join\b|String\(|Heredoc\b|Snippet\b|Block\b|extract[A-Z])/.test( + return /(\.indexOf\b|\.search\b|\.includes\b|\.match(All)?\b|\.slice\b|\.split\b|\.replace(All)?\b|\.trim(End)?\b|\.join\b|String\(|(?:YAML|yaml|JSON)\.parse\b|yaml\.load\b|Heredoc\b|Snippet\b|Block\b|extract[A-Z]|load[A-Z]|parse[A-Z])/.test( + initText, + ); +} + +function isExecutionResultDerivation(initText: string): boolean { + return /\b(?:spawnSync|execFileSync|execSync|run(?:Logged|Docker|Bash|WithLib|Embedded|Patch|Hermes|Openclaw|Daemon|Fetch|Command)\w*)\b/.test( initText, ); } +function looksLikeSourceFileExtensionFilter(text: string): boolean { + return /\.endsWith\(\s*["'`]\.(?:[cm]?[jt]sx?|mts|cts)["'`]\s*\)/.test(text); +} + +function looksLikeSourceTreeEnumeration(text: string): boolean { + return /\breaddirSync\s*\(/.test(text) && looksLikeSourceFileExtensionFilter(text); +} + function collectVariableDecls(sourceFile: ts.SourceFile): VariableDecl[] { const variables: VariableDecl[] = []; @@ -253,6 +296,7 @@ function collectProductionPathVars( [...pathVars].some((name) => textContainsIdentifier(initText, name)); if ( !looksLikeTestFixturePath(initText) && + !looksLikeDeclarativeConfigPath(initText) && (isRepositoryRoot || directlyNamesProductionPath || derivesNamedProductionPath) ) { pathVars.add(variable.name); @@ -269,13 +313,49 @@ function callTargetName(expression: ts.Expression): string | null { return null; } +function nestedSourceReadInNode( + sourceFile: ts.SourceFile, + root: ts.Node, + productionPathVars: ReadonlySet, +): SourceRead | null { + let sourceRead: SourceRead | null = null; + + function visit(node: ts.Node): void { + if (sourceRead) return; + if (isNestedFunctionLike(node)) return; + if ( + ts.isCallExpression(node) && + isReadFileCall(node) && + node.arguments.length > 0 && + isProductionPathExpression(node.arguments[0].getText(sourceFile), productionPathVars) + ) { + const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart()); + sourceRead = { + line: line + 1, + column: character + 1, + variable: "", + expression: node.getText(sourceFile), + }; + return; + } + ts.forEachChild(node, visit); + } + + visit(root); + return sourceRead; +} + function sourceReadFromInitializer( sourceFile: ts.SourceFile, variable: VariableDecl, productionPathVars: ReadonlySet, - sourceFunctions: ReadonlyMap, + sourceFunctions: ReadonlyMap, ): SourceRead | null { const init = variable.initializer; + const nestedRead = nestedSourceReadInNode(sourceFile, init, productionPathVars); + if (nestedRead) { + return { ...nestedRead, variable: variable.name }; + } if (!ts.isCallExpression(init)) { return null; } @@ -302,6 +382,14 @@ function sourceReadFromInitializer( if (!functionSourceRead) { return null; } + if ( + functionSourceRead.parameterizedPathRead && + !init.arguments.some((argument) => + isProductionPathExpression(argument.getText(sourceFile), productionPathVars), + ) + ) { + return null; + } const { line, character } = sourceFile.getLineAndCharacterOfPosition( variable.initializer.getStart(), @@ -310,7 +398,7 @@ function sourceReadFromInitializer( line: line + 1, column: character + 1, variable: variable.name, - expression: `${variable.initializer.getText(sourceFile)} -> ${functionSourceRead.expression}`, + expression: `${variable.initializer.getText(sourceFile)} -> ${functionSourceRead.sourceRead.expression}`, }; } @@ -324,13 +412,59 @@ function isNestedFunctionLike(node: ts.Node): boolean { ); } +function functionLikeNameAndBody(node: ts.Node): { + name: string; + body: ts.ConciseBody; + node: ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction; +} | null { + if (ts.isFunctionDeclaration(node) && node.name && node.body) { + return { name: node.name.text, body: node.body, node }; + } + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.initializer && + (ts.isArrowFunction(node.initializer) || ts.isFunctionExpression(node.initializer)) && + node.initializer.body + ) { + return { name: node.name.text, body: node.initializer.body, node: node.initializer }; + } + return null; +} + +function collectSourceTreeFunctionNames(sourceFile: ts.SourceFile): Set { + const names = new Set(); + + function visit(node: ts.Node): void { + const functionLike = functionLikeNameAndBody(node); + if (functionLike && looksLikeSourceTreeEnumeration(functionLike.body.getText(sourceFile))) { + names.add(functionLike.name); + } + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return names; +} + function collectSourceFunctions( sourceFile: ts.SourceFile, productionPathVars: ReadonlySet, -): Map { - const sourceFunctions = new Map(); +): Map { + const sourceFunctions = new Map(); + + function parameterNamesFor(node: { + parameters: ts.NodeArray; + }): string[] { + return node.parameters + .map((parameter) => (ts.isIdentifier(parameter.name) ? parameter.name.text : null)) + .filter((name): name is string => Boolean(name)); + } - function sourceReadFromExpression(expression: ts.Expression): SourceRead | null { + function sourceReadFromExpression( + expression: ts.Expression, + parameterNames: readonly string[], + ): { sourceRead: SourceRead; parameterizedPathRead: boolean } | null { if ( !ts.isCallExpression(expression) || !isReadFileCall(expression) || @@ -339,31 +473,70 @@ function collectSourceFunctions( return null; } const targetText = expression.arguments[0].getText(sourceFile); - if (!isProductionPathExpression(targetText, productionPathVars)) { + const parameterizedPathRead = parameterNames.some((name) => + textContainsIdentifier(targetText, name), + ); + if (!parameterizedPathRead && !isProductionPathExpression(targetText, productionPathVars)) { return null; } const { line, character } = sourceFile.getLineAndCharacterOfPosition(expression.getStart()); return { - line: line + 1, - column: character + 1, - variable: "", - expression: expression.getText(sourceFile), + parameterizedPathRead, + sourceRead: { + line: line + 1, + column: character + 1, + variable: "", + expression: expression.getText(sourceFile), + }, }; } - function visit(node: ts.Node): void { - if (ts.isFunctionDeclaration(node) && node.name) { - let sourceRead: SourceRead | null = null; - function visitFunctionBody(child: ts.Node): void { - if (sourceRead) return; - if (child !== node && isNestedFunctionLike(child)) return; - if (ts.isReturnStatement(child) && child.expression) { - sourceRead = sourceReadFromExpression(child.expression); + function registerSourceFunction( + name: string, + node: ts.FunctionDeclaration | ts.FunctionExpression | ts.ArrowFunction, + ): void { + const functionText = node.getText(sourceFile); + if (isExecutionResultDerivation(functionText)) return; + + let sourceRead: SourceRead | null = null; + let parameterizedPathRead = false; + const parameterNames = parameterNamesFor(node); + + function visitFunctionBody(child: ts.Node): void { + if (sourceRead) return; + if (child !== node && isNestedFunctionLike(child)) return; + if (ts.isCallExpression(child)) { + const result = sourceReadFromExpression(child, parameterNames); + if (result) { + sourceRead = result.sourceRead; + parameterizedPathRead = result.parameterizedPathRead; + return; } - ts.forEachChild(child, visitFunctionBody); } - if (node.body) visitFunctionBody(node.body); - if (sourceRead) sourceFunctions.set(node.name.text, sourceRead); + ts.forEachChild(child, visitFunctionBody); + } + + if (node.body) visitFunctionBody(node.body); + if (sourceRead) { + sourceFunctions.set(name, { + name, + sourceRead, + parameterNames, + parameterizedPathRead, + }); + } + } + + function visit(node: ts.Node): void { + if (ts.isFunctionDeclaration(node) && node.name) { + registerSourceFunction(node.name.text, node); + } else if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.initializer && + (ts.isArrowFunction(node.initializer) || ts.isFunctionExpression(node.initializer)) + ) { + registerSourceFunction(node.name.text, node.initializer); } ts.forEachChild(node, visit); } @@ -372,13 +545,82 @@ function collectSourceFunctions( return sourceFunctions; } +function collectSourceTreeShapeVars( + sourceFile: ts.SourceFile, + body: ts.Node, + variables: readonly VariableDecl[], + productionPathVars: ReadonlySet, +): { sourceVars: Set; pathVars: Set; sourceTreeFunctions: Set } { + const sourceTreeFunctions = collectSourceTreeFunctionNames(sourceFile); + const sourceVars = new Set(); + const pathVars = new Set(); + const bodyText = body.getText(sourceFile); + + for (const variable of variables) { + const init = variable.initializer; + const helperRead = + ts.isCallExpression(init) && + ts.isIdentifier(init.expression) && + sourceTreeFunctions.has(init.expression.text) && + init.arguments.some((argument) => + isProductionPathExpression(argument.getText(sourceFile), productionPathVars), + ); + const localCollector = + ts.isArrayLiteralExpression(init) && + textContainsIdentifier(bodyText, variable.name) && + looksLikeSourceTreeEnumeration(bodyText) && + new RegExp(`\\b${escapeRegExp(variable.name)}\\.push\\s*\\(`).test(bodyText); + + if (helperRead || localCollector) { + sourceVars.add(variable.name); + } + } + + let changed = true; + while (changed) { + changed = false; + for (const variable of variables) { + if (sourceVars.has(variable.name)) continue; + const initText = variable.initializer.getText(sourceFile); + if ([...sourceVars].some((name) => textContainsIdentifier(initText, name))) { + sourceVars.add(variable.name); + changed = true; + } + } + } + + function visit(node: ts.Node): void { + if (ts.isForOfStatement(node)) { + const expressionText = node.expression.getText(sourceFile); + const iteratesSourceTree = [...sourceVars].some((name) => + textContainsIdentifier(expressionText, name), + ); + if (iteratesSourceTree) { + const initializer = node.initializer; + if (ts.isVariableDeclarationList(initializer)) { + for (const declaration of initializer.declarations) { + if (ts.isIdentifier(declaration.name)) pathVars.add(declaration.name.text); + } + } else if (ts.isIdentifier(initializer)) { + pathVars.add(initializer.text); + } + } + } + ts.forEachChild(node, visit); + } + visit(body); + + return { sourceVars, pathVars, sourceTreeFunctions }; +} + function collectSourceVars( sourceFile: ts.SourceFile, variables: readonly VariableDecl[], productionPathVars: ReadonlySet, - sourceFunctions: ReadonlyMap, + sourceFunctions: ReadonlyMap, + initialSourceVars: ReadonlySet = new Set(), ): { sourceVars: Set; sourceReads: SourceRead[] } { - const sourceVars = new Set(); + const sourceVars = new Set(initialSourceVars); const sourceReads: SourceRead[] = []; for (const variable of variables) { @@ -403,7 +645,13 @@ function collectSourceVars( const referencesSource = [...sourceVars].some((name) => textContainsIdentifier(initText, name), ); - if (referencesSource && (isSourceTextLikeName(variable.name) || isTextDerivation(initText))) { + const readsProductionFileCollection = + isReadFileExpressionText(initText) && + [...productionPathVars].some((name) => textContainsIdentifier(initText, name)); + if ( + (referencesSource || readsProductionFileCollection) && + !isExecutionResultDerivation(initText) + ) { sourceVars.add(variable.name); changed = true; } @@ -518,10 +766,20 @@ function expressionReferencesSource( expression: ts.Expression, sourceVars: ReadonlySet, productionPathVars: ReadonlySet, + sourceTreeFunctions: ReadonlySet, ): boolean { const text = expression.getText(); + const callsSourceTreeHelper = + ts.isCallExpression(expression) && + ts.isIdentifier(expression.expression) && + sourceTreeFunctions.has(expression.expression.text) && + (expression.arguments.length === 0 || + expression.arguments.some((argument) => + isProductionPathExpression(argument.getText(), productionPathVars), + )); return ( [...sourceVars].some((name) => textContainsIdentifier(text, name)) || + callsSourceTreeHelper || (ts.isCallExpression(expression) && isReadFileCall(expression) && expression.arguments.length > 0 && @@ -534,6 +792,7 @@ function assertionFromExpectCall( node: ts.CallExpression, sourceVars: ReadonlySet, productionPathVars: ReadonlySet, + sourceTreeFunctions: ReadonlySet, ): Assertion | null { if ( sourceVars.size > 0 && @@ -569,7 +828,7 @@ function assertionFromExpectCall( if ( node.arguments.some((argument) => - expressionReferencesSource(argument, sourceVars, productionPathVars), + expressionReferencesSource(argument, sourceVars, productionPathVars, sourceTreeFunctions), ) ) { const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart()); @@ -590,10 +849,16 @@ function assertionFromCall( node: ts.CallExpression, sourceVars: ReadonlySet, productionPathVars: ReadonlySet, + sourceTreeFunctions: ReadonlySet, ): Assertion | null { return ( - assertionFromExpectCall(sourceFile, node, sourceVars, productionPathVars) || - assertionFromAssertCall(sourceFile, node, sourceVars, productionPathVars) + assertionFromExpectCall( + sourceFile, + node, + sourceVars, + productionPathVars, + sourceTreeFunctions, + ) || assertionFromAssertCall(sourceFile, node, sourceVars, productionPathVars) ); } @@ -641,12 +906,19 @@ function collectAssertionsInNode( root: ts.Node, sourceVars: ReadonlySet, productionPathVars: ReadonlySet, + sourceTreeFunctions: ReadonlySet = new Set(), ): Assertion[] { const assertions: Assertion[] = []; function visit(node: ts.Node): void { if (ts.isCallExpression(node)) { - const assertion = assertionFromCall(sourceFile, node, sourceVars, productionPathVars); + const assertion = assertionFromCall( + sourceFile, + node, + sourceVars, + productionPathVars, + sourceTreeFunctions, + ); if (assertion) assertions.push(assertion); } ts.forEachChild(node, visit); @@ -693,7 +965,7 @@ function fallbackLineScan(sourceFile: ts.SourceFile, root: ts.Node): Assertion[] function visit(node: ts.Node): void { if (ts.isCallExpression(node)) { - const assertion = assertionFromCall(sourceFile, node, sourceVars, new Set()); + const assertion = assertionFromCall(sourceFile, node, sourceVars, new Set(), new Set()); if (assertion) assertions.push(assertion); } ts.forEachChild(node, visit); @@ -702,10 +974,8 @@ function fallbackLineScan(sourceFile: ts.SourceFile, root: ts.Node): Assertion[] return assertions; } -function scanFile(absPath: string): SourceShapeCase[] { - const relPath = normalizePathText(relative(REPO_ROOT, absPath)); - const text = readFileSync(absPath, "utf-8"); - const sourceFile = ts.createSourceFile(absPath, text, ts.ScriptTarget.Latest, true); +function scanSourceText(fileName: string, relPath: string, text: string): SourceShapeCase[] { + const sourceFile = ts.createSourceFile(fileName, text, ts.ScriptTarget.Latest, true); const allVariables = collectVariableDecls(sourceFile); const cases: SourceShapeCase[] = []; @@ -715,15 +985,29 @@ function scanFile(absPath: string): SourceShapeCase[] { if (body) { const variables = scopedVariableDecls(sourceFile, allVariables, node, body); const productionPathVars = collectProductionPathVars(sourceFile, variables); - const sourceFunctions = collectSourceFunctions(sourceFile, productionPathVars); - const { sourceVars, sourceReads } = collectSourceVars( + const sourceTreeShapeVars = collectSourceTreeShapeVars( sourceFile, + body, variables, productionPathVars, + ); + const sourcePathVars = new Set([...productionPathVars, ...sourceTreeShapeVars.pathVars]); + const sourceFunctions = collectSourceFunctions(sourceFile, sourcePathVars); + const { sourceVars, sourceReads } = collectSourceVars( + sourceFile, + variables, + sourcePathVars, sourceFunctions, + sourceTreeShapeVars.sourceVars, ); const assertions = dedupeAssertions([ - ...collectAssertionsInNode(sourceFile, body, sourceVars, productionPathVars), + ...collectAssertionsInNode( + sourceFile, + body, + sourceVars, + sourcePathVars, + sourceTreeShapeVars.sourceTreeFunctions, + ), ...fallbackLineScan(sourceFile, body), ]); if (assertions.length > 0) { @@ -746,6 +1030,12 @@ function scanFile(absPath: string): SourceShapeCase[] { return cases; } +function scanFile(absPath: string): SourceShapeCase[] { + const relPath = normalizePathText(relative(REPO_ROOT, absPath)); + const text = readFileSync(absPath, "utf-8"); + return scanSourceText(absPath, relPath, text); +} + function scan(): Report { const cases = [...walkFiles(REPO_ROOT)].filter(isTestFile).flatMap(scanFile); const casesPerFile = new Map(); @@ -825,4 +1115,18 @@ function main(): void { } } -main(); +export function scanTextForTest(relPath: string, text: string): SourceShapeCase[] { + return scanSourceText(relPath, normalizePathText(relPath), text); +} + +function isDirectInvocation(): boolean { + const invoked = process.argv[1]; + return Boolean( + invoked && + (import.meta.url === `file://${invoked}` || invoked.endsWith("find-source-shape-tests.ts")), + ); +} + +if (isDirectInvocation()) { + main(); +} diff --git a/scripts/validate-configs.ts b/scripts/validate-configs.ts index 3c695fa74cf..cd24310fd43 100755 --- a/scripts/validate-configs.ts +++ b/scripts/validate-configs.ts @@ -9,8 +9,8 @@ // npx tsx scripts/validate-configs.ts # validate all known config files // npx tsx scripts/validate-configs.ts --file --schema # validate one file -import { readFileSync, readdirSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv from "ajv/dist/2020.js"; import YAML from "yaml"; @@ -26,6 +26,10 @@ type ConfigScalar = string | number | boolean | null; type ConfigValue = ConfigScalar | ConfigObject | ConfigValue[]; type ConfigObject = { [key: string]: ConfigValue }; +function pathRelativeToRepo(absPath: string): string { + return relative(REPO_ROOT, absPath).replaceAll("\\", "/"); +} + /** * Build the list of config files and their corresponding JSON Schemas. * Preset YAML files are discovered dynamically from the presets directory. @@ -39,14 +43,68 @@ function discoverTargets(): ConfigTarget[] { }, { schema: "schemas/sandbox-policy.schema.json", - files: ["nemoclaw-blueprint/policies/openclaw-sandbox.yaml"], + files: [ + "nemoclaw-blueprint/policies/openclaw-sandbox.yaml", + "nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml", + ], }, { schema: "schemas/openclaw-plugin.schema.json", files: ["nemoclaw/openclaw.plugin.json"], }, + { + schema: "schemas/router-pool-config.schema.json", + files: ["nemoclaw-blueprint/router/pool-config.yaml"], + }, ]; + const agentsDir = join(REPO_ROOT, "agents"); + try { + const agentPolicyFiles = readdirSync(agentsDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .flatMap((entry) => { + const base = `agents/${entry.name}`; + return [`${base}/policy-additions.yaml`, `${base}/policy-permissive.yaml`]; + }) + .filter((file) => existsSync(join(REPO_ROOT, file))); + if (agentPolicyFiles.length > 0) { + const sandboxPolicyTarget = targets.find( + (target) => target.schema === "schemas/sandbox-policy.schema.json", + ); + sandboxPolicyTarget?.files.push(...agentPolicyFiles); + } + } catch (err) { + const code = typeof err === "object" && err !== null && "code" in err ? err.code : undefined; + if (code !== "ENOENT" && code !== "ENOTDIR") throw err; + // agents directory may not exist — not an error + } + + const modelSetupDir = join(REPO_ROOT, "nemoclaw-blueprint", "model-specific-setup"); + try { + const modelSetupFiles: string[] = []; + const walkModelSetup = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const abs = join(dir, entry.name); + if (entry.isDirectory()) { + walkModelSetup(abs); + } else if (entry.isFile() && entry.name.endsWith(".json") && entry.name !== "schema.json") { + modelSetupFiles.push(pathRelativeToRepo(abs)); + } + } + }; + walkModelSetup(modelSetupDir); + if (modelSetupFiles.length > 0) { + targets.push({ + schema: "nemoclaw-blueprint/model-specific-setup/schema.json", + files: modelSetupFiles.sort(), + }); + } + } catch (err) { + const code = typeof err === "object" && err !== null && "code" in err ? err.code : undefined; + if (code !== "ENOENT" && code !== "ENOTDIR") throw err; + // model-specific setup directory may not exist — not an error + } + // Discover all preset YAML files dynamically. const presetsDir = join(REPO_ROOT, "nemoclaw-blueprint/policies/presets"); try { @@ -151,6 +209,8 @@ interface DangerousHostFinding { host: string; } +const ROUTER_API_BASE_HOST_ALLOWLIST: ReadonlySet = new Set(["integrate.api.nvidia.com"]); + /** * Walk a parsed policy document (full `network_policies` map or a preset * fragment with a `preset:` block) and return every endpoint whose host @@ -181,6 +241,38 @@ function findDangerousHosts(data: unknown): DangerousHostFinding[] { return findings; } +function findDangerousRouterApiBases(data: unknown): DangerousHostFinding[] { + const findings: DangerousHostFinding[] = []; + if (!data || typeof data !== "object") return findings; + const models = (data as Record).models; + if (!Array.isArray(models)) return findings; + + models.forEach((model, index) => { + if (!model || typeof model !== "object") return; + const apiBase = (model as Record).api_base; + if (typeof apiBase !== "string") return; + let url: URL; + try { + url = new URL(apiBase); + } catch { + return; + } + const hostname = url.hostname.toLowerCase(); + if ( + url.protocol !== "https:" || + isDangerousHost(hostname) || + !ROUTER_API_BASE_HOST_ALLOWLIST.has(hostname) + ) { + findings.push({ + path: `/models/${index}/api_base`, + host: apiBase, + }); + } + }); + + return findings; +} + /** * Entry point: validate all config files (or a single file via --file/--schema flags) * against their JSON Schemas, then run the dangerous-host semantic check. @@ -247,7 +339,7 @@ function main(): void { const schemaErrors = !valid && validate.errors ? validate.errors.length : 0; // Semantic check: walk the parsed doc and reject catch-all hosts. // Runs regardless of schema outcome so operators see all issues at once. - const dangerous = findDangerousHosts(data); + const dangerous = [...findDangerousHosts(data), ...findDangerousRouterApiBases(data)]; if (schemaErrors > 0 || dangerous.length > 0) { console.error(`FAIL: ${file}`); @@ -258,8 +350,8 @@ function main(): void { } for (const finding of dangerous) { console.error( - ` ${finding.path}: host "${finding.host}" grants access to any destination — ` + - `use a specific hostname (subdomain wildcards like "*.example.com" are allowed)`, + ` ${finding.path}: host "${finding.host}" is not allowed — ` + + `use a specific public hostname (subdomain wildcards like "*.example.com" are allowed for policy hosts)`, ); } totalErrors += schemaErrors + dangerous.length; @@ -279,7 +371,14 @@ function main(): void { } // Export for unit tests without re-running main(). -export { DANGEROUS_HOSTS, isDangerousHost, findDangerousHosts }; +export { + DANGEROUS_HOSTS, + ROUTER_API_BASE_HOST_ALLOWLIST, + isDangerousHost, + findDangerousHosts, + findDangerousRouterApiBases, + discoverTargets, +}; // Only run main() when invoked directly (skip on test `import`). if ( diff --git a/src/lib/cli/command-display-metadata.test.ts b/src/lib/cli/command-display-metadata.test.ts index 551c100c8bf..823abe99b0b 100644 --- a/src/lib/cli/command-display-metadata.test.ts +++ b/src/lib/cli/command-display-metadata.test.ts @@ -1,23 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import path from "node:path"; - import { Config as OclifConfig } from "@oclif/core"; import { describe, expect, it } from "vitest"; import { getRegisteredOclifCommandsMetadata } from "./oclif-metadata"; import { COMMANDS, visibleCommands } from "./command-registry"; -function* walkTsFiles(dir: string): Generator { - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) yield* walkTsFiles(fullPath); - else if (entry.isFile() && entry.name.endsWith(".ts") && !entry.name.endsWith(".test.ts")) yield fullPath; - } -} - describe("public command display metadata", () => { it("loads public display entries for root help and docs checks", () => { expect(COMMANDS.length).toBeGreaterThan(0); @@ -48,28 +37,4 @@ describe("public command display metadata", () => { expect(invalid).toEqual([]); }); - it("keeps public command discovery wrappers free of display metadata", () => { - const commandFiles = [...walkTsFiles(path.join(process.cwd(), "src", "commands"))].filter( - (file) => !file.includes(`${path.sep}internal${path.sep}`), - ); - const wrappersWithDisplayHelpers = commandFiles - .filter((file) => fs.readFileSync(file, "utf-8").includes("withCommandDisplay")) - .map((file) => path.relative(process.cwd(), file)); - - expect(wrappersWithDisplayHelpers).toEqual([]); - }); - - it("keeps non-internal command discovery files independent from legacy lib command re-exports", () => { - const commandFiles = [...walkTsFiles(path.join(process.cwd(), "src", "commands"))].filter( - (file) => !file.includes(`${path.sep}internal${path.sep}`), - ); - const legacyCommandReExports = commandFiles - .filter((file) => { - const body = fs.readFileSync(file, "utf-8"); - return /export \{ default \} from "[^"]*\/lib\/commands\//.test(body); - }) - .map((file) => path.relative(process.cwd(), file)); - - expect(legacyCommandReExports).toEqual([]); - }); }); diff --git a/src/lib/cli/oclif-pattern-discovery.test.ts b/src/lib/cli/oclif-pattern-discovery.test.ts index 8c360fe4a7c..aa8a4c8315e 100644 --- a/src/lib/cli/oclif-pattern-discovery.test.ts +++ b/src/lib/cli/oclif-pattern-discovery.test.ts @@ -7,33 +7,19 @@ import path from "node:path"; import { Config as OclifConfig } from "@oclif/core"; import { describe, expect, it } from "vitest"; -const COMMANDS_ROOT = path.join(process.cwd(), "src", "commands"); - -function expectedCommandIdsFromSourceCommands(dir = COMMANDS_ROOT, prefix = ""): string[] { - const ids: string[] = []; - for (const entry of fs.readdirSync(dir).sort()) { - const absolute = path.join(dir, entry); - const relative = path.join(prefix, entry); - const stat = fs.statSync(absolute); - if (stat.isDirectory()) { - ids.push(...expectedCommandIdsFromSourceCommands(absolute, relative)); - continue; - } - if (!entry.endsWith(".ts") || entry.endsWith(".test.ts")) continue; - const parsed = path.parse(relative); - const topics = parsed.dir.split(path.sep).filter(Boolean); - const command = parsed.name === "index" ? null : parsed.name; - ids.push([...topics, command].filter(Boolean).join(":")); - } - return ids.sort(); -} - describe("oclif pattern command discovery", () => { - it("discovers every command id from src/commands", async () => { + it("discovers representative command ids from oclif's pattern config", async () => { const config = await OclifConfig.load(process.cwd()); const discoveredIds = config.commands.map((command) => command.id).sort(); - expect(discoveredIds).toEqual(expectedCommandIdsFromSourceCommands()); + expect(discoveredIds).toEqual( + expect.arrayContaining([ + "onboard", + "sandbox:status", + "sandbox:channels:start", + "inference:get", + ]), + ); }); it("does not rely on the removed compatibility command index", () => { diff --git a/src/lib/cli/public-argv-translation-boundaries.test.ts b/src/lib/cli/public-argv-translation-boundaries.test.ts deleted file mode 100644 index 4da7077919c..00000000000 --- a/src/lib/cli/public-argv-translation-boundaries.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import path from "node:path"; - -import { describe, expect, it } from "vitest"; - -function importSpecifiersFor(repoPath: string): string[] { - const source = fs.readFileSync(path.join(process.cwd(), repoPath), "utf-8"); - const specifiers: string[] = []; - const importPattern = /import(?:\s+type)?\s+(?:[^";]+?\s+from\s+)?["']([^"']+)["']/g; - for (const match of source.matchAll(importPattern)) { - specifiers.push(match[1]); - } - return specifiers; -} - -describe("public argv translation boundaries", () => { - it("keeps public argv translation independent from runtime dispatch side effects", () => { - expect(importSpecifiersFor("src/lib/cli/public-argv-translation.ts").sort()).toEqual([ - "./oclif-metadata", - "./public-route-metadata", - ]); - }); - - it("keeps public dispatch responsible for registry recovery and oclif execution", () => { - const dispatchImports = importSpecifiersFor("src/lib/cli/public-dispatch.ts"); - expect(dispatchImports).toEqual( - expect.arrayContaining([ - "./argv-normalizer", - "./public-argv-translation", - ]), - ); - - const translationImports = importSpecifiersFor("src/lib/cli/public-argv-translation.ts"); - expect(translationImports).not.toContain("./oclif-runner"); - expect(translationImports).not.toContain("../state/registry"); - expect(translationImports).not.toContain("../registry-recovery-action"); - }); -}); diff --git a/src/lib/core/shell-quote.ts b/src/lib/core/shell-quote.ts index 55b88d51623..3eebed008d6 100644 --- a/src/lib/core/shell-quote.ts +++ b/src/lib/core/shell-quote.ts @@ -5,6 +5,6 @@ * Shell-quote a value for safe interpolation into bash -c strings. * Wraps in single quotes and escapes embedded single quotes. */ -export function shellQuote(value: string): string { +export function shellQuote(value: unknown): string { return `'${String(value).replace(/'/g, `'\\''`)}'`; } diff --git a/src/lib/messaging/manifest/types.test.ts b/src/lib/messaging/manifest/types.test.ts index 138beddbf86..f8914746085 100644 --- a/src/lib/messaging/manifest/types.test.ts +++ b/src/lib/messaging/manifest/types.test.ts @@ -1,9 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { readFileSync, readdirSync, statSync } from "node:fs"; -import path from "node:path"; - import { describe, expect, it } from "vitest"; import type { @@ -257,22 +254,6 @@ function findFunctionPaths(value: unknown, prefix = "$"): string[] { return []; } -function collectProductionFiles(dir: string): string[] { - return readdirSync(dir).flatMap((entry) => { - const absolute = path.join(dir, entry); - const stats = statSync(absolute); - if (stats.isDirectory()) return collectProductionFiles(absolute); - if (absolute.endsWith(".ts") && !absolute.endsWith(".test.ts")) return [absolute]; - return []; - }); -} - -function collectModuleSpecifiers(source: string): string[] { - const pattern = - /(?:import|export)\s+(?:type\s+)?(?:[^"']*?\s+from\s+)?["']([^"']+)["']/g; - return [...source.matchAll(pattern)].map((match) => match[1] ?? ""); -} - describe("messaging manifest type contracts", () => { it("serializes representative manifests without losing required fields", () => { const parsedTelegram = jsonRoundTrip(telegramManifest); @@ -311,30 +292,7 @@ describe("messaging manifest type contracts", () => { expect(wechatHookManifest.hooks[0]?.handler).toBe("wechat.ilinkLogin"); }); - it("keeps the new production module isolated from side-effect layers", () => { - const moduleRoot = path.join(import.meta.dirname, ".."); - const files = collectProductionFiles(moduleRoot); - const forbiddenFragments = [ - "gateway", - "state/registry", - "credentials", - "node:fs", - "node:child_process", - "child_process", - "adapters/openshell", - "src/commands", - "lib/actions", - ]; - - for (const file of files) { - const source = readFileSync(file, "utf8"); - const specifiers = collectModuleSpecifiers(source); - for (const fragment of forbiddenFragments) { - expect( - specifiers.some((specifier) => specifier.includes(fragment)), - `${path.relative(moduleRoot, file)} imports ${fragment}`, - ).toBe(false); - } - } - }); + // Import-layer isolation for the production manifest modules is enforced by + // scripts/checks/layer-import-boundaries.ts. Keep this unit test focused on + // manifest serialization and type contracts rather than walking source files. }); diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 0289b07afbd..38e22156770 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -23,6 +23,20 @@ import { prepareInitialSandboxCreatePolicy, } from "./initial-policy"; +const BASE_POLICY_FIXTURE = ` +version: 1 +filesystem_policy: + read_only: + - /usr + - /proc + read_write: + - /tmp +network_policies: + managed_inference: + name: managed_inference + endpoints: [] +`; + const tmpRoots: string[] = []; function tmpPolicy(content: string): string { @@ -41,12 +55,8 @@ afterEach(() => { describe("initial sandbox policy helpers", () => { it("removes /proc from direct GPU create policy so OpenShell can own GPU enrichment", () => { - const basePolicy = fs.readFileSync( - path.join(import.meta.dirname, "..", "..", "..", "nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"), - "utf-8", - ); - const gpuPolicy = buildDirectGpuPolicyYaml(basePolicy); - const baseDoc = YAML.parse(basePolicy); + const gpuPolicy = buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE); + const baseDoc = YAML.parse(BASE_POLICY_FIXTURE); const gpuDoc = YAML.parse(gpuPolicy); // /proc is added at runtime by OpenShell's GPU enrichment; @@ -58,11 +68,7 @@ describe("initial sandbox policy helpers", () => { }); it("adds /proc read-write when Docker GPU patch must own GPU enrichment", () => { - const basePolicy = fs.readFileSync( - path.join(import.meta.dirname, "..", "..", "..", "nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"), - "utf-8", - ); - const gpuPolicy = buildDirectGpuPolicyYaml(basePolicy, { procReadWrite: true }); + const gpuPolicy = buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE, { procReadWrite: true }); const gpuDoc = YAML.parse(gpuPolicy); expect(gpuDoc.filesystem_policy.read_only).not.toContain("/proc"); diff --git a/src/lib/runner.ts b/src/lib/runner.ts index 6afe71df059..4ea1ba57184 100644 --- a/src/lib/runner.ts +++ b/src/lib/runner.ts @@ -11,13 +11,12 @@ import { NAME_ALLOWED_FORMAT, NAME_MAX_LENGTH } from "./name-validation"; const { spawnSync } = require("child_process"); const path = require("path"); const { detectDockerHost } = require("./platform"); +const { shellQuote } = require("./core/shell-quote") as typeof import("./core/shell-quote"); const { buildSubprocessEnv } = require("./subprocess-env") as typeof import("./subprocess-env"); const ROOT = path.resolve(__dirname, "..", ".."); const SCRIPTS = path.join(ROOT, "scripts"); -type RunnerScalar = string | number | boolean | null | undefined; - type RunnerOptions = SpawnSyncOptions & { ignoreError?: boolean; suppressOutput?: boolean; @@ -310,14 +309,6 @@ function runCaptureEx(cmd: readonly string[], opts: Omit { } }); - it("keeps oclif flexible taxonomy enabled for space-separated native commands", () => { - const packageJson = JSON.parse(fs.readFileSync("package.json", "utf-8")) as { - oclif?: { flexibleTaxonomy?: boolean; topicSeparator?: string }; - }; - - expect(packageJson.oclif?.flexibleTaxonomy).toBe(true); - expect(packageJson.oclif?.topicSeparator).toBe(" "); - }); it("uses the alias binary name in native oclif help", () => { const result = spawnSync( diff --git a/test/e2e-advisor-dispatch.test.ts b/test/e2e-advisor-dispatch.test.ts index b61aa956e9c..a8b7e31babf 100644 --- a/test/e2e-advisor-dispatch.test.ts +++ b/test/e2e-advisor-dispatch.test.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import path from "node:path"; import { describe, expect, it } from "vitest"; import { @@ -13,7 +11,21 @@ import { validateGitRef, } from "../tools/e2e-advisor/dispatch.mts"; -const ROOT = path.resolve(import.meta.dirname, ".."); +const NIGHTLY_E2E_WORKFLOW_FIXTURE = ` +jobs: + network-policy-e2e: + if: github.event_name != 'workflow_dispatch' || inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',network-policy-e2e,') + steps: [] + cloud-e2e: + if: github.event_name != 'workflow_dispatch' || inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',cloud-e2e,') + steps: [] + report-to-pr: + steps: [] + notify-on-failure: + steps: [] + scorecard: + steps: [] +`; function pullRequest(authorAssociation = "MEMBER", overrides = {}) { return { @@ -33,7 +45,7 @@ function pullRequest(authorAssociation = "MEMBER", overrides = {}) { } function nightlyWorkflowText(): string { - return fs.readFileSync(path.join(ROOT, ".github/workflows/nightly-e2e.yaml"), "utf8"); + return NIGHTLY_E2E_WORKFLOW_FIXTURE; } function advisorResult(job = "network-policy-e2e") { @@ -87,10 +99,7 @@ describe("E2E advisor auto-dispatch planning", () => { }); it("dispatches all required jobs without applying the retired max-jobs cap", () => { - const workflowText = fs.readFileSync( - path.join(ROOT, ".github/workflows/nightly-e2e.yaml"), - "utf8", - ); + const workflowText = nightlyWorkflowText(); const plan = planAutoDispatch({ result: { confidence: "high", diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index 278ca69c425..83861004030 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -1068,18 +1068,6 @@ describe("Brev GPU runtime setup", () => { ); }); - it("runs Docker GPU sandbox inference proof with the OpenShell proxy env", () => { - const script = fs.readFileSync(path.join(REPO_DIR, "test/e2e/test-gpu-e2e.sh"), "utf-8"); - - expect(script).toContain('[[ "$SANDBOX_INFERENCE_URL" == https://inference.local/* ]]'); - expect(script).toContain('SANDBOX_INFERENCE_DOCKER_EXEC_ENV=('); - expect(script).toContain('--env "HTTPS_PROXY=${INFERENCE_PROXY_URL}"'); - expect(script).toContain('INFERENCE_NO_PROXY="localhost,127.0.0.1,::1,${INFERENCE_PROXY_HOST}"'); - expect(script).toContain("curl -skS --max-time 90"); - expect(script).toContain( - 'docker exec "${SANDBOX_INFERENCE_DOCKER_EXEC_ENV[@]}" "$sandbox_container_id"', - ); - }); }); describe.runIf(hasRequiredVars && hasAuthenticatedBrev)("Brev E2E", () => { diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts index 9d3a89a6835..5f72e490549 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts @@ -991,20 +991,6 @@ describe("Phase 1.E install dispatcher splits", () => { expect(r.stdout + r.stderr).not.toMatch(/install-curl|install-ollama|install-launchable/); }); - it("repo_current_install_should_use_full_cli_build_script", () => { - const script = fs.readFileSync(path.join(INSTALL_DIR, "repo-current.sh"), "utf8"); - expect(script).toContain("npm run build:cli"); - expect(script).not.toContain("./node_modules/.bin/tsc -p tsconfig.src.json"); - expect(script).not.toContain("./node_modules/.bin/tsc -p nemoclaw-blueprint/tsconfig.json"); - }); - - it("repo_current_install_should_verify_generated_oclif_metadata", () => { - const script = fs.readFileSync(path.join(INSTALL_DIR, "repo-current.sh"), "utf8"); - const buildScript = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, "package.json"), "utf8")).scripts?.["build:cli"] ?? ""; - expect(buildScript).toContain("generate-oclif-metadata-manifest.js"); - expect(script).toContain("dist/lib/cli/oclif-command-metadata.generated.json"); - }); - it("install_should_dispatch_to_install_curl_helper_for_public_installer_profile", () => { const r = dispatchDryRun("public-installer"); expect(r.status, r.stderr).toBe(0); diff --git a/test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts b/test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts index 88382fa2726..42d6eabf0e0 100644 --- a/test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts @@ -20,25 +20,7 @@ import { loadMetadataFromDir } from "../runtime/resolver/load.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const E2E_DIR = path.join(REPO_ROOT, "test/e2e"); const VALIDATION_SUITES_DIR = path.join(E2E_DIR, "validation_suites"); -const README_PATH = path.join(E2E_DIR, "docs", "README.md"); - describe("Phase 11 final hygiene", () => { - it("e2e_readme_should_document_scenario_runner", () => { - expect(fs.existsSync(README_PATH)).toBe(true); - const raw = fs.readFileSync(README_PATH, "utf8"); - // Key developer-facing concepts must be documented. - expect(raw).toMatch(/setup scenario/i); - expect(raw).toMatch(/expected state/i); - expect(raw).toMatch(/suite/i); - expect(raw).toMatch(/assertion ID|PASS: /i); - expect(raw).toMatch(/scenario coverage report/i); - expect(raw).toMatch(/issue #3588/); - expect(raw).toMatch(/run-scenario\.sh/); - expect(raw).toMatch(/run-suites\.sh/); - // Adding-a-scenario guidance must exist. - expect(raw).toMatch(/adding a new setup scenario|how to add/i); - }); - it("all_suite_scripts_should_exist", () => { const meta = loadMetadataFromDir(E2E_DIR); const missing: string[] = []; @@ -83,13 +65,4 @@ describe("Phase 11 final hygiene", () => { expect(problems, problems.join("\n")).toEqual([]); }); - it("should_not_reference_retired_e2e_entrypoints", () => { - // At this point we have not retired any entrypoints. This guard test - // asserts that `run-scenario.sh` and `run-suites.sh` are the canonical - // new entrypoints documented in the README, so that when old scripts - // are retired in a follow-up, the guard is ready to be tightened. - const raw = fs.readFileSync(README_PATH, "utf8"); - expect(raw).toMatch(/run-scenario\.sh/); - expect(raw).toMatch(/run-suites\.sh/); - }); }); diff --git a/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts b/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts index 0bf5cdca2f7..61ce48b429e 100644 --- a/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-scenario-additional-families.test.ts @@ -15,7 +15,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import yaml from "js-yaml"; import { loadMetadataFromDir } from "../runtime/resolver/load.ts"; import { resolveScenario } from "../runtime/resolver/plan.ts"; @@ -45,9 +44,7 @@ function planOnly(scenarioId: string): { stdout: string; stderr: string; status: describe("Issue 3812: inference/provider suite families", () => { it("test_should_route_inference_suite_families_to_domain_specific_steps", () => { - const suites = yaml.load(fs.readFileSync(path.join(E2E_DIR, "validation_suites/suites.yaml"), "utf8")) as { - suites: Record; - }; + const { suites } = loadMetadataFromDir(E2E_DIR); for (const family of ["inference-routing", "inference-switch", "kimi-compatibility", "ollama-auth-proxy", "model-router"]) { const scripts = suites.suites[family]?.steps?.map((step) => step.script ?? "") ?? []; expect(scripts.length, family).toBeGreaterThan(0); diff --git a/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts b/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts index 5860733e4ad..11b4655d655 100644 --- a/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-scenarios-workflow.test.ts @@ -1,93 +1,62 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, it, expect } from "vitest"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; -import yaml from "js-yaml"; -const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); -const WORKFLOW_PATH = path.join(REPO_ROOT, ".github/workflows/e2e-scenarios.yaml"); +import { describe, expect, it } from "vitest"; -type AnyRecord = Record; -type WorkflowStep = { - id?: string; - if?: string; - name?: string; - run?: string; - uses?: string; - with?: AnyRecord; -}; +import { validateE2eScenariosWorkflowBoundary } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; -function loadWorkflow(): AnyRecord { - expect(fs.existsSync(WORKFLOW_PATH), `workflow missing at ${WORKFLOW_PATH}`).toBe(true); - const raw = fs.readFileSync(WORKFLOW_PATH, "utf8"); - return yaml.load(raw) as AnyRecord; -} - -function workflowJob(workflow: AnyRecord, jobId: string): AnyRecord { - const jobs = workflow.jobs as Record | undefined; - const job = jobs?.[jobId]; - expect(job, `missing workflow job ${jobId}`).toBeTruthy(); - return job ?? {}; -} - -function workflowSteps(workflow: AnyRecord, jobId: string): WorkflowStep[] { - const value = workflowJob(workflow, jobId).steps; - expect(Array.isArray(value), `workflow job ${jobId} missing steps`).toBe(true); - return (Array.isArray(value) ? value : []) as WorkflowStep[]; -} - -function namedStep(workflow: AnyRecord, jobId: string, stepName: string): WorkflowStep { - const step = workflowSteps(workflow, jobId).find((candidate) => candidate.name === stepName); - expect(step, `missing step '${stepName}' in ${jobId}`).toBeTruthy(); - return step ?? {}; -} - -function uploadArtifactStep(workflow: AnyRecord, jobId: string, stepName: string): WorkflowStep { - const step = namedStep(workflow, jobId, stepName); - expect(step.uses).toMatch(/^actions\/upload-artifact@(?:v4|[a-f0-9]{40})$/); - return step; -} - -describe("e2e-scenarios workflow", () => { - it("e2e_scenarios_workflow_should_have_dispatch_inputs", () => { - const wf = loadWorkflow(); - // YAML `on:` parses as the literal key "true" in some parsers — handle both. - const on = (wf.on ?? wf[true as unknown as string]) as AnyRecord | undefined; - expect(on, "workflow missing 'on' trigger").toBeTruthy(); - const dispatch = on?.workflow_dispatch as AnyRecord | undefined; - expect(dispatch, "workflow missing workflow_dispatch").toBeTruthy(); - const inputs = dispatch?.inputs as AnyRecord | undefined; - expect(inputs).toBeTruthy(); - expect(inputs).toHaveProperty("scenario"); - expect(inputs).not.toHaveProperty("plan_only"); - expect(inputs).toHaveProperty("suite_filter"); - }); - - it("e2e_scenarios_workflow_should_call_run_scenario_without_plan_only", () => { - const wf = loadWorkflow(); - const runScenario = namedStep(wf, "run-scenario", "Run scenario"); - expect(runScenario.run).toContain("bash test/e2e/runtime/run-scenario.sh"); - expect(runScenario.run).not.toContain("--plan-only"); - }); - - it("e2e_scenarios_workflow_should_upload_artifacts", () => { - const wf = loadWorkflow(); - const upload = uploadArtifactStep(wf, "run-scenario", "Upload scenario artifacts"); - expect(upload.with?.name).toBe("e2e-scenario-${{ inputs.scenario }}"); - expect(upload.with?.path).toContain(".e2e/"); - expect(upload.with?.["include-hidden-files"]).toBe(true); +describe("e2e-scenarios workflow boundary", () => { + it("keeps scenario execution manual/reusable and artifact-safe", () => { + expect(validateE2eScenariosWorkflowBoundary()).toEqual([]); }); - it("e2e_scenarios_workflow_should_be_manual_only", () => { - const wf = loadWorkflow(); - const on = (wf.on ?? wf[true as unknown as string]) as AnyRecord | undefined; - expect(on).toBeTruthy(); - const keys = Object.keys(on ?? {}); - // Manual-only: must not trigger on push, pull_request, or schedule. - expect(keys).not.toContain("push"); - expect(keys).not.toContain("pull_request"); - expect(keys).not.toContain("schedule"); + it("flags unsafe trigger and contract regressions", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-scenarios-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + fs.writeFileSync( + workflowPath, + ` +"on": + pull_request_target: {} +permissions: + contents: write +jobs: + run-scenario: + runs-on: ubuntu-latest + steps: + - name: Run scenario + run: bash test/e2e/runtime/run-scenario.sh --plan-only + - name: Upload scenario artifacts + uses: actions/upload-artifact@v4 + with: + name: bad-name + path: test/e2e/logs/ +`, + ); + + try { + const errors = validateE2eScenariosWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "workflow must support workflow_dispatch", + "workflow must support workflow_call", + "workflow must not run on pull_request_target", + "workflow permissions.contents must be read", + "workflow missing resolve-runner job", + "run-scenario job must use the resolved runner output", + "Run scenario step must not use retired --plan-only flag", + "run-scenario job missing step: Run scenario in WSL", + "artifact upload name must include the scenario input", + "artifact upload must include hidden .e2e files", + "artifact upload path must include .e2e/", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } }); }); diff --git a/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts b/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts index b797cca14c0..a51eeaf947b 100644 --- a/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts @@ -6,11 +6,8 @@ import { spawnSync, type SpawnSyncReturns } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import yaml from "js-yaml"; - const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const RUN_SUITES = path.join(REPO_ROOT, "test/e2e/runtime/run-suites.sh"); -const SUITES_YAML = path.join(REPO_ROOT, "test/e2e/validation_suites/suites.yaml"); function runSuites(args: string[], env: Record = {}): SpawnSyncReturns { return spawnSync("bash", [RUN_SUITES, ...args], { @@ -203,16 +200,6 @@ describe("run-suites.sh", () => { } }); - it("rebuild_and_upgrade_suites_should_resolve_to_domain_specific_steps", () => { - const doc = yaml.load(fs.readFileSync(SUITES_YAML, "utf8")) as { suites: Record }> }; - for (const suiteId of ["rebuild", "upgrade"]) { - const scripts = doc.suites[suiteId].steps.map((step) => step.script); - expect(scripts.length).toBeGreaterThan(0); - expect(scripts.every((script) => script.startsWith("rebuild_upgrade/"))).toBe(true); - expect(scripts.some((script) => script.startsWith("smoke/"))).toBe(false); - } - }); - it("rebuild_and_upgrade_suites_should_emit_stable_assertion_ids_in_dry_run", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-suite-")); try { diff --git a/test/fetch-guard-patch-regression.test.ts b/test/fetch-guard-patch-regression.test.ts index 418afcf3ee3..b6da45783da 100644 --- a/test/fetch-guard-patch-regression.test.ts +++ b/test/fetch-guard-patch-regression.test.ts @@ -98,10 +98,25 @@ function createSedWrapper(tmp: string): string { [ "#!/usr/bin/env bash", "set -euo pipefail", - 'if [ "${1:-}" = "-i" ] && [ "${2:-}" = "-E" ]; then', - " expr=$3", - " shift 3", - ' for file in "$@"; do perl -0pi -e "$expr" "$file"; done', + 'if [ "${1:-}" = "-i" ]; then', + " extended=0", + ' if [ "${2:-}" = "-E" ]; then', + " extended=1", + " expr=$3", + " shift 3", + " else", + " expr=$2", + " shift 2", + " fi", + ' for file in "$@"; do', + " tmp=$(mktemp)", + ' if [ "$extended" = "1" ]; then', + ' /usr/bin/sed -E "$expr" "$file" > "$tmp"', + " else", + ' /usr/bin/sed "$expr" "$file" > "$tmp"', + " fi", + ' mv "$tmp" "$file"', + " done", " exit 0", "fi", 'exec /usr/bin/sed "$@"', @@ -111,10 +126,15 @@ function createSedWrapper(tmp: string): string { return fakeBin; } -function runFetchGuardPatchBlock(dist: string, tmp: string, version = "2026.5.18") { +function runDockerfilePatchBlock( + dist: string, + tmp: string, + endMarker: string, + version = "2026.5.18", +) { const command = dockerRunCommandBetween( "# Patch OpenClaw media fetch for proxy-only sandbox", - "# --- Patch 3: follow symlinks in plugin-install path checks (#2203)", + endMarker, ).replaceAll("/usr/local/lib/node_modules/openclaw/dist", dist); const scriptPath = path.join(tmp, "patch.sh"); fs.writeFileSync( @@ -134,18 +154,21 @@ function runFetchGuardPatchBlock(dist: string, tmp: string, version = "2026.5.18 }); } +function runFetchGuardPatchBlock(dist: string, tmp: string, version = "2026.5.18") { + return runDockerfilePatchBlock( + dist, + tmp, + "# --- Patch 3: follow symlinks in plugin-install path checks (#2203)", + version, + ); +} + describe("fetch-guard patch regression guard", () => { it("fails the image build when the NemoClaw OpenClaw plugin cannot install", () => { const command = dockerRunCommandBetween( "# Install NemoClaw plugin into OpenClaw", "# SECURITY: Clear any gateway auth token", ); - expect(command).toContain("openclaw plugins install /opt/nemoclaw"); - expect(command).toContain("openclaw plugins enable nemoclaw"); - expect(command).toContain("openclaw plugins inspect nemoclaw --json"); - expect(command).not.toContain("--dangerously-force-unsafe-install"); - expect(command).not.toMatch(/openclaw plugins install \/opt\/nemoclaw[^&|]*(?:\|\|\s*true|2>&1)/); - const script = [ "openclaw() {", ' if [ "${1:-} ${2:-} ${3:-}" = "plugins install /opt/nemoclaw" ]; then return 42; fi', @@ -186,6 +209,107 @@ describe("fetch-guard patch regression guard", () => { ); }); + it("applies the Dockerfile OpenClaw compatibility patch block to executable fixtures", async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-patches-")); + const dist = path.join(tmp, "dist"); + fs.mkdirSync(dist, { recursive: true }); + fs.writeFileSync(path.join(tmp, "package.json"), '{"type":"module"}\n'); + const symlinkTarget = path.join(tmp, "real-install-base"); + const symlinkBase = path.join(tmp, "install-base-link"); + fs.mkdirSync(symlinkTarget); + fs.symlinkSync(symlinkTarget, symlinkBase); + + const fetchGuardPath = path.join(dist, "fetch-guard-fixture.js"); + const installSafePath = path.join(dist, "install-safe-path-fixture.js"); + const installPackageDirPath = path.join(dist, "install-package-dir-fixture.js"); + const clientPath = path.join(dist, "client-fixture.js"); + const serverPath = path.join(dist, "server.impl-fixture.js"); + + fs.writeFileSync( + fetchGuardPath, + [ + "const withStrictGuardedFetchMode = Symbol('strict');", + "const withTrustedEnvProxyGuardedFetchMode = Symbol('trusted');", + "globalThis.proxyChecks = [];", + "async function assertExplicitProxyAllowed(proxyUrl) { globalThis.proxyChecks.push(proxyUrl); throw new Error('proxy rejected'); }", + "globalThis.assertExplicitProxyAllowed = assertExplicitProxyAllowed;", + "export { withStrictGuardedFetchMode as a, withTrustedEnvProxyGuardedFetchMode as b };", + "", + ].join("\n"), + ); + fs.writeFileSync( + installSafePath, + [ + 'import fs from "node:fs/promises";', + "export async function acceptsBaseDir(baseDir) {", + " const baseLstat = await fs.lstat(baseDir);", + " return baseLstat.isDirectory();", + "}", + "", + ].join("\n"), + ); + fs.writeFileSync( + installPackageDirPath, + [ + 'import fs from "node:fs/promises";', + "export async function assertInstallBaseStable(params) {", + " const baseLstat = await fs.lstat(params.installBaseDir);", + " if (baseLstat.isSymbolicLink()) throw new Error('symlink');", + " if (await fs.realpath(params.installBaseDir) !== params.expectedRealPath) throw new Error('drift');", + " return baseLstat.isDirectory();", + "}", + "", + ].join("\n"), + ); + fs.writeFileSync(clientPath, "export const DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 15e3;\n"); + fs.writeFileSync(serverPath, "export const DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 15e3;\n"); + + try { + const patch = runDockerfilePatchBlock( + dist, + tmp, + "# Patch OpenClaw chat.send gateway behavior", + CURRENT_REVIEWED_OPENCLAW_PATCH_CLASSIFIER_VERSION, + ); + expect(patch.status, `${patch.stdout}${patch.stderr}`).toBe(0); + expect(patch.stdout).toContain("Patch 1 applied"); + expect(patch.stdout).toContain("Patch 2 applied"); + + const fetchGuard = await import(`${fetchGuardPath}?${Date.now()}`); + expect(fetchGuard.a).toBe(fetchGuard.b); + const previousSandboxEnv = process.env.OPENSHELL_SANDBOX; + process.env.OPENSHELL_SANDBOX = "1"; + try { + await (globalThis as any).assertExplicitProxyAllowed("http://10.200.0.1:3128"); + } finally { + if (previousSandboxEnv === undefined) { + delete process.env.OPENSHELL_SANDBOX; + } else { + process.env.OPENSHELL_SANDBOX = previousSandboxEnv; + } + } + expect((globalThis as any).proxyChecks).toEqual([]); + + const installSafe = await import(`${installSafePath}?${Date.now()}`); + await expect(installSafe.acceptsBaseDir(symlinkBase)).resolves.toBe(true); + + const installPackageDir = await import(`${installPackageDirPath}?${Date.now()}`); + await expect( + installPackageDir.assertInstallBaseStable({ + installBaseDir: symlinkBase, + expectedRealPath: fs.realpathSync(symlinkBase), + }), + ).resolves.toBe(true); + + const client = await import(`${clientPath}?${Date.now()}`); + const server = await import(`${serverPath}?${Date.now()}`); + expect(client.DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS).toBe(60_000); + expect(server.DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS).toBe(60_000); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("rewrites strict media fetch exports and makes proxy validation sandbox-aware", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fetch-guard-")); const dist = path.join(tmp, "dist"); @@ -233,108 +357,6 @@ if (globalThis.proxyChecks.length !== 0) throw new Error('sandbox proxy validati } }); - it("keeps the Dockerfile OpenClaw source-shape patches aligned with current dist", () => { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-patches-")); - const dist = path.join(tmp, "dist"); - fs.mkdirSync(dist, { recursive: true }); - fs.writeFileSync( - path.join(dist, "fetch-guard-test.js"), - [ - "const withStrictGuardedFetchMode = Symbol('strict');", - "const withTrustedEnvProxyGuardedFetchMode = Symbol('trusted');", - "async function assertExplicitProxyAllowed(proxyUrl) { throw new Error(proxyUrl); }", - "export { withStrictGuardedFetchMode as a, withTrustedEnvProxyGuardedFetchMode as b };", - "", - ].join("\n"), - ); - fs.writeFileSync( - path.join(dist, "install-safe-path-test.js"), - "const baseLstat = await fs.lstat(baseDir);\n", - ); - fs.writeFileSync( - path.join(dist, "install-package-dir-test.js"), - [ - "async function assertInstallBaseStable(params) {", - " const baseLstat = await fs.lstat(params.installBaseDir);", - " if (baseLstat.isSymbolicLink()) throw new Error('symlink');", - "}", - "", - ].join("\n"), - ); - fs.writeFileSync( - path.join(dist, "client-test.js"), - "const DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 15e3;\n", - ); - fs.writeFileSync( - path.join(dist, "server.impl-test.js"), - "const DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 15e3;\n", - ); - - const command = dockerRunCommandBetween( - "# Patch OpenClaw media fetch for proxy-only sandbox", - "# Patch OpenClaw chat.send gateway behavior", - ).replaceAll("/usr/local/lib/node_modules/openclaw/dist", dist); - const fakeBin = path.join(tmp, "bin"); - fs.mkdirSync(fakeBin); - const sedWrapper = path.join(fakeBin, "sed"); - fs.writeFileSync( - sedWrapper, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - "extended=0", - 'if [ "${1:-}" = "-i" ]; then', - ' if [ "${2:-}" = "-E" ]; then', - " extended=1", - " expr=$3", - " shift 3", - " else", - " expr=$2", - " shift 2", - " fi", - ' for file in "$@"; do', - " tmp=$(mktemp)", - ' if [ "$extended" = "1" ]; then', - ' /usr/bin/sed -E "$expr" "$file" > "$tmp"', - " else", - ' /usr/bin/sed "$expr" "$file" > "$tmp"', - " fi", - ' mv "$tmp" "$file"', - " done", - " exit 0", - "fi", - 'exec /usr/bin/sed "$@"', - ].join("\n"), - { mode: 0o755 }, - ); - const scriptPath = path.join(tmp, "patch-all.sh"); - fs.writeFileSync(scriptPath, ["#!/usr/bin/env bash", command].join("\n"), { mode: 0o700 }); - - try { - const patch = spawnSync("bash", [scriptPath], { - encoding: "utf-8", - env: { ...process.env, PATH: `${fakeBin}:${process.env.PATH || ""}` }, - timeout: 5000, - }); - expect(patch.status, `${patch.stdout}${patch.stderr}`).toBe(0); - const patched = fs - .readdirSync(dist) - .map((file) => fs.readFileSync(path.join(dist, file), "utf-8")); - expect(patched.join("\n")).not.toContain("DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 15e3"); - expect(patched.join("\n")).not.toContain("DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 1e4"); - expect(patched.join("\n").match(/DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 6e4/g)).toHaveLength( - 2, - ); - expect(fs.readFileSync(path.join(dist, "install-safe-path-test.js"), "utf-8")).toContain( - "const baseLstat = await fs.stat(baseDir)", - ); - expect(fs.readFileSync(path.join(dist, "install-package-dir-test.js"), "utf-8")).toContain( - "const baseLstat = await fs.stat(params.installBaseDir)", - ); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } - }); it("applies the proxy validator patch while the target function still exists", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fetch-guard-proxy-skip-")); diff --git a/test/hermes-sandbox-workflow.test.ts b/test/hermes-sandbox-workflow.test.ts deleted file mode 100644 index 58dadb08e49..00000000000 --- a/test/hermes-sandbox-workflow.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import path from "node:path"; -import { describe, expect, it } from "vitest"; - -const repoRoot = path.resolve(import.meta.dirname, ".."); - -describe("Hermes sandbox image workflow", () => { - it("checks the current generated config path copied into the image", () => { - const workflow = fs.readFileSync( - path.join(repoRoot, ".github/workflows/sandbox-images-and-e2e.yaml"), - "utf8", - ); - - expect(workflow).toContain("test -r /opt/nemoclaw-hermes-config/generate-config.ts"); - expect(workflow).not.toContain("/opt/nemoclaw-generate-config.ts"); - }); -}); diff --git a/test/install-preflight.test.ts b/test/install-preflight.test.ts index 0a2add05698..5336a00d4cb 100644 --- a/test/install-preflight.test.ts +++ b/test/install-preflight.test.ts @@ -529,10 +529,9 @@ exit 98 }); it("piped --help does not show the placeholder installer version", () => { - const result = spawnSync("bash", ["-s", "--", "--help"], { + const result = spawnSync("bash", ["-lc", `cat ${JSON.stringify(INSTALLER)} | bash -s -- --help`], { cwd: os.tmpdir(), encoding: "utf-8", - input: fs.readFileSync(INSTALLER, "utf-8"), }); expect(result.status).toBe(0); @@ -542,10 +541,9 @@ exit 98 }); it("piped --version omits the placeholder installer version", () => { - const result = spawnSync("bash", ["-s", "--", "--version"], { + const result = spawnSync("bash", ["-lc", `cat ${JSON.stringify(INSTALLER)} | bash -s -- --version`], { cwd: os.tmpdir(), encoding: "utf-8", - input: fs.readFileSync(INSTALLER, "utf-8"), }); expect(result.status).toBe(0); @@ -3646,9 +3644,8 @@ main() { }`, ); - const result = spawnSync("bash", ["-s", "--", "--version"], { + const result = spawnSync("bash", ["-lc", `cat ${JSON.stringify(rootInstaller)} | bash -s -- --version`], { cwd: repoLike, - input: fs.readFileSync(rootInstaller, "utf-8"), encoding: "utf-8", env: { ...process.env, diff --git a/test/layer-import-boundaries.test.ts b/test/layer-import-boundaries.test.ts index 9fa14079e88..95c648d59c7 100644 --- a/test/layer-import-boundaries.test.ts +++ b/test/layer-import-boundaries.test.ts @@ -41,6 +41,56 @@ describe("CLI layer import boundaries", () => { } }); + it("keeps messaging manifests isolated from side-effect layers", () => { + const fixture = path.join( + REPO_ROOT, + "src", + "lib", + "messaging", + "manifest", + `__boundary-fs-${process.pid}.ts`, + ); + try { + fs.writeFileSync(fixture, 'import { readFileSync } from "node:fs";\nexport const value = readFileSync;\n'); + const result = spawnSync(TSX, [BOUNDARY_SCRIPT], { + cwd: REPO_ROOT, + encoding: "utf-8", + }); + + expect(result.status).toBe(1); + expect(`${result.stdout}${result.stderr}`).toContain( + "messaging manifest modules must not import node:fs", + ); + } finally { + fs.rmSync(fixture, { force: true }); + } + }); + + it("blocks bare fs imports in messaging manifests", () => { + const fixture = path.join( + REPO_ROOT, + "src", + "lib", + "messaging", + "manifest", + `__boundary-bare-fs-${process.pid}.ts`, + ); + try { + fs.writeFileSync(fixture, 'import { readFile } from "fs/promises";\nexport const value = readFile;\n'); + const result = spawnSync(TSX, [BOUNDARY_SCRIPT], { + cwd: REPO_ROOT, + encoding: "utf-8", + }); + + expect(result.status).toBe(1); + expect(`${result.stdout}${result.stderr}`).toContain( + "messaging manifest modules must not import fs", + ); + } finally { + fs.rmSync(fixture, { force: true }); + } + }); + it("counts only classes that extend Command as oclif command classes", () => { const fixture = path.join(REPO_ROOT, "src", "commands", `__boundary-implements-${process.pid}.ts`); try { diff --git a/test/policies.test.ts b/test/policies.test.ts index 0bba7520fa3..d608dc49114 100644 --- a/test/policies.test.ts +++ b/test/policies.test.ts @@ -44,6 +44,17 @@ function requirePresetContent(content: string | null): string { return content; } +function parsePresetYaml(presetName: string): Record { + return YAML.parse(requirePresetContent(policies.loadPreset(presetName))) as Record; +} + +function parseRepoYaml(relativePath: string): Record { + return YAML.parse(fs.readFileSync(path.join(REPO_ROOT, relativePath), "utf-8")) as Record< + string, + any + >; +} + function runPolicyAdd( confirmAnswer: string, extraArgs: string[] = [], @@ -205,15 +216,7 @@ describe("policies", () => { // Apex and *.web.whatsapp.com (fallback nodes w1.web.whatsapp.com, // w2.web.whatsapp.com, ...) share the same shape so reconnects do // not surprise the operator. - const presetPath = path.join( - import.meta.dirname, - "..", - "nemoclaw-blueprint", - "policies", - "presets", - "whatsapp.yaml", - ); - const parsed = YAML.parse(fs.readFileSync(presetPath, "utf-8")); + const parsed = parsePresetYaml("whatsapp"); const endpoints: Array> = parsed?.network_policies?.whatsapp?.endpoints ?? []; @@ -238,15 +241,7 @@ describe("policies", () => { // wildcard keeps the preset future-proof without expanding trust // beyond Meta-controlled infrastructure. Mirrors the jira preset's // *.atlassian.net wildcard. - const presetPath = path.join( - import.meta.dirname, - "..", - "nemoclaw-blueprint", - "policies", - "presets", - "whatsapp.yaml", - ); - const parsed = YAML.parse(fs.readFileSync(presetPath, "utf-8")); + const parsed = parsePresetYaml("whatsapp"); const endpoints: Array> = parsed?.network_policies?.whatsapp?.endpoints ?? []; @@ -275,15 +270,7 @@ describe("policies", () => { // which Meta now rejects on pair. Scope is pinned to that single // file path with GET only so the rule does not turn into a general // raw.githubusercontent.com escape hatch. - const presetPath = path.join( - import.meta.dirname, - "..", - "nemoclaw-blueprint", - "policies", - "presets", - "whatsapp.yaml", - ); - const parsed = YAML.parse(fs.readFileSync(presetPath, "utf-8")); + const parsed = parsePresetYaml("whatsapp"); const endpoints: Array> = parsed?.network_policies?.whatsapp?.endpoints ?? []; @@ -1324,11 +1311,7 @@ exit 1 }); it("Hermes Discord REST mutations are scoped to discord.com", () => { - const content = fs.readFileSync( - path.join(REPO_ROOT, "agents/hermes/policy-additions.yaml"), - "utf8", - ); - const parsed = YAML.parse(content); + const parsed = parseRepoYaml("agents/hermes/policy-additions.yaml"); const networkPolicies = parsed.network_policies as Record< string, { @@ -1381,11 +1364,7 @@ exit 1 }); it("Hermes GitHub policy does not whitelist the absent gh CLI (#2179)", () => { - const content = fs.readFileSync( - path.join(REPO_ROOT, "agents/hermes/policy-additions.yaml"), - "utf8", - ); - const parsed = YAML.parse(content); + const parsed = parseRepoYaml("agents/hermes/policy-additions.yaml"); const githubPolicy = parsed.network_policies?.github as | { binaries?: Array<{ path?: string }> } | undefined; @@ -1423,22 +1402,14 @@ exit 1 // `brew install ` cannot extract bottles or manage the // Cellar/opt symlinks at runtime, and the brew preset's binary // whitelist becomes dead code. - const parsed = YAML.parse( - fs.readFileSync( - path.join(REPO_ROOT, "nemoclaw-blueprint/policies/openclaw-sandbox.yaml"), - "utf-8", - ), - ); + const parsed = parseRepoYaml("nemoclaw-blueprint/policies/openclaw-sandbox.yaml"); expect(parsed.filesystem_policy.read_write).toContain("/home/linuxbrew"); }); it("OpenClaw permissive policies preserve baseline read_write paths (#3916)", () => { - const baseline = YAML.parse( - fs.readFileSync( - path.join(REPO_ROOT, "nemoclaw-blueprint/policies/openclaw-sandbox.yaml"), - "utf-8", - ), - ) as { filesystem_policy?: { read_write?: string[] } }; + const baseline = parseRepoYaml("nemoclaw-blueprint/policies/openclaw-sandbox.yaml") as { + filesystem_policy?: { read_write?: string[] }; + }; const baselineReadWrite = baseline.filesystem_policy?.read_write ?? []; const permissivePolicyPaths = [ "nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml", @@ -1446,9 +1417,9 @@ exit 1 ]; for (const relativePath of permissivePolicyPaths) { - const parsed = YAML.parse( - fs.readFileSync(path.join(REPO_ROOT, relativePath), "utf-8"), - ) as { filesystem_policy?: { read_write?: string[] } }; + const parsed = parseRepoYaml(relativePath) as { + filesystem_policy?: { read_write?: string[] }; + }; expect(parsed.filesystem_policy?.read_write, relativePath).toEqual( expect.arrayContaining(baselineReadWrite), ); diff --git a/test/pr-review-advisor.test.ts b/test/pr-review-advisor.test.ts index d02d02757ea..1a7b00b2aea 100644 --- a/test/pr-review-advisor.test.ts +++ b/test/pr-review-advisor.test.ts @@ -5,13 +5,14 @@ import fs from "node:fs"; import path from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; import { afterEach, describe, expect, it, vi } from "vitest"; -import YAML from "yaml"; import { buildComment } from "../tools/pr-review-advisor/comment.mts"; import { buildSystemPrompt, classifyMonolithDelta, classifyTestDepth, normalizeReviewResult, readTrustedSecurityReviewSkill, renderDetailedReview, renderSummary } from "../tools/pr-review-advisor/analyze.mts"; import { githubGraphql } from "../tools/advisors/github.mts"; +import { validatePrReviewAdvisorWorkflowBoundary } from "../tools/pr-review-advisor/workflow-boundary.mts"; const ROOT = path.resolve(import.meta.dirname, ".."); + type ReviewMetadata = Parameters[1]; function metadata(overrides: Partial = {}): ReviewMetadata { @@ -40,6 +41,11 @@ function metadata(overrides: Partial = {}): ReviewMetadata { } as ReviewMetadata; } +function loadAdvisorSchema(): Record { + const schemaPath = path.join(ROOT, "tools", "pr-review-advisor", "schema.json"); + return JSON.parse(fs.readFileSync(schemaPath, "utf-8")) as Record; +} + function validResult(overrides = {}) { return { version: 1, @@ -146,7 +152,7 @@ describe("PR review advisor", () => { }); it("loads the checked-in security review skill into the advisor prompt", () => { - const schema = JSON.parse(fs.readFileSync(path.join(ROOT, "tools/pr-review-advisor/schema.json"), "utf8")); + const schema = loadAdvisorSchema(); const skill = readTrustedSecurityReviewSkill(); const prompt = buildSystemPrompt(schema, skill); @@ -282,7 +288,7 @@ describe("PR review advisor", () => { }); it("normalizes output that validates against the JSON schema", () => { - const schema = JSON.parse(fs.readFileSync(path.join(ROOT, "tools/pr-review-advisor/schema.json"), "utf8")); + const schema = loadAdvisorSchema(); const ajv = new Ajv2020({ strict: false }); const validate = ajv.compile(schema); const result = normalizeReviewResult(validResult(), metadata()); @@ -291,35 +297,63 @@ describe("PR review advisor", () => { expect(validate(result)).toBe(true); }); - it("keeps the workflow inside the same trusted-code boundary as other advisors", () => { - const workflow = YAML.parse( - fs.readFileSync(path.join(ROOT, ".github/workflows/pr-review-advisor.yaml"), "utf8"), - ); - const steps = workflow.jobs.review.steps; - const trustedCheckout = steps.find((step: { name?: string }) => - step.name === "Checkout trusted advisor code (main)" - ); - const prCheckout = steps.find((step: { name?: string }) => - step.name === "Checkout PR workspace (read-only data)" - ); - const installStep = steps.find((step: { name?: string }) => step.name === "Install Pi SDK"); - const analyzeStep = steps.find((step: { name?: string }) => step.name === "Run PR review advisor"); + it("keeps the workflow inside the trusted-code boundary", () => { + expect(validatePrReviewAdvisorWorkflowBoundary()).toEqual([]); + }); - expect(workflow.on).toHaveProperty("pull_request"); - expect(workflow.on).not.toHaveProperty("pull_request_target"); - expect(trustedCheckout).toMatchObject({ - with: { repository: "NVIDIA/NemoClaw", ref: "main", path: "advisor", "persist-credentials": false }, - }); - expect(prCheckout).toMatchObject({ with: { path: "pr-workdir", "persist-credentials": false } }); - const commentStep = steps.find((step: { name?: string }) => step.name === "Post PR review advisor comment"); + it("flags trusted-code boundary workflow regressions", () => { + const tmp = fs.mkdtempSync(path.join(ROOT, ".tmp-pr-advisor-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + fs.writeFileSync( + workflowPath, + ` +"on": + pull_request_target: {} +permissions: + contents: write +jobs: + review: + continue-on-error: true + steps: + - name: Checkout trusted advisor code (main) + uses: actions/checkout@v4 + with: + repository: NVIDIA/NemoClaw + ref: main + path: advisor + persist-credentials: true + - name: Checkout PR workspace (read-only data) + uses: actions/checkout@0123456789abcdef0123456789abcdef01234567 + with: + ref: refs/pull/\${{ github.event.pull_request.head.sha }}/merge + path: pr-workdir + persist-credentials: false +`, + ); - for (const step of steps.filter((step: { uses?: string }) => step.uses)) { - expect(step.uses).toMatch(/@[0-9a-f]{40}(?:\s*#.*)?$/); + try { + const errors = validatePrReviewAdvisorWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "workflow must run on pull_request, not only trusted-target events", + "workflow must not run untrusted PR code under pull_request_target", + "workflow permissions.contents must be read", + "review job must not be globally continue-on-error", + "PR checkout must use the pull request head SHA as inert analysis data", + ]), + ); + expect(errors.some((error) => error.includes("full commit SHA"))).toBe(true); + expect(errors.some((error) => error.includes("persist-credentials=false"))).toBe(true); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); } - expect(installStep.run.includes("--ignore-scripts")).toBe(true); - expect(analyzeStep.run.includes("$ADVISOR_DIR/tools/pr-review-advisor/analyze.mts")).toBe(true); - expect(analyzeStep.run).toContain("trusted main checkout does not yet contain analyze.mts"); - expect(analyzeStep.run).toContain("pr-review-advisor-final-result.json"); - expect(commentStep.run).toContain("trusted main checkout does not yet contain comment.mts"); }); + + it("reports workflow parse failures through boundary errors", () => { + const missingPath = path.join(ROOT, ".tmp-pr-advisor-missing", "workflow.yaml"); + expect(validatePrReviewAdvisorWorkflowBoundary(missingPath)).toEqual([ + `failed to read or parse workflow: ${missingPath}`, + ]); + }); + }); diff --git a/test/pre-push-typecheck-config.test.ts b/test/pre-push-typecheck-config.test.ts deleted file mode 100644 index f0a34aa0aaa..00000000000 --- a/test/pre-push-typecheck-config.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import path from "node:path"; - -import { describe, expect, it } from "vitest"; - -const REPO_ROOT = path.join(import.meta.dirname, ".."); -const PRE_COMMIT_CONFIG = path.join(REPO_ROOT, ".pre-commit-config.yaml"); - -function hookBlock(id: string): string { - const config = fs.readFileSync(PRE_COMMIT_CONFIG, "utf-8"); - const start = config.indexOf(` - id: ${id}\n`); - expect(start).toBeGreaterThanOrEqual(0); - const nextHook = config.indexOf("\n - id: ", start + 1); - return config.slice(start, nextHook === -1 ? undefined : nextHook); -} - -describe("pre-push TypeScript checks", () => { - it("runs CLI typecheck for src and test TypeScript changes", () => { - const block = hookBlock("tsc-cli"); - - expect(block).toContain("entry: npx tsc -p tsconfig.cli.json"); - expect(block).toContain("stages: [pre-push]"); - expect(block).toContain("always_run: true"); - expect(block).toContain( - String.raw`files: ^(bin|scripts|src|test|nemoclaw-blueprint/scripts)/.*\.(ts|tsx)$|^tsconfig\.cli\.json$`, - ); - expect(block).not.toContain("types_or:"); - }); -}); diff --git a/test/preinstall-node-version.test.ts b/test/preinstall-node-version.test.ts index 39c67b7924d..5123f63b98f 100644 --- a/test/preinstall-node-version.test.ts +++ b/test/preinstall-node-version.test.ts @@ -9,15 +9,6 @@ import { describe, expect, it } from "vitest"; const REPO_ROOT = path.join(import.meta.dirname, ".."); const SCRIPT_PATH = path.join(REPO_ROOT, "scripts/check-node-version.js"); -const PACKAGE_JSON_PATH = path.join(REPO_ROOT, "package.json"); - -function readPackageJson(): { - scripts?: Record; - engines?: { node?: string }; -} { - return JSON.parse(fs.readFileSync(PACKAGE_JSON_PATH, "utf-8")); -} - describe("preinstall node-version guard (#2399)", () => { it("scripts/check-node-version.js exists and is executable", () => { expect(fs.existsSync(SCRIPT_PATH)).toBe(true); @@ -27,16 +18,6 @@ describe("preinstall node-version guard (#2399)", () => { expect(stat.mode & 0o111).toBeGreaterThan(0); }); - it("package.json wires the script as preinstall", () => { - const pkg = readPackageJson(); - expect(pkg.scripts?.preinstall).toBe("node scripts/check-node-version.js"); - }); - - it("package.json declares engines.node so the guard has something to enforce", () => { - const pkg = readPackageJson(); - expect(pkg.engines?.node).toBeDefined(); - expect(pkg.engines?.node).toMatch(/\d+\.\d+\.\d+/); - }); it("guard exits 0 on a Node version that satisfies the declared range", () => { // The current process is the same Node version that npm install uses, and diff --git a/test/runner.test.ts b/test/runner.test.ts index f36eb8d0b0f..ba22cebcb6e 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -612,41 +612,6 @@ describe("regression guards", () => { } }); - it("keeps a single shellQuote definition in the root CLI codebase", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const searchRoots = [path.join(repoRoot, "bin"), path.join(repoRoot, "src")]; - const files: string[] = []; - function walk(dir: string): void { - for (const f of fs.readdirSync(dir, { withFileTypes: true })) { - if (f.isDirectory() && f.name !== "node_modules") walk(path.join(dir, f.name)); - else if (f.name.endsWith(".js") || f.name.endsWith(".ts")) - files.push(path.join(dir, f.name)); - } - } - for (const root of searchRoots) { - walk(root); - } - - const defs = []; - for (const file of files) { - let src: string; - try { - src = fs.readFileSync(file, "utf-8"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; - throw error; - } - if (src.includes("function shellQuote")) { - defs.push(path.relative(repoRoot, file)); - } - } - // runner.ts (CJS consumers) and core/shell-quote.ts (ESM consumers like config-io.ts) - expect(defs.sort()).toEqual([ - path.join("src", "lib", "core", "shell-quote.ts"), - path.join("src", "lib", "runner.ts"), - ]); - }); - it("CLI rejects malicious sandbox names before shell commands (e2e)", () => { const canaryDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-canary-")); const canary = path.join(canaryDir, "executed"); diff --git a/test/security-binaries-restriction.test.ts b/test/security-binaries-restriction.test.ts deleted file mode 100644 index 648fc468951..00000000000 --- a/test/security-binaries-restriction.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, it, expect } from "vitest"; -import fs from "node:fs"; -import path from "node:path"; - -const BASELINE = path.join( - import.meta.dirname, - "..", - "nemoclaw-blueprint", - "policies", - "openclaw-sandbox.yaml", -); -const PRESETS_DIR = path.join( - import.meta.dirname, - "..", - "nemoclaw-blueprint", - "policies", - "presets", -); - -describe("binaries restriction: baseline policy", () => { - it("every network_policies entry has a binaries section", () => { - // Parse YAML manually (no yaml dependency) — find all top-level keys under network_policies - // and verify each has a "binaries:" line within its block - const yaml = fs.readFileSync(BASELINE, "utf-8"); - const lines = yaml.split("\n"); - let inNetworkPolicies = false; - let currentBlock = null; - const blocks = []; - - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (/^network_policies:/.test(line)) { - inNetworkPolicies = true; - continue; - } - if (inNetworkPolicies && /^\S/.test(line) && line.trim() !== "") { - if (currentBlock) blocks.push(currentBlock); - currentBlock = null; - inNetworkPolicies = false; - continue; - } - if (!inNetworkPolicies) continue; - // Top-level entry under network_policies (2-space indent, not a comment) - if (/^ {2}(?!#)\S.*:\s*$/.test(line)) { - if (currentBlock) blocks.push(currentBlock); - currentBlock = { name: line.trim().replace(/:$/, ""), startLine: i + 1, lines: [line] }; - continue; - } - if (currentBlock) currentBlock.lines.push(line); - } - if (currentBlock) blocks.push(currentBlock); - - expect(blocks.length).toBeGreaterThan(0); - - const violators = blocks.filter((b) => !b.lines.some((l) => /^\s+binaries:/.test(l))); - - expect(violators.map((b) => b.name)).toEqual([]); - }); -}); - -describe("binaries restriction: policy presets", () => { - it("every preset YAML has a binaries section", () => { - const presets = fs.readdirSync(PRESETS_DIR).filter((f) => f.endsWith(".yaml")); - expect(presets.length).toBeGreaterThan(0); - - const missing = []; - for (const file of presets) { - const content = fs.readFileSync(path.join(PRESETS_DIR, file), "utf-8"); - if (!/^\s+binaries:\s*$/m.test(content)) { - missing.push(file); - } - } - - expect(missing).toEqual([]); - }); -}); diff --git a/test/security-c4-manifest-traversal.test.ts b/test/security-c4-manifest-traversal.test.ts index d969d2efe64..75839fac151 100644 --- a/test/security-c4-manifest-traversal.test.ts +++ b/test/security-c4-manifest-traversal.test.ts @@ -430,42 +430,3 @@ describe("C-4 fix: restoreSnapshotToHost rejects path traversal", () => { } }); }); - -// ═══════════════════════════════════════════════════════════════════ -// 3. Regression guard — migration-state.ts must contain the validation -// ═══════════════════════════════════════════════════════════════════ -describe("C-4 regression: migration-state.ts contains path validation", () => { - /** Extract the restoreSnapshotToHost function body from the source. */ - function getRestoreFnBody() { - const src = fs.readFileSync( - path.join(import.meta.dirname, "..", "nemoclaw", "src", "commands", "migration-state.ts"), - "utf-8", - ); - const fnStart = src.indexOf("function restoreSnapshotToHost"); - expect(fnStart !== -1).toBeTruthy(); - return src.slice(fnStart); - } - - it("restoreSnapshotToHost calls isWithinRoot on manifest.stateDir", () => { - const fnBody = getRestoreFnBody(); - expect(/isWithinRoot\s*\(\s*manifest\.stateDir/.test(fnBody)).toBeTruthy(); - }); - - it("restoreSnapshotToHost calls isWithinRoot on manifest.configPath", () => { - const fnBody = getRestoreFnBody(); - expect(/isWithinRoot\s*\(\s*manifest\.configPath/.test(fnBody)).toBeTruthy(); - }); - - it("restoreSnapshotToHost validates manifest.homeDir against trusted root", () => { - const fnBody = getRestoreFnBody(); - expect(/isWithinRoot\s*\(\s*manifest\.homeDir/.test(fnBody)).toBeTruthy(); - }); - - it("restoreSnapshotToHost fails closed when hasExternalConfig is true with missing configPath", () => { - const fnBody = getRestoreFnBody(); - expect( - /manifest\.hasExternalConfig\b/.test(fnBody) && - /typeof\s+manifest\.configPath\s*!==\s*["']string["']/.test(fnBody), - ).toBeTruthy(); - }); -}); diff --git a/test/source-shape-scanner.test.ts b/test/source-shape-scanner.test.ts new file mode 100644 index 00000000000..fb47de77bd5 --- /dev/null +++ b/test/source-shape-scanner.test.ts @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { scanTextForTest } from "../scripts/find-source-shape-tests"; + +function detectedCaseNames(source: string): string[] { + return scanTextForTest("test/virtual-source-shape.test.ts", source).map((entry) => entry.name); +} + +describe("source-shape scanner", () => { + it("detects source reads through variable-declared arrow helpers", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import path from "node:path"; + import { expect, it } from "vitest"; + + const loadSource = (repoPath: string) => readFileSync(path.join(process.cwd(), repoPath), "utf8"); + + it("asserts source text", () => { + const source = loadSource("src/lib/example.ts"); + expect(source).toContain("implementation detail"); + }); + `); + + expect(cases).toEqual(["asserts source text"]); + }); + + it("detects source-tree walks that feed source text assertions", () => { + const cases = detectedCaseNames(` + import fs from "node:fs"; + import path from "node:path"; + import { expect, it } from "vitest"; + + function collectProductionFiles(dir: string): string[] { + return fs.readdirSync(dir).flatMap((entry) => { + const absolute = path.join(dir, entry); + const stats = fs.statSync(absolute); + if (stats.isDirectory()) return collectProductionFiles(absolute); + if (absolute.endsWith(".ts") && !absolute.endsWith(".test.ts")) return [absolute]; + return []; + }); + } + + it("asserts import boundaries by reading source files", () => { + const files = collectProductionFiles(path.join(process.cwd(), "src/lib/example")); + for (const file of files) { + const source = fs.readFileSync(file, "utf8"); + const specifiers = source.match(/node:fs/g) ?? []; + expect(specifiers).toEqual([]); + } + }); + `); + + expect(cases).toEqual(["asserts import boundaries by reading source files"]); + }); + + it("detects direct assertions against source-tree helper results", () => { + const cases = detectedCaseNames(` + import fs from "node:fs"; + import path from "node:path"; + import { expect, it } from "vitest"; + + function expectedIds(dir = path.join(process.cwd(), "src/commands")): string[] { + return fs.readdirSync(dir).flatMap((entry) => { + if (!entry.endsWith(".ts") || entry.endsWith(".test.ts")) return []; + return [entry.replace(/\\.ts$/, "")]; + }); + } + + it("asserts discovered command ids", () => { + expect(["onboard"]).toEqual(expectedIds()); + }); + `); + + expect(cases).toEqual(["asserts discovered command ids"]); + }); + + it("detects source reads through variable-declared function expression helpers", () => { + const cases = detectedCaseNames(` + import fs from "node:fs"; + import path from "node:path"; + import { expect, it } from "vitest"; + + const loadSource = function (repoPath: string) { + return fs.readFileSync(path.join(process.cwd(), repoPath), "utf8"); + }; + + it("asserts function-expression source text", () => { + const source = loadSource("scripts/example.sh"); + expect(source).not.toContain("implementation detail"); + }); + `); + + expect(cases).toEqual(["asserts function-expression source text"]); + }); + + it("does not treat uncalled source-reader helpers as source text", () => { + const cases = detectedCaseNames(` + import { readFileSync } from "node:fs"; + import { expect, it } from "vitest"; + + const loadSource = () => readFileSync("src/lib/example.ts", "utf8"); + + it("asserts helper shape only", () => { + expect(loadSource).toBeTypeOf("function"); + }); + `); + + expect(cases).toEqual([]); + }); +}); diff --git a/test/validate-config-schemas.test.ts b/test/validate-config-schemas.test.ts index 7faeea1cf9f..0306ba20ece 100644 --- a/test/validate-config-schemas.test.ts +++ b/test/validate-config-schemas.test.ts @@ -9,13 +9,15 @@ * Vitest project. */ -import { readFileSync, readdirSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, it, expect } from "vitest"; import Ajv, { type ValidateFunction } from "ajv/dist/2020.js"; import YAML from "yaml"; +import { discoverTargets } from "../scripts/validate-configs"; + const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); function repoPath(...segments: string[]): string { @@ -88,6 +90,34 @@ function expectValid(validate: ValidateFunction, data: object, label: string): v } } +// ── Validation target discovery ───────────────────────────────────────────── + +describe("config validation target discovery", () => { + const targets = discoverTargets(); + const filesBySchema = new Map(targets.map((target) => [target.schema, target.files])); + const sandboxPolicyFiles = filesBySchema.get("schemas/sandbox-policy.schema.json") ?? []; + + it("includes every binary-scoped sandbox policy family", () => { + expect(sandboxPolicyFiles).toEqual( + expect.arrayContaining([ + "nemoclaw-blueprint/policies/openclaw-sandbox.yaml", + "nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml", + "agents/hermes/policy-additions.yaml", + "agents/hermes/policy-permissive.yaml", + "agents/openclaw/policy-permissive.yaml", + ]), + ); + }); + + it("discovers model-specific setup manifests", () => { + expect(filesBySchema.get("nemoclaw-blueprint/model-specific-setup/schema.json") ?? []).toEqual( + expect.arrayContaining([ + "nemoclaw-blueprint/model-specific-setup/openclaw/kimi-k2.6-managed-inference.json", + ]), + ); + }); +}); + // ── Blueprint ──────────────────────────────────────────────────────────────── describe("blueprint.schema.json", () => { @@ -182,6 +212,38 @@ describe("blueprint.schema.json", () => { }); }); +// ── Model Router pool config ──────────────────────────────────────────────── + +describe("router-pool-config.schema.json", () => { + const validate = compileSchema("schemas/router-pool-config.schema.json"); + const data = loadYAML(repoPath("nemoclaw-blueprint/router/pool-config.yaml")); + + it("pool-config.yaml passes schema validation", () => { + expectValid(validate, data, "pool-config.yaml"); + }); + + it("rejects router pool config without routing settings", () => { + const bad = cloneObject(data); + delete bad.routing; + expect(validate(bad)).toBe(false); + }); + + it("rejects router pool config models without LiteLLM model IDs", () => { + const root = asRecord(data); + const firstModel = asRecord(Array.isArray(root.models) ? root.models[0] : undefined); + const { litellm_model: _litellmModel, ...modelWithoutId } = firstModel; + const bad = { ...root, models: [modelWithoutId] }; + expect(validate(bad)).toBe(false); + }); + + it("rejects router pool config api_base without HTTPS", () => { + const root = asRecord(data); + const firstModel = asRecord(Array.isArray(root.models) ? root.models[0] : undefined); + const bad = { ...root, models: [{ ...firstModel, api_base: "http://integrate.api.nvidia.com/v1" }] }; + expect(validate(bad)).toBe(false); + }); +}); + // ── Base sandbox policy ────────────────────────────────────────────────────── describe("sandbox-policy.schema.json", () => { @@ -192,6 +254,26 @@ describe("sandbox-policy.schema.json", () => { expectValid(validate, data, "openclaw-sandbox.yaml"); }); + it("openclaw-sandbox-permissive.yaml passes schema validation", () => { + expectValid( + validate, + loadYAML(repoPath("nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml")), + "openclaw-sandbox-permissive.yaml", + ); + }); + + for (const file of [ + "agents/openclaw/policy-permissive.yaml", + "agents/hermes/policy-additions.yaml", + "agents/hermes/policy-permissive.yaml", + ]) { + if (existsSync(repoPath(file))) { + it(`${file} passes schema validation`, () => { + expectValid(validate, loadYAML(repoPath(file)), file); + }); + } + } + it("rejects policy with missing network_policies", () => { const bad = cloneObject(data); delete bad.network_policies; @@ -209,6 +291,7 @@ describe("sandbox-policy.schema.json", () => { network_policies: { test_service: { name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], endpoints: [{ host: "api.example.com", port: 443, protocol: "rest" }], }, }, @@ -216,12 +299,26 @@ describe("sandbox-policy.schema.json", () => { expect(validate(bad)).toBe(false); }); + it("rejects sandbox-policy network entries without explicit binary scoping", () => { + const bad = { + version: 1, + network_policies: { + test_service: { + name: "Test Service", + endpoints: [{ host: "api.example.com", port: 443, access: "full" }], + }, + }, + }; + expect(validate(bad)).toBe(false); + }); + it("accepts sandbox-policy native WebSocket text rules and credential rewrite", () => { const valid = { version: 1, network_policies: { test_service: { name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], endpoints: [ { host: "gateway.example.com", @@ -248,6 +345,7 @@ describe("sandbox-policy.schema.json", () => { network_policies: { slack: { name: "Slack", + binaries: [{ path: "/usr/bin/node" }], endpoints: [ { host: "api.slack.com", @@ -270,6 +368,7 @@ describe("sandbox-policy.schema.json", () => { network_policies: { test_service: { name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], endpoints: [{ host: "gateway.example.com", port: 443, protocol: "websocket" }], }, }, @@ -320,6 +419,7 @@ describe("policy-preset.schema.json", () => { network_policies: { test_service: { name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], endpoints: [{ host: "api.example.com", port: 443, protocol: "rest" }], }, }, @@ -327,12 +427,26 @@ describe("policy-preset.schema.json", () => { expect(validate(bad)).toBe(false); }); + it("rejects preset network entries without explicit binary scoping", () => { + const bad = { + preset: { name: "test", description: "test" }, + network_policies: { + test_service: { + name: "Test Service", + endpoints: [{ host: "api.example.com", port: 443, access: "full" }], + }, + }, + }; + expect(validate(bad)).toBe(false); + }); + it("accepts preset native WebSocket text rules and credential rewrite", () => { const valid = { preset: { name: "test", description: "test" }, network_policies: { test_service: { name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], endpoints: [ { host: "gateway.example.com", @@ -359,6 +473,7 @@ describe("policy-preset.schema.json", () => { network_policies: { slack: { name: "Slack", + binaries: [{ path: "/usr/bin/node" }], endpoints: [ { host: "api.slack.com", @@ -381,6 +496,7 @@ describe("policy-preset.schema.json", () => { network_policies: { test_service: { name: "Test Service", + binaries: [{ path: "/usr/bin/node" }], endpoints: [{ host: "gateway.example.com", port: 443, protocol: "websocket" }], }, }, @@ -394,19 +510,24 @@ describe("policy-preset.schema.json", () => { describe("openclaw-plugin.schema.json", () => { const validate = compileSchema("schemas/openclaw-plugin.schema.json"); const data = loadJSON(repoPath("nemoclaw/openclaw.plugin.json")); + const validPluginFixture = { + id: "fixture-plugin", + name: "Fixture Plugin", + version: "1.2.3", + description: "Schema fixture", + }; it("openclaw.plugin.json passes schema validation", () => { expectValid(validate, data, "openclaw.plugin.json"); }); it("rejects plugin with missing id", () => { - const bad = cloneObject(data); - delete bad.id; + const { id: _id, ...bad } = validPluginFixture; expect(validate(bad)).toBe(false); }); it("rejects plugin with invalid version format", () => { - const bad = { ...cloneObject(data), version: "not-semver" }; + const bad = { ...validPluginFixture, version: "not-semver" }; expect(validate(bad)).toBe(false); }); }); diff --git a/test/validate-configs-dangerous-hosts.test.ts b/test/validate-configs-dangerous-hosts.test.ts index acd8d888ca7..75f54fc03c2 100644 --- a/test/validate-configs-dangerous-hosts.test.ts +++ b/test/validate-configs-dangerous-hosts.test.ts @@ -13,7 +13,9 @@ import { describe, expect, it } from "vitest"; import { DANGEROUS_HOSTS, + ROUTER_API_BASE_HOST_ALLOWLIST, findDangerousHosts, + findDangerousRouterApiBases, isDangerousHost, } from "../scripts/validate-configs"; @@ -59,6 +61,34 @@ describe("isDangerousHost", () => { }); }); +describe("findDangerousRouterApiBases", () => { + it("allows the public NVIDIA Build endpoint", () => { + expect( + findDangerousRouterApiBases({ + models: [{ api_base: "https://integrate.api.nvidia.com/v1" }], + }), + ).toEqual([]); + expect(ROUTER_API_BASE_HOST_ALLOWLIST.has("integrate.api.nvidia.com")).toBe(true); + }); + + it.each([ + "http://integrate.api.nvidia.com/v1", + "https://localhost/v1", + "https://127.0.0.1/v1", + "https://10.0.0.5/v1", + "https://metadata.google.internal/v1", + ])("flags unsafe router api_base %s", (apiBase) => { + const findings = findDangerousRouterApiBases({ models: [{ api_base: apiBase }] }); + expect(findings).toEqual([{ path: "/models/0/api_base", host: apiBase }]); + }); + + it("tolerates malformed shapes", () => { + expect(findDangerousRouterApiBases(null)).toEqual([]); + expect(findDangerousRouterApiBases({ models: "not an array" })).toEqual([]); + expect(findDangerousRouterApiBases({ models: [{ api_base: "not a url" }] })).toEqual([]); + }); +}); + describe("findDangerousHosts", () => { it("returns [] for documents with no network_policies", () => { expect(findDangerousHosts({ version: 1 })).toEqual([]); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts new file mode 100644 index 00000000000..04b13bcd2ad --- /dev/null +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-scenarios.yaml"); + +type WorkflowRecord = Record; +type WorkflowStep = WorkflowRecord & { name?: string; run?: string; uses?: string; with?: WorkflowRecord }; + +function asRecord(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; +} + +function asSteps(value: unknown): WorkflowStep[] { + return Array.isArray(value) + ? (value.filter((entry) => asRecord(entry) === entry) as WorkflowStep[]) + : []; +} + +function stringValue(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function namedStep(steps: readonly WorkflowStep[], name: string): WorkflowStep | undefined { + return steps.find((step) => step.name === name); +} + +function requireInput(errors: string[], inputs: WorkflowRecord, name: string): void { + if (!Object.hasOwn(inputs, name)) errors.push(`workflow_dispatch missing input: ${name}`); +} + +function requireStep(errors: string[], steps: readonly WorkflowStep[], name: string): WorkflowStep | undefined { + const step = namedStep(steps, name); + if (!step) errors.push(`run-scenario job missing step: ${name}`); + return step; +} + +function requireRunContains(errors: string[], step: WorkflowStep | undefined, expected: string): void { + if (!step) return; + if (!stringValue(step.run).includes(expected)) { + errors.push(`step '${step.name ?? ""}' run script must include ${expected}`); + } +} + +export function validateE2eScenariosWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, +): string[] { + const workflow = asRecord(YAML.parse(readFileSync(workflowPath, "utf-8"))); + const errors: string[] = []; + const triggers = asRecord(workflow.on ?? workflow[true as unknown as string]); + + const workflowDispatch = asRecord(triggers.workflow_dispatch); + const workflowCall = asRecord(triggers.workflow_call); + if (Object.keys(workflowDispatch).length === 0) errors.push("workflow must support workflow_dispatch"); + if (Object.keys(workflowCall).length === 0) errors.push("workflow must support workflow_call"); + for (const unsafe of ["push", "pull_request", "pull_request_target", "schedule"]) { + if (Object.hasOwn(triggers, unsafe)) errors.push(`workflow must not run on ${unsafe}`); + } + + const dispatchInputs = asRecord(workflowDispatch.inputs); + requireInput(errors, dispatchInputs, "scenario"); + requireInput(errors, dispatchInputs, "suite_filter"); + if (Object.hasOwn(dispatchInputs, "plan_only")) { + errors.push("workflow_dispatch must not expose retired plan_only input"); + } + + const permissions = asRecord(workflow.permissions); + if (permissions.contents !== "read") errors.push("workflow permissions.contents must be read"); + + const jobs = asRecord(workflow.jobs); + const resolveRunner = asRecord(jobs["resolve-runner"]); + if (Object.keys(resolveRunner).length === 0) errors.push("workflow missing resolve-runner job"); + const runScenario = asRecord(jobs["run-scenario"]); + if (Object.keys(runScenario).length === 0) errors.push("workflow missing run-scenario job"); + if (runScenario["runs-on"] !== "${{ needs.resolve-runner.outputs.runner }}") { + errors.push("run-scenario job must use the resolved runner output"); + } + + const steps = asSteps(runScenario.steps); + const normalRun = requireStep(errors, steps, "Run scenario"); + requireRunContains(errors, normalRun, "bash test/e2e/runtime/run-scenario.sh"); + requireRunContains(errors, normalRun, '"$SCENARIO"'); + requireRunContains(errors, normalRun, "exit \"$rc\""); + if (stringValue(normalRun?.run).includes("--plan-only")) { + errors.push("Run scenario step must not use retired --plan-only flag"); + } + + const wslRun = requireStep(errors, steps, "Run scenario in WSL"); + requireRunContains(errors, wslRun, "bash test/e2e/runtime/run-scenario.sh"); + requireRunContains(errors, wslRun, '"$SCENARIO"'); + + const upload = requireStep(errors, steps, "Upload scenario artifacts"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-scenario-${{ inputs.scenario }}") { + errors.push("artifact upload name must include the scenario input"); + } + if (uploadWith["include-hidden-files"] !== true) { + errors.push("artifact upload must include hidden .e2e files"); + } + if (!stringValue(uploadWith.path).includes(".e2e/")) { + errors.push("artifact upload path must include .e2e/"); + } + + return errors; +} diff --git a/tools/pr-review-advisor/workflow-boundary.mts b/tools/pr-review-advisor/workflow-boundary.mts new file mode 100644 index 00000000000..c0e11dd358e --- /dev/null +++ b/tools/pr-review-advisor/workflow-boundary.mts @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "pr-review-advisor.yaml"); + +type WorkflowRecord = Record; + +type WorkflowStep = WorkflowRecord & { + name?: string; + run?: string; + uses?: string; + with?: WorkflowRecord; +}; + +function asRecord(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; +} + +function asSteps(value: unknown): WorkflowStep[] { + return Array.isArray(value) ? (value.filter((entry) => asRecord(entry) === entry) as WorkflowStep[]) : []; +} + +function stringValue(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function booleanValue(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined; +} + +function namedStep(steps: readonly WorkflowStep[], name: string): WorkflowStep | undefined { + return steps.find((step) => step.name === name); +} + +function usesPinnedAction(uses: string): boolean { + return /^[^@\s]+\/[^@\s]+@[0-9a-f]{40}(?:\s*#.*)?$/.test(uses); +} + +function requireStep( + errors: string[], + steps: readonly WorkflowStep[], + name: string, +): WorkflowStep | undefined { + const step = namedStep(steps, name); + if (!step) errors.push(`missing workflow step: ${name}`); + return step; +} + +function requireStepWith( + errors: string[], + step: WorkflowStep | undefined, + key: string, + expected: string | boolean, +): void { + if (!step) return; + const actual = asRecord(step.with)[key]; + if (actual !== expected) { + errors.push(`step '${step.name ?? ""}' expected with.${key}=${String(expected)}`); + } +} + +function requireRunContains( + errors: string[], + step: WorkflowStep | undefined, + expected: string, +): void { + if (!step) return; + const run = stringValue(step.run); + if (!run.includes(expected)) { + errors.push(`step '${step.name ?? ""}' run script must include ${expected}`); + } +} + +export function validatePrReviewAdvisorWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, +): string[] { + const errors: string[] = []; + let workflow: WorkflowRecord; + try { + workflow = asRecord(YAML.parse(readFileSync(workflowPath, "utf-8"))); + } catch { + errors.push(`failed to read or parse workflow: ${workflowPath}`); + return errors; + } + + const triggers = asRecord(workflow.on ?? workflow[true as unknown as string]); + if (!Object.hasOwn(triggers, "pull_request")) { + errors.push("workflow must run on pull_request, not only trusted-target events"); + } + if (Object.hasOwn(triggers, "pull_request_target")) { + errors.push("workflow must not run untrusted PR code under pull_request_target"); + } + + const reviewJob = asRecord(asRecord(workflow.jobs).review); + const steps = asSteps(reviewJob.steps); + if (steps.length === 0) errors.push("review job must declare steps"); + + for (const step of steps) { + if (step.uses && !usesPinnedAction(step.uses)) { + errors.push(`step '${step.name ?? step.uses}' must pin action uses to a full commit SHA`); + } + } + + const trustedCheckout = requireStep(errors, steps, "Checkout trusted advisor code (main)"); + requireStepWith(errors, trustedCheckout, "repository", "NVIDIA/NemoClaw"); + requireStepWith(errors, trustedCheckout, "ref", "main"); + requireStepWith(errors, trustedCheckout, "path", "advisor"); + requireStepWith(errors, trustedCheckout, "persist-credentials", false); + + const prCheckout = requireStep(errors, steps, "Checkout PR workspace (read-only data)"); + requireStepWith(errors, prCheckout, "path", "pr-workdir"); + requireStepWith(errors, prCheckout, "persist-credentials", false); + const prRef = stringValue(asRecord(prCheckout?.with).ref).trim(); + if (prRef !== "${{ github.event.pull_request.head.sha }}") { + errors.push("PR checkout must use the pull request head SHA as inert analysis data"); + } + + const dispatchCheckout = requireStep(errors, steps, "Checkout dispatch workspace (read-only data)"); + requireStepWith(errors, dispatchCheckout, "path", "pr-workdir"); + requireStepWith(errors, dispatchCheckout, "persist-credentials", false); + + const install = requireStep(errors, steps, "Install Pi SDK"); + requireRunContains(errors, install, "--ignore-scripts"); + requireRunContains(errors, install, "$ADVISOR_DIR/node_modules"); + + const analyze = requireStep(errors, steps, "Run PR review advisor"); + requireRunContains(errors, analyze, "cd \"$ADVISOR_WORKDIR\""); + requireRunContains(errors, analyze, "$ADVISOR_DIR/tools/pr-review-advisor/analyze.mts"); + requireRunContains(errors, analyze, "$ADVISOR_DIR/tools/pr-review-advisor/schema.json"); + + const comment = requireStep(errors, steps, "Post PR review advisor comment"); + requireRunContains(errors, comment, "$ADVISOR_DIR/tools/pr-review-advisor/comment.mts"); + + const permissions = asRecord(workflow.permissions); + if (permissions.contents !== "read") errors.push("workflow permissions.contents must be read"); + if (booleanValue(reviewJob["continue-on-error"]) === true) { + errors.push("review job must not be globally continue-on-error"); + } + + return errors; +}