diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 25fd5118b8..7fe1dfccfc 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -14,7 +14,7 @@ "src/lib/cli/branding.ts": 87, "src/lib/cli/nemoclaw-oclif-command.ts": 106, "src/lib/cli/terminal-style.ts": 43, - "src/lib/core/json-types.ts": 37, + "src/lib/core/json-types.ts": 36, "src/lib/core/ports.ts": 89, "src/lib/core/shell-quote.ts": 28, "src/lib/core/url-utils.ts": 29, diff --git a/nemoclaw/src/security/credential-filter.ts b/nemoclaw/src/security/credential-filter.ts index 5408f73f6f..6ceab70240 100644 --- a/nemoclaw/src/security/credential-filter.ts +++ b/nemoclaw/src/security/credential-filter.ts @@ -1,238 +1,21 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// -// Credential stripping for host→sandbox migration snapshots. -// Kept in parity with src/lib/security/credential-filter.ts so migration -// cannot leave channel tokens, env secrets, or auth headers in the sandbox. -import { isObjectRecord, type UnknownRecord } from "../shared/object-record.js"; - -export const CREDENTIAL_PLACEHOLDER = "[STRIPPED_BY_MIGRATION]"; - -/** - * Basenames that MUST NOT be copied into snapshot bundles. - */ -export const CREDENTIAL_SENSITIVE_BASENAMES = new Set([ - "auth-profiles.json", - "auth.json", - "chatgpt-auth.json", -]); - -const CREDENTIAL_FIELDS = new Set([ - "apikey", - "api_key", - "token", - "secret", - "password", - "pass", - "passwd", - "resolvedkey", -]); - -const CREDENTIAL_FIELD_PATTERN = - /^(?:(?:personal[._-]?)?access|refresh|client|bearer|oauth|auth|api|private|public|signing|session|bot|app|resolved)[._-]?(?:tokens?|keys?|secrets?|passwords?|passphrases?|credentials?)$/i; - -const ENV_SECRET_FIELD_PATTERN = - /^(?:[A-Z0-9]+_)*(?:TOKEN|KEY|SECRET|PASSWORD|PASSWD|PASS|PASSPHRASE|CREDENTIAL)S?$/; - -const CREDENTIAL_HEADER_NAMES: ReadonlySet = new Set([ - "authorization", - "proxy-authorization", - "cookie", - "set-cookie", -]); - -const HEADER_CREDENTIAL_PATTERN = /-(?:key|token|secret|password|passphrase|credential|auth)s?$/i; - -const PUBLIC_KEY_FIELD_PATTERN = /(?:^|[._-])public[._-]?keys?$/i; - -const SAFE_CREDENTIAL_PLACEHOLDER_PATTERNS: readonly RegExp[] = [ - /^openshell:resolve:env:[A-Za-z0-9_]+$/, - /^Bearer\s+openshell:resolve:env:[A-Za-z0-9_]+$/i, - /^xoxb-OPENSHELL-RESOLVE-ENV-[A-Za-z0-9_]+$/, - /^xapp-OPENSHELL-RESOLVE-ENV-[A-Za-z0-9_]+$/, -]; - -const SAFE_CREDENTIAL_PLACEHOLDER_LITERALS: ReadonlySet = new Set([ - "unused", +// sourceOfTruth: nemoclaw/src/shared/credential-filter-boundary.cts +// Keep this package entry wrapper implementation-free so migration and the CLI +// execute the same credential-stripping rules. +export { + CONTEXT_PATTERNS as CONTEXT_SECRET_PATTERNS, CREDENTIAL_PLACEHOLDER, -]); - -/** - * Context-anchored secret shapes mirrored from - * src/lib/security/secret-patterns.ts. The plugin package cannot import - * src/lib at runtime, so a repository-level parity test pins this copy. - */ -export const CONTEXT_SECRET_PATTERNS: readonly RegExp[] = [ - /(?<=Bearer\s+)[A-Za-z0-9_.+/=-]{10,}/gi, - /(?<=(?:^|[^A-Za-z0-9])(?:[A-Za-z0-9]{1,128}_(?:KEY|TOKEN|SECRET|CREDENTIAL|PASSWORD|PASSWD|PASS)|(?:X[-_])?API[-_]KEY|TOKEN|SECRET|CREDENTIAL|PASSWORD|PASSWD|PASS)["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/gi, - /(?<=(?:^|[^A-Za-z0-9])(?:[A-Za-z0-9]{1,128}(?:Token|Secret|Credential)|[A-Za-z0-9]{0,128}(?:[Aa]ccess|[Rr]efresh|[Cc]lient|[Bb]earer|[Aa]uth|[Aa][Pp][Ii]|[Pp]rivate|[Ss]igning|[Ss]ession|[Bb]ot|[Aa]pp|[Rr]esolved)Key|[A-Za-z0-9]{1,128}(?:Password|Passwd|Pass))["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/g, - /(?<=(?:^|[^A-Za-z0-9])KEY["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/g, -]; - -/** - * High-confidence raw secret shapes used as a value-level backstop. - * Kept aligned with TOKEN_PREFIX / STRUCTURED / SECRET_BLOCK patterns from - * src/lib/security/secret-patterns.ts (plugin package cannot import src/lib). - */ -const VALUE_SECRET_PATTERNS: readonly RegExp[] = [ - /nvapi-[A-Za-z0-9_-]{10,}/, - /nvcf-[A-Za-z0-9_-]{10,}/, - /ghp_[A-Za-z0-9_-]{10,}/, - /(?:github_pat_)[A-Za-z0-9_]{30,}/, - /sk-proj-[A-Za-z0-9_-]{10,}/, - /sk-ant-[A-Za-z0-9_-]{10,}/, - /sk-[A-Za-z0-9_-]{20,}/, - /(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}/, - /A(?:K|S)IA[A-Z0-9]{16}/, - /hf_[A-Za-z0-9]{10,}/, - /glpat-[A-Za-z0-9_-]{10,}/, - /gsk_[A-Za-z0-9]{10,}/, - /pypi-[A-Za-z0-9_-]{10,}/, - /\bbot\d{8,10}:[A-Za-z0-9_-]{35}\b/, - /\b\d{8,10}:[A-Za-z0-9_-]{35}\b/, - /\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b/, - /tvly-[A-Za-z0-9_-]{10,}/, - /lsv2_(?:pt|sk)_[A-Za-z0-9]{10,}(?:_[A-Za-z0-9]+)*/, - /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/, - /-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9]+ )?PRIVATE KEY-----/, - ...CONTEXT_SECRET_PATTERNS, -]; - -function hasPassCredentialSegment(key: string): boolean { - const normalized = key - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2") - .replace(/([a-z0-9])([A-Z])/g, "$1_$2") - .replace(/[^A-Za-z0-9]+/g, "_") - .replace(/^_+|_+$/g, "") - .toLowerCase(); - return ( - normalized === "pass" || - normalized === "passwd" || - normalized.endsWith("_pass") || - normalized.endsWith("_passwd") - ); -} - -export function isCredentialField(key: string): boolean { - if (PUBLIC_KEY_FIELD_PATTERN.test(key)) return false; - return ( - CREDENTIAL_FIELDS.has(key.toLowerCase()) || - CREDENTIAL_FIELD_PATTERN.test(key) || - hasPassCredentialSegment(key) || - ENV_SECRET_FIELD_PATTERN.test(key) || - HEADER_CREDENTIAL_PATTERN.test(key) || - CREDENTIAL_HEADER_NAMES.has(key.toLowerCase()) - ); -} - -export function valueLooksLikeSecret(value: string): boolean { - return VALUE_SECRET_PATTERNS.some((pattern) => { - pattern.lastIndex = 0; - const matched = pattern.test(value); - pattern.lastIndex = 0; - return matched; - }); -} - -export function isSafeCredentialPlaceholder(value: unknown): boolean { - if (typeof value !== "string") return false; - const withoutScheme = value.replace(/^Bearer\s+/i, ""); - if ( - SAFE_CREDENTIAL_PLACEHOLDER_LITERALS.has(value) || - SAFE_CREDENTIAL_PLACEHOLDER_LITERALS.has(withoutScheme) - ) { - return true; - } - return SAFE_CREDENTIAL_PLACEHOLDER_PATTERNS.some((pattern) => pattern.test(value)); -} - -function scrubConfigValue(value: unknown): unknown { - if (typeof value === "string") { - if (isSafeCredentialPlaceholder(value)) return value; - return valueLooksLikeSecret(value) ? CREDENTIAL_PLACEHOLDER : value; - } - return stripCredentials(value); -} - -function cliFlagName(token: string): string | null { - const match = /^--?([A-Za-z0-9][A-Za-z0-9._-]*)$/.exec(token); - return match ? match[1] : null; -} - -function scrubArrayElement(value: unknown, previous: unknown): unknown { - if (typeof value !== "string") return stripCredentials(value); - if (isSafeCredentialPlaceholder(value)) return value; - - const eq = value.indexOf("="); - if (eq > 0 && value.startsWith("-")) { - const flagName = cliFlagName(value.slice(0, eq)); - if (flagName && isCredentialField(flagName)) { - const inlineValue = value.slice(eq + 1); - return isSafeCredentialPlaceholder(inlineValue) - ? value - : `${value.slice(0, eq)}=${CREDENTIAL_PLACEHOLDER}`; - } - } - - if (!value.startsWith("-") && typeof previous === "string") { - const prevFlag = cliFlagName(previous); - if (prevFlag && isCredentialField(prevFlag)) return CREDENTIAL_PLACEHOLDER; - } - - return valueLooksLikeSecret(value) ? CREDENTIAL_PLACEHOLDER : value; -} - -/** - * Recursively strip credential fields from a JSON-like object. - */ -export function stripCredentials(obj: unknown): unknown { - if (obj === null || obj === undefined) return obj; - if (typeof obj !== "object") return obj; - if (Array.isArray(obj)) { - return obj.map((value, index) => scrubArrayElement(value, obj[index - 1])); - } - if (!isObjectRecord(obj)) return obj; - - const result: UnknownRecord = {}; - for (const [key, value] of Object.entries(obj)) { - if (isCredentialField(key)) { - result[key] = - value === null || value === undefined || isSafeCredentialPlaceholder(value) - ? value - : CREDENTIAL_PLACEHOLDER; - } else { - result[key] = scrubConfigValue(value); - } - } - return result; -} - -/** - * Strip credentials from a shell-style environment file body. - * - * Field-name matching handles ordinary secret variables while the value-shape - * backstop catches provider tokens stored under an otherwise benign key. - */ -export function sanitizeEnvFileContent(content: string): string { - return content - .split("\n") - .map((line) => { - const trimmed = line.trim(); - if (trimmed === "" || trimmed.startsWith("#")) return line; - const eq = line.indexOf("="); - if (eq <= 0) return line; - const rawKey = line.slice(0, eq).trim(); - const key = rawKey.replace(/^export\s+/i, "").trim(); - const value = line.slice(eq + 1); - if (isSafeCredentialPlaceholder(value)) return line; - if (!key) return line; - if (!isCredentialField(key) && !valueLooksLikeSecret(value)) return line; - return `${line.slice(0, eq)}=${CREDENTIAL_PLACEHOLDER}`; - }) - .join("\n"); -} - -export function isSensitiveFile(filename: string): boolean { - return CREDENTIAL_SENSITIVE_BASENAMES.has(filename.toLowerCase()); -} + CREDENTIAL_SENSITIVE_BASENAMES, + isConfigObject, + isConfigValue, + isCredentialField, + isSafeCredentialPlaceholder, + isSensitiveFile, + sanitizeEnvFileContent, + stripCredentials, + valueLooksLikeSecret, +} from "../shared/credential-filter-boundary.cjs"; + +export type { ConfigObject, ConfigValue } from "../shared/credential-filter-boundary.cjs"; diff --git a/nemoclaw/src/shared/credential-filter-boundary.cts b/nemoclaw/src/shared/credential-filter-boundary.cts new file mode 100644 index 0000000000..b67a37be31 --- /dev/null +++ b/nemoclaw/src/shared/credential-filter-boundary.cts @@ -0,0 +1,269 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +function frozenPatterns(patterns: RegExp[]): readonly RegExp[] { + return Object.freeze(patterns); +} + +/** Token-prefix patterns that match standalone secrets. */ +export const TOKEN_PREFIX_PATTERNS: readonly RegExp[] = frozenPatterns([ + /nvapi-[A-Za-z0-9_-]{10,}/g, + /nvcf-[A-Za-z0-9_-]{10,}/g, + /ghp_[A-Za-z0-9_-]{10,}/g, + /(?:github_pat_)[A-Za-z0-9_]{30,}/g, + /sk-proj-[A-Za-z0-9_-]{10,}/g, + /sk-ant-[A-Za-z0-9_-]{10,}/g, + /sk-[A-Za-z0-9_-]{20,}/g, + /(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}/g, + /A(?:K|S)IA[A-Z0-9]{16}/g, + /hf_[A-Za-z0-9]{10,}/g, + /glpat-[A-Za-z0-9_-]{10,}/g, + /gsk_[A-Za-z0-9]{10,}/g, + /pypi-[A-Za-z0-9_-]{10,}/g, + /\bbot\d{8,10}:[A-Za-z0-9_-]{35}\b/g, + /\b\d{8,10}:[A-Za-z0-9_-]{35}\b/g, + /\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b/g, + /tvly-[A-Za-z0-9_-]{10,}/g, + /lsv2_(?:pt|sk)_[A-Za-z0-9]{10,}(?:_[A-Za-z0-9]+)*/g, +]); + +/** Structured standalone tokens without a provider-specific prefix. */ +export const STRUCTURED_TOKEN_PATTERNS: readonly RegExp[] = frozenPatterns([ + /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/g, +]); + +/** Context-anchored patterns that require a credential label or auth scheme. */ +export const CONTEXT_PATTERNS: readonly RegExp[] = frozenPatterns([ + /(?<=Bearer\s+)[A-Za-z0-9_.+/=-]{10,}/gi, + /(?<=(?:^|[^A-Za-z0-9])(?:[A-Za-z0-9]{1,128}_(?:KEY|TOKEN|SECRET|CREDENTIAL|PASSWORD|PASSWD|PASS)|(?:X[-_])?API[-_]KEY|TOKEN|SECRET|CREDENTIAL|PASSWORD|PASSWD|PASS)["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/gi, + /(?<=(?:^|[^A-Za-z0-9])(?:[A-Za-z0-9]{1,128}(?:Token|Secret|Credential)|[A-Za-z0-9]{0,128}(?:[Aa]ccess|[Rr]efresh|[Cc]lient|[Bb]earer|[Aa]uth|[Aa][Pp][Ii]|[Pp]rivate|[Ss]igning|[Ss]ession|[Bb]ot|[Aa]pp|[Rr]esolved)Key|[A-Za-z0-9]{1,128}(?:Password|Passwd|Pass))["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/g, + /(?<=(?:^|[^A-Za-z0-9])KEY["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/g, +]); + +/** Multi-line secret blocks without a token prefix. */ +export const SECRET_BLOCK_PATTERNS: readonly RegExp[] = frozenPatterns([ + /-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9]+ )?PRIVATE KEY-----/g, +]); + +/** All supported secret patterns. */ +export const SECRET_PATTERNS: readonly RegExp[] = frozenPatterns([ + ...TOKEN_PREFIX_PATTERNS, + ...STRUCTURED_TOKEN_PATTERNS, + ...SECRET_BLOCK_PATTERNS, + ...CONTEXT_PATTERNS, +]); + +/** Token prefixes covered by the debug.sh fallback when Node is unavailable. */ +export const EXPECTED_SHELL_PREFIXES = ["nvapi-", "nvcf-", "ghp_", "sk-", "tvly-"]; + +/** Match pass/passwd only as a complete or terminal credential-name segment. */ +export function hasPassCredentialSegment(key: string): boolean { + const normalized = key + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2") + .replace(/([a-z0-9])([A-Z])/g, "$1_$2") + .replace(/[^A-Za-z0-9]+/g, "_") + .replace(/^_+|_+$/g, "") + .toLowerCase(); + return ( + normalized === "pass" || + normalized === "passwd" || + normalized.endsWith("_pass") || + normalized.endsWith("_passwd") + ); +} + +export type ConfigValue = + | null + | undefined + | boolean + | number + | string + | ConfigValue[] + | ConfigObject; + +export type ConfigObject = { [key: string]: ConfigValue }; + +export const CREDENTIAL_PLACEHOLDER = "[STRIPPED_BY_MIGRATION]"; + +export const CREDENTIAL_SENSITIVE_BASENAMES = new Set([ + "auth-profiles.json", + "auth.json", + "chatgpt-auth.json", +]); + +const CREDENTIAL_FIELDS = new Set([ + "apikey", + "api_key", + "token", + "secret", + "password", + "pass", + "passwd", + "resolvedkey", +]); +const CREDENTIAL_FIELD_PATTERN = + /^(?:(?:personal[._-]?)?access|refresh|client|bearer|oauth|auth|api|private|public|signing|session|bot|app|resolved)[._-]?(?:tokens?|keys?|secrets?|passwords?|passphrases?|credentials?)$/i; +const ENV_SECRET_FIELD_PATTERN = + /^(?:[A-Z0-9]+_)*(?:TOKEN|KEY|SECRET|PASSWORD|PASSWD|PASS|PASSPHRASE|CREDENTIAL)S?$/; +const CREDENTIAL_HEADER_NAMES: ReadonlySet = new Set([ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", +]); +const HEADER_CREDENTIAL_PATTERN = /-(?:key|token|secret|password|passphrase|credential|auth)s?$/i; +// Public keys are verification material. Exempt publicKey, PUBLIC_KEY, +// public-key, and prefixed forms before applying credential-name patterns. +const PUBLIC_KEY_FIELD_PATTERN = /(?:^|[._-])public[._-]?keys?$/i; +// OpenShell resolves these references at launch. Replacing a reference with a +// stripped marker would produce a credential-free but unusable restored config. +const SAFE_CREDENTIAL_PLACEHOLDER_PATTERNS: readonly RegExp[] = [ + /^openshell:resolve:env:[A-Za-z0-9_]+$/, + /^Bearer\s+openshell:resolve:env:[A-Za-z0-9_]+$/i, + /^xoxb-OPENSHELL-RESOLVE-ENV-[A-Za-z0-9_]+$/, + /^xapp-OPENSHELL-RESOLVE-ENV-[A-Za-z0-9_]+$/, +]; +const SAFE_CREDENTIAL_PLACEHOLDER_LITERALS: ReadonlySet = new Set([ + "unused", + CREDENTIAL_PLACEHOLDER, +]); + +function isObjectRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function isCredentialField(key: string): boolean { + if (PUBLIC_KEY_FIELD_PATTERN.test(key)) return false; + return ( + CREDENTIAL_FIELDS.has(key.toLowerCase()) || + CREDENTIAL_FIELD_PATTERN.test(key) || + hasPassCredentialSegment(key) || + ENV_SECRET_FIELD_PATTERN.test(key) || + HEADER_CREDENTIAL_PATTERN.test(key) || + CREDENTIAL_HEADER_NAMES.has(key.toLowerCase()) + ); +} + +export function valueLooksLikeSecret(value: string): boolean { + for (const pattern of SECRET_PATTERNS) { + pattern.lastIndex = 0; + if (pattern.test(value)) return true; + } + return false; +} + +export function isSafeCredentialPlaceholder(value: unknown): boolean { + if (typeof value !== "string") return false; + const withoutScheme = value.replace(/^Bearer\s+/i, ""); + if ( + SAFE_CREDENTIAL_PLACEHOLDER_LITERALS.has(value) || + SAFE_CREDENTIAL_PLACEHOLDER_LITERALS.has(withoutScheme) + ) { + return true; + } + return SAFE_CREDENTIAL_PLACEHOLDER_PATTERNS.some((pattern) => pattern.test(value)); +} + +export function isConfigObject(value: unknown): value is ConfigObject { + return isObjectRecord(value); +} + +export function isConfigValue(value: unknown): value is ConfigValue { + if (value === null || value === undefined) return true; + if (typeof value === "boolean" || typeof value === "number" || typeof value === "string") { + return true; + } + if (Array.isArray(value)) return value.every((entry) => isConfigValue(entry)); + if (!isConfigObject(value)) return false; + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return false; + return Object.values(value).every((entry) => isConfigValue(entry)); +} + +function scrubConfigValue(value: unknown): unknown { + if (typeof value === "string") { + if (isSafeCredentialPlaceholder(value)) return value; + return valueLooksLikeSecret(value) ? CREDENTIAL_PLACEHOLDER : value; + } + return stripCredentials(value); +} + +function cliFlagName(token: string): string | null { + const match = /^--?([A-Za-z0-9][A-Za-z0-9._-]*)$/.exec(token); + return match ? match[1] : null; +} + +function scrubArrayElement(value: unknown, previous: unknown): unknown { + if (typeof value !== "string") return stripCredentials(value); + if (isSafeCredentialPlaceholder(value)) return value; + + const eq = value.indexOf("="); + if (eq > 0 && value.startsWith("-")) { + const flagName = cliFlagName(value.slice(0, eq)); + if (flagName && isCredentialField(flagName)) { + const inlineValue = value.slice(eq + 1); + return isSafeCredentialPlaceholder(inlineValue) + ? value + : `${value.slice(0, eq)}=${CREDENTIAL_PLACEHOLDER}`; + } + } + + if (typeof previous === "string") { + const previousFlag = cliFlagName(previous); + if (previousFlag && isCredentialField(previousFlag)) return CREDENTIAL_PLACEHOLDER; + } + + return valueLooksLikeSecret(value) ? CREDENTIAL_PLACEHOLDER : value; +} + +export function stripCredentials(obj: null): null; +export function stripCredentials(obj: undefined): undefined; +export function stripCredentials(obj: boolean): boolean; +export function stripCredentials(obj: number): number; +export function stripCredentials(obj: string): string; +export function stripCredentials(obj: T): T; +export function stripCredentials(obj: T): T; +export function stripCredentials(obj: ConfigValue): ConfigValue; +export function stripCredentials(obj: unknown): unknown; +export function stripCredentials(obj: unknown): unknown { + if (obj === null || obj === undefined || typeof obj !== "object") return obj; + if (Array.isArray(obj)) { + return obj.map((value, index) => scrubArrayElement(value, obj[index - 1])); + } + if (!isObjectRecord(obj)) return obj; + + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + if (isCredentialField(key)) { + result[key] = + value === null || value === undefined || isSafeCredentialPlaceholder(value) + ? value + : CREDENTIAL_PLACEHOLDER; + } else { + result[key] = scrubConfigValue(value); + } + } + return result; +} + +export function sanitizeEnvFileContent(content: string): string { + return content + .split("\n") + .map((line) => { + const trimmed = line.trim(); + if (trimmed === "" || trimmed.startsWith("#")) return line; + const eq = line.indexOf("="); + if (eq <= 0) return line; + const rawKey = line.slice(0, eq).trim(); + const key = rawKey.replace(/^export\s+/i, "").trim(); + const value = line.slice(eq + 1); + if (isSafeCredentialPlaceholder(value) || !key) return line; + if (!isCredentialField(key) && !valueLooksLikeSecret(value)) return line; + return `${line.slice(0, eq)}=${CREDENTIAL_PLACEHOLDER}`; + }) + .join("\n"); +} + +export function isSensitiveFile(filename: string): boolean { + return CREDENTIAL_SENSITIVE_BASENAMES.has(filename.toLowerCase()); +} diff --git a/nemoclaw/src/shared/credential-filter-boundary.test.ts b/nemoclaw/src/shared/credential-filter-boundary.test.ts new file mode 100644 index 0000000000..168135f2a1 --- /dev/null +++ b/nemoclaw/src/shared/credential-filter-boundary.test.ts @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import type { ConfigValue } from "./credential-filter-boundary.cjs"; + +import { + CONTEXT_PATTERNS, + CREDENTIAL_PLACEHOLDER, + isConfigObject, + isConfigValue, + isCredentialField, + isSafeCredentialPlaceholder, + isSensitiveFile, + sanitizeEnvFileContent, + SECRET_BLOCK_PATTERNS, + SECRET_PATTERNS, + stripCredentials, + STRUCTURED_TOKEN_PATTERNS, + TOKEN_PREFIX_PATTERNS, + valueLooksLikeSecret, +} from "./credential-filter-boundary.cjs"; + +function asConfigValue(value: ConfigValue): ConfigValue { + return value; +} + +describe("shared credential filter", () => { + it("preserves the ConfigValue return contract (#8291)", () => { + const value = asConfigValue({ token: "opaque-secret-value" }); + const result = stripCredentials(value); + + expect(result).toEqual({ token: CREDENTIAL_PLACEHOLDER }); + expect(stripCredentials(null as ConfigValue)).toBeNull(); + expectTypeOf(result).toEqualTypeOf(); + }); + + it("accepts only recursively representable configuration values (#8291)", () => { + const nullPrototype = Object.assign(Object.create(null), { enabled: true }); + + expect(isConfigObject({})).toBe(true); + expect(isConfigObject(nullPrototype)).toBe(true); + expect(isConfigObject(null)).toBe(false); + expect(isConfigObject([])).toBe(false); + expect(isConfigValue(undefined)).toBe(true); + expect(isConfigValue(["model", 1, false, null, nullPrototype])).toBe(true); + expect(isConfigValue(new Date())).toBe(false); + expect(isConfigValue({ nested: new Date() })).toBe(false); + expect(isConfigValue(["model", new Date()])).toBe(false); + }); + + it("classifies credential names without treating public keys as secrets (#8291)", () => { + for (const field of [ + "botToken", + "bot_token", + "appToken", + "access_token", + "personal_access_token", + "refresh-token", + "client_secret", + "auth_token", + "oauth_token", + "apikey", + "Token", + "GITHUB_TOKEN", + "Authorization", + "X-API-Key", + "DB_PASS", + ]) { + expect(isCredentialField(field), field).toBe(true); + } + for (const field of ["publicKey", "public.key", "GITHUB_PUBLIC_KEY", "NODE_ENV", "model"]) { + expect(isCredentialField(field), field).toBe(false); + } + }); + + it("recognizes raw secrets while preserving credential references (#8291)", () => { + for (const value of [ + "ghp_abcdefghijklmnopqrstuvwxyz0123456789", + "Bearer opaque-migration-secret", + "GITHUB_TOKEN=opaque-secret-value-123", + ["-----BEGIN", "PRIVATE KEY-----\nopaque\n-----END PRIVATE KEY-----"].join(" "), + ]) { + expect(valueLooksLikeSecret(value), value).toBe(true); + } + for (const value of [ + "unused", + "Bearer unused", + "openshell:resolve:env:GITHUB_TOKEN", + "Bearer openshell:resolve:env:REMOTE_MCP_TOKEN", + "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_TOKEN", + ]) { + expect(isSafeCredentialPlaceholder(value), value).toBe(true); + } + expect(valueLooksLikeSecret("keep-me")).toBe(false); + }); + + it("freezes exported pattern collections and tolerates caller lastIndex state (#8291)", () => { + for (const patterns of [ + TOKEN_PREFIX_PATTERNS, + STRUCTURED_TOKEN_PATTERNS, + CONTEXT_PATTERNS, + SECRET_BLOCK_PATTERNS, + SECRET_PATTERNS, + ]) { + expect(Object.isFrozen(patterns)).toBe(true); + expect(() => (patterns as RegExp[]).push(/caller-added-secret/g)).toThrow(TypeError); + } + + TOKEN_PREFIX_PATTERNS[0].lastIndex = Number.MAX_SAFE_INTEGER; + expect(valueLooksLikeSecret("nvapi-abcdefghijklmnop")).toBe(true); + }); + + it("strips object, array, header, and environment credentials (#8291)", () => { + const fixture = { + channels: { + slack: { + accounts: { + default: { + botToken: "xoxb-raw-slack-token", + appToken: "xapp-raw-app-token", + }, + }, + }, + }, + mcp: { + headers: { Authorization: "Bearer opaque-migration-secret" }, + env: { GITHUB_TOKEN: "ghp_abcdefghijklmnopqrstuvwxyz0123456789", NODE_ENV: "test" }, + args: ["--api-key", "opaque-secret-value", "--verbose"], + }, + oauth: { + access_token: "opaque-access-value", + refresh_token: "opaque-refresh-value", + client_secret: "opaque-client-value", + }, + apiKey: "openshell:resolve:env:NVIDIA_API_KEY", + publicKey: "verify-me", + model: "keep-me", + }; + + expect(stripCredentials(fixture)).toEqual({ + channels: { + slack: { + accounts: { + default: { + botToken: CREDENTIAL_PLACEHOLDER, + appToken: CREDENTIAL_PLACEHOLDER, + }, + }, + }, + }, + mcp: { + headers: { Authorization: CREDENTIAL_PLACEHOLDER }, + env: { GITHUB_TOKEN: CREDENTIAL_PLACEHOLDER, NODE_ENV: "test" }, + args: ["--api-key", CREDENTIAL_PLACEHOLDER, "--verbose"], + }, + oauth: { + access_token: CREDENTIAL_PLACEHOLDER, + refresh_token: CREDENTIAL_PLACEHOLDER, + client_secret: CREDENTIAL_PLACEHOLDER, + }, + apiKey: "openshell:resolve:env:NVIDIA_API_KEY", + publicKey: "verify-me", + model: "keep-me", + }); + + expect( + sanitizeEnvFileContent( + [ + "CUSTOM=ghp_abcdefghijklmnopqrstuvwxyz0123456789", + "ENDPOINT=Bearer opaque-migration-secret", + "SAFE=openshell:resolve:env:SAFE", + "NODE_ENV=production", + "", + ].join("\n"), + ), + ).toBe( + [ + `CUSTOM=${CREDENTIAL_PLACEHOLDER}`, + `ENDPOINT=${CREDENTIAL_PLACEHOLDER}`, + "SAFE=openshell:resolve:env:SAFE", + "NODE_ENV=production", + "", + ].join("\n"), + ); + }); + + it("strips separated and inline CLI credentials without consuming later flags (#8291)", () => { + expect( + stripCredentials([ + "--api-key", + "-opaque-secret-value", + "--token=opaque-inline-value", + "--api-key=openshell:resolve:env:NVIDIA_API_KEY", + "--verbose", + ]), + ).toEqual([ + "--api-key", + CREDENTIAL_PLACEHOLDER, + `--token=${CREDENTIAL_PLACEHOLDER}`, + "--api-key=openshell:resolve:env:NVIDIA_API_KEY", + "--verbose", + ]); + }); + + it("classifies only credential-sensitive basenames (#8291)", () => { + expect(isSensitiveFile("auth.json")).toBe(true); + expect(isSensitiveFile("AUTH-PROFILES.JSON")).toBe(true); + expect(isSensitiveFile("config.json")).toBe(false); + }); +}); diff --git a/nemoclaw/tsconfig.shared.json b/nemoclaw/tsconfig.shared.json index 901561e8c9..ee8bcaf2ae 100644 --- a/nemoclaw/tsconfig.shared.json +++ b/nemoclaw/tsconfig.shared.json @@ -6,6 +6,7 @@ }, "include": [ "src/shared/banner-boundary.cts", + "src/shared/credential-filter-boundary.cts", "src/shared/openshell-policy-boundary.cts", "src/shared/sandbox-name.cts", "src/shared/snapshot-sanitizer-boundary.cts" diff --git a/nemoclaw/vitest.project.ts b/nemoclaw/vitest.project.ts index 43d3836abd..9972188ad6 100644 --- a/nemoclaw/vitest.project.ts +++ b/nemoclaw/vitest.project.ts @@ -5,6 +5,10 @@ import path from "node:path"; const repositoryRoot = path.resolve(import.meta.dirname, ".."); const canonicalBannerBoundary = path.resolve(import.meta.dirname, "src/shared/banner-boundary.cts"); +const canonicalCredentialFilterBoundary = path.resolve( + import.meta.dirname, + "src/shared/credential-filter-boundary.cts", +); const canonicalOpenShellPolicyBoundary = path.resolve( import.meta.dirname, "src/shared/openshell-policy-boundary.cts", @@ -48,6 +52,10 @@ const pluginVitestProjectOptions = { find: /^.*banner-boundary\.cjs$/, replacement: canonicalBannerBoundary, }, + { + find: /^.*credential-filter-boundary\.cjs$/, + replacement: canonicalCredentialFilterBoundary, + }, { find: /^.*openshell-policy-boundary\.cjs$/, replacement: canonicalOpenShellPolicyBoundary, diff --git a/src/lib/security/credential-filter.ts b/src/lib/security/credential-filter.ts index a817d36e5f..e87fa77041 100644 --- a/src/lib/security/credential-filter.ts +++ b/src/lib/security/credential-filter.ts @@ -1,14 +1,5 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// -// Shared credential-stripping logic for config files. -// -// Used by: -// - sandbox-state.ts (rebuild backup/restore) -// - migration-state.ts (host→sandbox onboarding migration) -// -// Credentials must never be baked into sandbox filesystems or local backups. -// They are injected at runtime via OpenShell's provider credential mechanism. import { randomUUID } from "node:crypto"; import { @@ -23,8 +14,33 @@ import { import { basename, dirname, join } from "node:path"; import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; -import { isObjectRecord } from "../core/json-types"; -import { hasPassCredentialSegment, SECRET_PATTERNS } from "./secret-patterns"; +import { + isConfigObject, + isConfigValue, + sanitizeEnvFileContent, + stripCredentials, +} from "../../../nemoclaw/dist/shared/credential-filter-boundary.cjs"; +import type { + ConfigObject, + ConfigValue, +} from "../../../nemoclaw/dist/shared/credential-filter-boundary.cjs"; + +export { + CREDENTIAL_PLACEHOLDER, + CREDENTIAL_SENSITIVE_BASENAMES, + isConfigObject, + isConfigValue, + isCredentialField, + isSafeCredentialPlaceholder, + isSensitiveFile, + sanitizeEnvFileContent, + stripCredentials, + valueLooksLikeSecret, +} from "../../../nemoclaw/dist/shared/credential-filter-boundary.cjs"; +export type { + ConfigObject, + ConfigValue, +} from "../../../nemoclaw/dist/shared/credential-filter-boundary.cjs"; function parseJson(text: string): T { return JSON.parse(text); @@ -57,41 +73,7 @@ function writeFileAtomically(filePath: string, contents: string): void { renameSync(tmpPath, filePath); } -/** - * JSON-like configuration value supported by credential stripping. - */ -export type ConfigValue = - | null - | undefined - | boolean - | number - | string - | ConfigValue[] - | ConfigObject; - -/** - * JSON-like configuration object supported by credential stripping. - */ -export type ConfigObject = { [key: string]: ConfigValue }; - -const CREDENTIAL_PLACEHOLDER = "[STRIPPED_BY_MIGRATION]"; - -/** - * File basenames that contain sensitive auth material and should be - * excluded from backups entirely. - */ -export const CREDENTIAL_SENSITIVE_BASENAMES = new Set([ - "auth-profiles.json", - "auth.json", - "chatgpt-auth.json", -]); - -/** - * Dependency lockfiles may contain package metadata that resembles credentials - * (for example package names or tarball URLs with `sk-` substrings). They do - * not store NemoClaw runtime credentials and should not fail snapshot leak - * checks. - */ +/** Dependency lockfiles do not store NemoClaw runtime credentials. */ const SNAPSHOT_CREDENTIAL_SCAN_EXCLUDED_BASENAMES = new Set([ ".package-lock.json", "package-lock.json", @@ -106,293 +88,7 @@ export function isDependencyLockfile(filename: string): boolean { return SNAPSHOT_CREDENTIAL_SCAN_EXCLUDED_BASENAMES.has(basename(filename).toLowerCase()); } -/** - * Credential field names that MUST be stripped from config files. - */ -const CREDENTIAL_FIELDS = new Set([ - "apikey", - "api_key", - "token", - "secret", - "password", - "pass", - "passwd", - "resolvedkey", -]); - -/** - * Pattern-based detection for credential field names not covered by the - * explicit set above. Matches common suffixes like accessToken, privateKey, - * clientSecret, etc. `bot`/`app` cover the OpenClaw channel token fields - * (`botToken`, `appToken`) used by Slack/Telegram accounts. - */ -const CREDENTIAL_FIELD_PATTERN = - /^(?:(?:personal[._-]?)?access|refresh|client|bearer|oauth|auth|api|private|public|signing|session|bot|app|resolved)[._-]?(?:tokens?|keys?|secrets?|passwords?|passphrases?|credentials?)$/i; - -/** - * Environment-variable-style secret names (SCREAMING_SNAKE_CASE) such as an MCP - * server's `env: { GITHUB_TOKEN, BRAVE_API_KEY, TOKEN }` block. These are not - * camelCase, so the suffix pattern above misses them. Matches an all-uppercase - * name that is, or ends in, a secret word (`TOKEN`, `KEY`, `SECRET`, - * `PASSWORD`, `PASSWD`, `PASS`, `PASSPHRASE`, `CREDENTIAL`, optionally pluralized) — covering both - * the prefixed (`GITHUB_TOKEN`) and bare (`TOKEN`) forms — while leaving benign - * env vars like `NODE_ENV`, `LOG_LEVEL`, or `PATH` untouched. - */ -const ENV_SECRET_FIELD_PATTERN = - /^(?:[A-Z0-9]+_)*(?:TOKEN|KEY|SECRET|PASSWORD|PASSWD|PASS|PASSPHRASE|CREDENTIAL)S?$/; - -/** - * Well-known HTTP auth header names (matched case-insensitively) whose entire - * value is a credential but that do not end in a secret word. Remote MCP - * servers in openclaw.json may carry these (issue #5027). - */ -const CREDENTIAL_HEADER_NAMES: ReadonlySet = new Set([ - "authorization", - "proxy-authorization", - "cookie", - "set-cookie", -]); - -/** - * Hyphen-delimited header-style names ending in a secret word, e.g. - * `X-API-Key`, `X-API-Token`, `X-Auth-Token`, `Private-Token`. The required - * hyphen before the secret word keeps camelCase settings such as `maxTokens` - * (no hyphen) from matching, so only header-shaped names are scrubbed. - */ -const HEADER_CREDENTIAL_PATTERN = /-(?:key|token|secret|password|passphrase|credential|auth)s?$/i; - -/** - * Public keys are verification material, not secrets, so they must never be - * scrubbed even though they end in `Key`/`KEY`. Covers `publicKey`, - * `PUBLIC_KEY`, `public-key`, and prefixed forms like `X-Public-Key` / - * `GITHUB_PUBLIC_KEY`. Checked before the secret patterns below. - */ -const PUBLIC_KEY_FIELD_PATTERN = /(?:^|[._-])public[._-]?keys?$/i; - -/** - * Check whether a field name should be treated as credential-bearing. - */ -export function isCredentialField(key: string): boolean { - if (PUBLIC_KEY_FIELD_PATTERN.test(key)) return false; - return ( - CREDENTIAL_FIELDS.has(key.toLowerCase()) || - CREDENTIAL_FIELD_PATTERN.test(key) || - hasPassCredentialSegment(key) || - ENV_SECRET_FIELD_PATTERN.test(key) || - HEADER_CREDENTIAL_PATTERN.test(key) || - CREDENTIAL_HEADER_NAMES.has(key.toLowerCase()) - ); -} - -/** - * Value-level backstop: whether a string value matches a known secret format - * (provider key prefixes like `sk-`/`ghp_`/`xoxb-`, `Bearer `, etc.). - * This scrubs raw secrets that sit under an unrecognized key name — e.g. a - * custom MCP auth header — without over-stripping benign settings, since these - * patterns only match credential-shaped values. Resolve placeholders are - * checked by the caller and never reach here. - */ -export function valueLooksLikeSecret(value: string): boolean { - for (const pattern of SECRET_PATTERNS) { - // SECRET_PATTERNS carry the global flag, so reset before each stateful test. - pattern.lastIndex = 0; - if (pattern.test(value)) return true; - } - return false; -} - -/** - * Value patterns that are references to a credential, not the credential - * itself. NemoClaw never stores raw secrets in agent config files; provider - * keys and channel tokens are written as OpenShell `resolve:env:` - * placeholders (the `` is an environment variable name, not secret - * material) and resolved at gateway launch from OpenShell provider storage. - * Slack's Bolt SDK rejects a bare placeholder, so its tokens use the - * `xoxb`/`xapp` prefixed variants. `unused` is the sentinel OpenClaw writes for - * a provider whose auth is proxy-injected. Preserving these lets a sanitized - * backup restore into a working config (issue #5027) instead of replacing the - * reference with a dead `[STRIPPED_BY_MIGRATION]` marker. - */ -const SAFE_CREDENTIAL_PLACEHOLDER_PATTERNS: readonly RegExp[] = [ - /^openshell:resolve:env:[A-Za-z0-9_]+$/, - // Auth headers carry the resolve reference after a `Bearer ` scheme prefix. - /^Bearer\s+openshell:resolve:env:[A-Za-z0-9_]+$/i, - /^xoxb-OPENSHELL-RESOLVE-ENV-[A-Za-z0-9_]+$/, - /^xapp-OPENSHELL-RESOLVE-ENV-[A-Za-z0-9_]+$/, -]; - -const SAFE_CREDENTIAL_PLACEHOLDER_LITERALS: ReadonlySet = new Set([ - "unused", - CREDENTIAL_PLACEHOLDER, -]); - -/** - * Whether a value under a credential-named field is a non-secret reference - * that must be preserved rather than scrubbed. - */ -export function isSafeCredentialPlaceholder(value: ConfigValue): boolean { - if (typeof value !== "string") return false; - // Accept an optional `Bearer ` auth scheme in front of a safe literal, e.g. - // `Authorization: "Bearer unused"` for proxy-injected credentials. - const withoutScheme = value.replace(/^Bearer\s+/i, ""); - if ( - SAFE_CREDENTIAL_PLACEHOLDER_LITERALS.has(value) || - SAFE_CREDENTIAL_PLACEHOLDER_LITERALS.has(withoutScheme) - ) { - return true; - } - return SAFE_CREDENTIAL_PLACEHOLDER_PATTERNS.some((pattern) => pattern.test(value)); -} - -/** - * Narrow an unknown value to a JSON-like configuration object. - */ -export function isConfigObject(value: ConfigValue | object): value is ConfigObject { - return isObjectRecord(value); -} - -/** - * Narrow an unknown value to a JSON-like configuration value. - */ -export function isConfigValue(value: ConfigValue | object): value is ConfigValue { - if (value === null || value === undefined) return true; - if (typeof value === "boolean" || typeof value === "number" || typeof value === "string") { - return true; - } - if (Array.isArray(value)) { - return value.every((entry) => isConfigValue(entry)); - } - if (!isConfigObject(value)) { - return false; - } - - const prototype = Object.getPrototypeOf(value); - if (prototype !== Object.prototype && prototype !== null) { - return false; - } - - return Object.values(value).every((entry) => isConfigValue(entry)); -} - -/** - * Recursively strip credential fields from a JSON-like object. - * Returns a new object with sensitive values replaced by a placeholder. - */ -export function stripCredentials(obj: null): null; -export function stripCredentials(obj: undefined): undefined; -export function stripCredentials(obj: boolean): boolean; -export function stripCredentials(obj: number): number; -export function stripCredentials(obj: string): string; -export function stripCredentials(obj: T): T; -export function stripCredentials(obj: T): T; -export function stripCredentials(obj: ConfigValue): ConfigValue; -export function stripCredentials(obj: ConfigValue): ConfigValue { - if (obj === null || obj === undefined) return obj; - if (typeof obj !== "object") return obj; - if (Array.isArray(obj)) { - // Arrays (e.g. an MCP server's `args`) can carry secrets both by shape and - // by CLI-flag context: `["--api-key", ""]` or `"--api-key="`. - return obj.map((value, index) => scrubArrayElement(value, obj[index - 1])); - } - - const result: ConfigObject = {}; - for (const [key, value] of Object.entries(obj)) { - if (isCredentialField(key)) { - // Preserve unset credential fields and non-secret references (OpenShell - // resolve placeholders, the `unused` sentinel); scrub raw secrets. - result[key] = - value === null || value === undefined || isSafeCredentialPlaceholder(value) - ? value - : CREDENTIAL_PLACEHOLDER; - } else { - result[key] = scrubConfigValue(value); - } - } - return result; -} - -/** - * Scrub a value found under a non-credential key (or in an array): a raw secret - * detected by shape is replaced with the placeholder; resolve references are - * preserved; everything else recurses through stripCredentials. - */ -function scrubConfigValue(value: ConfigValue): ConfigValue { - if (typeof value === "string") { - if (isSafeCredentialPlaceholder(value)) return value; - return valueLooksLikeSecret(value) ? CREDENTIAL_PLACEHOLDER : value; - } - return stripCredentials(value); -} - -/** Bare flag name from a CLI token: `--api-key` → `api-key`, else null. */ -function cliFlagName(token: string): string | null { - const match = /^--?([A-Za-z0-9][A-Za-z0-9._-]*)$/.exec(token); - return match ? match[1] : null; -} - -/** - * Scrub one array element, adding CLI-flag context to the shape backstop: a - * value passed inline as `--api-key=` or positionally right after a - * credential flag (`["--api-key", ""]`) is scrubbed even when the - * secret itself is opaque (no recognizable prefix). MCP servers commonly pass - * credentials this way. - */ -function scrubArrayElement(value: ConfigValue, previous: ConfigValue): ConfigValue { - if (typeof value !== "string") return stripCredentials(value); - if (isSafeCredentialPlaceholder(value)) return value; - - // Inline `--flag=value` form. - const eq = value.indexOf("="); - if (eq > 0 && value.startsWith("-")) { - const flagName = cliFlagName(value.slice(0, eq)); - if (flagName && isCredentialField(flagName)) { - const inlineValue = value.slice(eq + 1); - return isSafeCredentialPlaceholder(inlineValue) - ? value - : `${value.slice(0, eq)}=${CREDENTIAL_PLACEHOLDER}`; - } - } - - // Positional value immediately after a credential flag. Skip when this token - // is itself a flag (starts with `-`) so a value-less flag does not swallow - // the next flag. - if (!value.startsWith("-") && typeof previous === "string") { - const prevFlag = cliFlagName(previous); - if (prevFlag && isCredentialField(prevFlag)) return CREDENTIAL_PLACEHOLDER; - } - - return valueLooksLikeSecret(value) ? CREDENTIAL_PLACEHOLDER : value; -} - -/** - * Strip credential fields from a KEY=value env file body. - * Uses the same field-name rules as JSON scrubbing so `DB_PASS` and - * `PASSPHRASE` are stripped while benign names like `KEYBOARD_LAYOUT` - * and `NODE_ENV` are preserved. - */ -export function sanitizeEnvFileContent(content: string): string { - return content - .split("\n") - .map((line) => { - const trimmed = line.trim(); - if (trimmed === "" || trimmed.startsWith("#")) return line; - const eq = line.indexOf("="); - if (eq <= 0) return line; - const rawKey = line.slice(0, eq).trim(); - // Shell-sourced .env files often use `export KEY=value`. - const key = rawKey.replace(/^export\s+/i, "").trim(); - const value = line.slice(eq + 1); - if (isSafeCredentialPlaceholder(value)) return line; - if (!key) return line; - if (!isCredentialField(key) && !valueLooksLikeSecret(value)) return line; - return `${line.slice(0, eq)}=${CREDENTIAL_PLACEHOLDER}`; - }) - .join("\n"); -} - -/** - * Strip credential lines from a `.env` file in-place. - */ +/** Strip credential lines from a `.env` file in-place. */ export function sanitizeEnvFile( filePath: string, writeSanitized: (targetPath: string, contents: string) => void = writeFileAtomically, @@ -423,7 +119,7 @@ function toConfigValue(value: unknown): ConfigValue | typeof UNREPRESENTABLE_CON } return items; } - if (!isObjectRecord(value)) return UNREPRESENTABLE_CONFIG_VALUE; + if (!isConfigObject(value)) return UNREPRESENTABLE_CONFIG_VALUE; const result: ConfigObject = {}; for (const [key, entry] of Object.entries(value)) { const converted = toConfigValue(entry); @@ -461,12 +157,7 @@ export function sanitizeYamlConfigContent(rawConfig: string): string | null { * * The filename is used only to decide whether a non-JSON document is an * allowed YAML target. Returns null when the input cannot be sanitized. - * - * A credential-free document is returned byte for byte. Re-emitting it through - * `JSON.stringify` would change indentation, key spacing, and the trailing - * newline without redacting anything. Hermes hashes its WhatsApp bridge manifest - * to decide whether the installed dependencies are current, and reads that - * reformat as a change. + * Credential-free JSON is returned byte for byte. */ export function sanitizeConfigFileContent(configName: string, rawConfig: string): string | null { try { @@ -478,8 +169,6 @@ export function sanitizeConfigFileContent(configName: string, rawConfig: string) config = withoutGateway; } const stripped = stripCredentials(config); - // Compare against the document as parsed, before the `gateway` section was - // dropped, so removing that section still counts as a change. if (JSON.stringify(stripped) === JSON.stringify(parsed)) return rawConfig; return JSON.stringify(stripped, null, 2); } catch { @@ -493,11 +182,7 @@ export function sanitizeConfigFileContent(configName: string, rawConfig: string) return null; } -/** - * Strip credential fields from a Hermes YAML config file in-place. - * Removes the "gateway" section when present (auth tokens — regenerated - * at startup), matching JSON sanitization. - */ +/** Strip credential fields from a Hermes YAML config file in-place. */ export function sanitizeYamlConfigFile( configPath: string, writeSanitized: (targetPath: string, contents: string) => void = writeFileAtomically, @@ -517,11 +202,7 @@ export function sanitizeYamlConfigFile( /** * Strip credential fields from a JSON or YAML config file in-place. - * Removes the "gateway" section (contains auth tokens — regenerated at startup). - * JSON is preferred when the file parses as JSON; otherwise YAML is tried - * so Hermes `config.yaml` secrets are scrubbed from rebuild backups. - * Returns false when a YAML/YML target cannot be sanitized so callers can - * fail closed instead of retaining the raw file. + * Returns false when the input cannot be sanitized safely. */ export function sanitizeConfigFile(configPath: string): boolean { const rawConfig = readRegularFileNoFollow(configPath); @@ -537,17 +218,7 @@ export function sanitizeConfigFile(configPath: string): boolean { } } -/** - * Check if a filename should be excluded from backups entirely. - */ -export function isSensitiveFile(filename: string): boolean { - return CREDENTIAL_SENSITIVE_BASENAMES.has(filename.toLowerCase()); -} - -/** - * Return whether a snapshot file should be scanned for credential-looking - * payloads by coarse-grained E2E leak checks. - */ +/** Return whether coarse E2E leak checks should scan a snapshot file. */ export function shouldScanSnapshotFileForCredentials(filename: string): boolean { const normalizedBasename = basename(filename).toLowerCase(); if (isDependencyLockfile(normalizedBasename)) return false; diff --git a/src/lib/security/secret-patterns.ts b/src/lib/security/secret-patterns.ts index 08107db5e4..e595fefca5 100644 --- a/src/lib/security/secret-patterns.ts +++ b/src/lib/security/secret-patterns.ts @@ -1,100 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -/** - * Canonical secret redaction patterns — single source of truth. - * - * All TypeScript consumers import through src/lib/security/redact.ts (#2381). - * debug.sh delegates to the compiled redact module when node is available; - * its sed fallback only covers the prefixes in EXPECTED_SHELL_PREFIXES. - * - * Ref: https://github.com/NVIDIA/NemoClaw/issues/2381 - * Ref: https://github.com/NVIDIA/NemoClaw/issues/1736 - */ - -/** Token-prefix patterns that match standalone (no context needed). */ -export const TOKEN_PREFIX_PATTERNS: RegExp[] = [ - // NVIDIA - /nvapi-[A-Za-z0-9_-]{10,}/g, - /nvcf-[A-Za-z0-9_-]{10,}/g, - // GitHub - /ghp_[A-Za-z0-9_-]{10,}/g, - /(?:github_pat_)[A-Za-z0-9_]{30,}/g, - // OpenAI (sk-proj- before sk- so the more specific prefix matches first) - /sk-proj-[A-Za-z0-9_-]{10,}/g, - /sk-ant-[A-Za-z0-9_-]{10,}/g, - /sk-[A-Za-z0-9_-]{20,}/g, - // Slack (consolidated class covers xoxb-, xoxp-, xoxa-, xoxs-, xapp-) - /(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}/g, - // AWS access key IDs (AKIA = long-term, ASIA = temporary/session) - /A(?:K|S)IA[A-Z0-9]{16}/g, - // HuggingFace - /hf_[A-Za-z0-9]{10,}/g, - // GitLab - /glpat-[A-Za-z0-9_-]{10,}/g, - // Groq - /gsk_[A-Za-z0-9]{10,}/g, - // PyPI - /pypi-[A-Za-z0-9_-]{10,}/g, - // Telegram bot tokens (8-10 digit bot ID + 35-char secret) - /\bbot\d{8,10}:[A-Za-z0-9_-]{35}\b/g, - /\b\d{8,10}:[A-Za-z0-9_-]{35}\b/g, - // Discord bot tokens (base64 user ID . timestamp . HMAC) - /\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b/g, - // Tavily - /tvly-[A-Za-z0-9_-]{10,}/g, - // LangSmith (personal access tokens: lsv2_pt_; service keys: lsv2_sk_) - // Match every underscore-delimited segment so redaction cannot expose a key tail. - /lsv2_(?:pt|sk)_[A-Za-z0-9]{10,}(?:_[A-Za-z0-9]+)*/g, -]; - -/** Structured standalone tokens without a provider-specific prefix. */ -export const STRUCTURED_TOKEN_PATTERNS: RegExp[] = [ - // Compact JWT protected headers are JSON objects, whose base64url encoding - // starts with eyJ. Provider-prefixed opaque tokens are covered above. - /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/g, -]; - -/** Context-anchored patterns (require a prefix like KEY=, Bearer, etc.). */ -export const CONTEXT_PATTERNS: RegExp[] = [ - /(?<=Bearer\s+)[A-Za-z0-9_.+/=-]{10,}/gi, - /(?<=(?:^|[^A-Za-z0-9])(?:[A-Za-z0-9]{1,128}_(?:KEY|TOKEN|SECRET|CREDENTIAL|PASSWORD|PASSWD|PASS)|(?:X[-_])?API[-_]KEY|TOKEN|SECRET|CREDENTIAL|PASSWORD|PASSWD|PASS)["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/gi, - /(?<=(?:^|[^A-Za-z0-9])(?:[A-Za-z0-9]{1,128}(?:Token|Secret|Credential)|[A-Za-z0-9]{0,128}(?:[Aa]ccess|[Rr]efresh|[Cc]lient|[Bb]earer|[Aa]uth|[Aa][Pp][Ii]|[Pp]rivate|[Ss]igning|[Ss]ession|[Bb]ot|[Aa]pp|[Rr]esolved)Key|[A-Za-z0-9]{1,128}(?:Password|Passwd|Pass))["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/g, - /(?<=(?:^|[^A-Za-z0-9])KEY["']?(?:[ \t]{0,32}[=:][ \t]{0,32}|[ \t]{1,32})["']?)[^\s'"]{10,}/g, -]; - -/** Match pass/passwd only as a complete or terminal credential-name segment. */ -export function hasPassCredentialSegment(key: string): boolean { - const normalized = key - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2") - .replace(/([a-z0-9])([A-Z])/g, "$1_$2") - .replace(/[^A-Za-z0-9]+/g, "_") - .replace(/^_+|_+$/g, "") - .toLowerCase(); - return ( - normalized === "pass" || - normalized === "passwd" || - normalized.endsWith("_pass") || - normalized.endsWith("_passwd") - ); -} - -/** Multi-line or JSON-escaped secret blocks that do not have a token prefix. */ -export const SECRET_BLOCK_PATTERNS: RegExp[] = [ - /-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9]+ )?PRIVATE KEY-----/g, -]; - -/** All secret patterns combined. */ -export const SECRET_PATTERNS: RegExp[] = [ - ...TOKEN_PREFIX_PATTERNS, - ...STRUCTURED_TOKEN_PATTERNS, - ...SECRET_BLOCK_PATTERNS, - ...CONTEXT_PATTERNS, -]; - -/** - * Token prefixes covered by the debug.sh sed fallback. - * The primary path delegates to node; this fallback only runs when - * node or dist/ is unavailable. Consistency test verifies these appear. - */ -export const EXPECTED_SHELL_PREFIXES = ["nvapi-", "nvcf-", "ghp_", "sk-", "tvly-"]; +// sourceOfTruth: nemoclaw/src/shared/credential-filter-boundary.cts +// generatedBoundary: build:cli emits the canonical .cjs/.d.cts before this +// module is compiled. Keep this compatibility import path implementation-free. +export { + CONTEXT_PATTERNS, + EXPECTED_SHELL_PREFIXES, + hasPassCredentialSegment, + SECRET_BLOCK_PATTERNS, + SECRET_PATTERNS, + STRUCTURED_TOKEN_PATTERNS, + TOKEN_PREFIX_PATTERNS, +} from "../../../nemoclaw/dist/shared/credential-filter-boundary.cjs"; diff --git a/test/credential-filter-parity.test.ts b/test/credential-filter-parity.test.ts deleted file mode 100644 index 3637cf9fa7..0000000000 --- a/test/credential-filter-parity.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - isCredentialField as isPluginCredentialField, - isSafeCredentialPlaceholder as isPluginSafePlaceholder, - valueLooksLikeSecret as pluginValueLooksLikeSecret, - sanitizeEnvFileContent as sanitizePluginEnvFileContent, - stripCredentials as stripPluginCredentials, -} from "../nemoclaw/src/security/credential-filter.js"; -import { - isCredentialField as isSharedCredentialField, - isSafeCredentialPlaceholder as isSharedSafePlaceholder, - sanitizeEnvFileContent as sanitizeSharedEnvFileContent, - valueLooksLikeSecret as sharedValueLooksLikeSecret, - stripCredentials as stripSharedCredentials, -} from "../src/lib/security/credential-filter.js"; - -describe("credential filter parity", () => { - it("keeps plugin and shared classification rules aligned", () => { - const fields = [ - "botToken", - "bot_token", - "appToken", - "access_token", - "personal_access_token", - "refresh-token", - "client_secret", - "auth_token", - "oauth_token", - "apikey", - "Token", - "GITHUB_TOKEN", - "Authorization", - "X-API-Key", - "DB_PASS", - "publicKey", - "NODE_ENV", - "model", - ]; - const values = [ - "ghp_abcdefghijklmnopqrstuvwxyz0123456789", - "Bearer opaque-migration-secret", - "openshell:resolve:env:GITHUB_TOKEN", - "Bearer openshell:resolve:env:REMOTE_MCP_TOKEN", - "keep-me", - ]; - - for (const field of fields) { - expect(isPluginCredentialField(field), field).toBe(isSharedCredentialField(field)); - } - for (const value of values) { - expect(pluginValueLooksLikeSecret(value), value).toBe(sharedValueLooksLikeSecret(value)); - expect(isPluginSafePlaceholder(value), value).toBe(isSharedSafePlaceholder(value)); - } - }); - - it("produces identical object and env-file sanitization", () => { - const fixture = { - channels: { - slack: { - accounts: { - default: { - botToken: "xoxb-raw-slack-token", - appToken: "xapp-raw-app-token", - }, - }, - }, - }, - mcp: { - headers: { Authorization: "Bearer opaque-migration-secret" }, - env: { GITHUB_TOKEN: "ghp_abcdefghijklmnopqrstuvwxyz0123456789", NODE_ENV: "test" }, - args: ["--api-key", "opaque-secret-value", "--verbose"], - }, - oauth: { - access_token: "opaque-access-value", - refresh_token: "opaque-refresh-value", - client_secret: "opaque-client-value", - }, - model: "keep-me", - }; - const envFixture = [ - "CUSTOM=ghp_abcdefghijklmnopqrstuvwxyz0123456789", - "ENDPOINT=Bearer opaque-migration-secret", - "NODE_ENV=production", - "", - ].join("\n"); - - expect(stripPluginCredentials(fixture)).toEqual(stripSharedCredentials(fixture)); - expect(sanitizePluginEnvFileContent(envFixture)).toBe(sanitizeSharedEnvFileContent(envFixture)); - }); -}); diff --git a/test/nemoclaw-plugin-secret-pattern-parity.test.ts b/test/nemoclaw-plugin-secret-pattern-parity.test.ts deleted file mode 100644 index 7704aa8df7..0000000000 --- a/test/nemoclaw-plugin-secret-pattern-parity.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; -import { CONTEXT_SECRET_PATTERNS } from "../nemoclaw/src/security/credential-filter.ts"; -import { CONTEXT_PATTERNS } from "../src/lib/security/secret-patterns.ts"; - -function fingerprint(patterns: readonly RegExp[]): string[] { - return patterns.map((pattern) => `${pattern.source}::${pattern.flags}`); -} - -describe("NemoClaw plugin secret-pattern parity", () => { - it("matches every canonical context pattern source and flag", () => { - expect(CONTEXT_PATTERNS.length).toBeGreaterThan(0); - expect(fingerprint(CONTEXT_SECRET_PATTERNS)).toEqual(fingerprint(CONTEXT_PATTERNS)); - }); -}); diff --git a/test/package-contract/credential-filter-boundary.test.ts b/test/package-contract/credential-filter-boundary.test.ts new file mode 100644 index 0000000000..9e7d727d59 --- /dev/null +++ b/test/package-contract/credential-filter-boundary.test.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const repoRoot = path.join(import.meta.dirname, "..", ".."); +const url = (...segments: string[]) => pathToFileURL(path.join(repoRoot, ...segments)).href; + +describe("credential filter package boundary", () => { + it("resolves both package wrappers to one generated implementation (#8291)", () => { + // Native resolution bypasses the Vitest source alias so this checks the + // generated files that the published CLI and plugin load. + const script = + `const cli = await import(${JSON.stringify(url("dist/lib/security/credential-filter.js"))});` + + `const plugin = await import(${JSON.stringify(url("nemoclaw/dist/security/credential-filter.js"))});` + + `const patterns = await import(${JSON.stringify(url("dist/lib/security/secret-patterns.js"))});` + + `const boundary = await import(${JSON.stringify( + url("nemoclaw/dist/shared/credential-filter-boundary.cjs"), + )});` + + `const fixture = {headers:{Authorization:"Bearer opaque-package-contract-secret"},args:["--api-key","opaque-value"],model:"keep-me"};` + + `process.stdout.write(JSON.stringify([cli.stripCredentials === boundary.stripCredentials, plugin.stripCredentials === boundary.stripCredentials, cli.sanitizeEnvFileContent === boundary.sanitizeEnvFileContent, plugin.sanitizeEnvFileContent === boundary.sanitizeEnvFileContent, patterns.SECRET_PATTERNS === boundary.SECRET_PATTERNS, cli.stripCredentials(fixture), plugin.stripCredentials(fixture)]));`; + const output = execFileSync(process.execPath, ["--input-type=module", "-e", script], { + cwd: repoRoot, + encoding: "utf8", + timeout: 30_000, + }); + const placeholder = "[STRIPPED_BY_MIGRATION]"; + const expected = { + headers: { Authorization: placeholder }, + args: ["--api-key", placeholder], + model: "keep-me", + }; + expect(JSON.parse(output)).toEqual([true, true, true, true, true, expected, expected]); + }); + + it("includes the CommonJS module and declaration in the plugin build (#8291)", () => { + const sharedDirectory = path.join(repoRoot, "nemoclaw", "dist", "shared"); + expect(fs.existsSync(path.join(sharedDirectory, "credential-filter-boundary.cjs"))).toBe(true); + expect(fs.existsSync(path.join(sharedDirectory, "credential-filter-boundary.d.cts"))).toBe( + true, + ); + expect(fs.existsSync(path.join(sharedDirectory, "credential-filter-boundary.js"))).toBe(false); + }); +}); diff --git a/test/plugin-vitest-project.test.ts b/test/plugin-vitest-project.test.ts index e0e0be2321..8ce92d9c43 100644 --- a/test/plugin-vitest-project.test.ts +++ b/test/plugin-vitest-project.test.ts @@ -61,6 +61,13 @@ describe("plugin Vitest project contract", () => { find: /^.*banner-boundary\.cjs$/, replacement: path.join(repositoryRoot, "nemoclaw/src/shared/banner-boundary.cts"), }, + { + find: /^.*credential-filter-boundary\.cjs$/, + replacement: path.join( + repositoryRoot, + "nemoclaw/src/shared/credential-filter-boundary.cts", + ), + }, { find: /^.*openshell-policy-boundary\.cjs$/, replacement: path.join(repositoryRoot, "nemoclaw/src/shared/openshell-policy-boundary.cts"), diff --git a/vitest.config.ts b/vitest.config.ts index d734c4e851..f15325716e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -21,6 +21,9 @@ const { isCi, silent } = resolveVitestFeedback(); const LIVE_E2E_PROJECT_TIMEOUT_MS = 30 * 60 * 1000; const runLiveE2E = shouldRunLiveE2E(); const canonicalBannerBoundary = path.resolve("nemoclaw/src/shared/banner-boundary.cts"); +const canonicalCredentialFilterBoundary = path.resolve( + "nemoclaw/src/shared/credential-filter-boundary.cts", +); const canonicalOpenShellPolicyBoundary = path.resolve( "nemoclaw/src/shared/openshell-policy-boundary.cts", ); @@ -36,6 +39,10 @@ const canonicalSourceAliases = [ find: /^.*banner-boundary\.cjs$/, replacement: canonicalBannerBoundary, }, + { + find: /^.*credential-filter-boundary\.cjs$/, + replacement: canonicalCredentialFilterBoundary, + }, { find: /^.*openshell-policy-boundary\.cjs$/, replacement: canonicalOpenShellPolicyBoundary,