diff --git a/.github/workflows/codebase-growth-guardrails.yaml b/.github/workflows/codebase-growth-guardrails.yaml index fec12fcc875..c7ca5d44b9d 100644 --- a/.github/workflows/codebase-growth-guardrails.yaml +++ b/.github/workflows/codebase-growth-guardrails.yaml @@ -107,7 +107,32 @@ jobs: EOF exit 1 + - name: Check out the trusted base revision + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + + - name: Detect guardrail tools on the base revision + id: tools + # The tools live in the base tree. On the PR that first adds them, the + # base revision predates them, so skip until the change lands on base. + run: | + set -euo pipefail + if [ -f tools/growth-guardrails/test-size-budget.mts ] \ + && [ -f tools/growth-guardrails/test-conditionals.mts ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "Trusted base revision does not yet contain the growth-guardrail tools; the policy applies once this change lands on the base branch." + fi + + - name: Install trusted dependencies + if: steps.tools.outputs.present == 'true' + run: npm ci --ignore-scripts --no-audit --no-fund + - name: Require changed test files to stay within size budget + if: steps.tools.outputs.present == 'true' env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -117,271 +142,10 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | set -euo pipefail - - node <<'NODE' - // Fetch every blob we need in batched GraphQL queries instead of N - // sequential REST /contents/ calls. Each REST call has no retry and - // the endpoint intermittently returns 5xx; N calls per run compounded - // that flake into a ~30-50% job failure rate on fork PRs. This step - // now issues ~2 GraphQL requests (BASE budget + one batched HEAD - // query for budget/legacy/changed files) and retries each outbound - // request on 5xx and network errors. Behaviour is otherwise - // identical to the previous implementation. - const BUDGET_FILE = "ci/test-file-size-budget.json"; - const FALLBACK_BUDGET = '{"defaultMaxLines":1500,"legacyMaxLines":{}}'; - const TEST_FILE_RE = /^(test|src|nemoclaw\/src)\/.*\.(test|spec)\.(ts|js|mts|mjs|cts|cjs)$/; - const GRAPHQL_URL = "https://api.github.com/graphql"; - const GRAPHQL_BATCH_SIZE = 25; - const RETRY_ATTEMPTS = 4; - const RETRY_BASE_MS = 250; - const RETRY_MAX_MS = 4000; - const { BASE_SHA, GH_TOKEN, HEAD_REPO, HEAD_SHA, PR_NUMBER, REPO } = process.env; - const headers = { Authorization: `Bearer ${GH_TOKEN}`, "X-GitHub-Api-Version": "2022-11-28" }; - const violations = []; - - function countLines(text) { - return text === "" ? 0 : (text.match(/\r\n|\r|\n/g)?.length ?? 0) + (/(?:\r\n|\r|\n)$/.test(text) ? 0 : 1); - } - - function parseBudget(text, label) { - const budget = JSON.parse(text); - const legacyMaxLines = budget.legacyMaxLines ?? {}; - if (!Number.isInteger(budget.defaultMaxLines) || budget.defaultMaxLines <= 0) { - throw new Error(`${label} must define positive integer defaultMaxLines`); - } - if (typeof legacyMaxLines !== "object" || legacyMaxLines === null || Array.isArray(legacyMaxLines)) { - throw new Error(`${label} legacyMaxLines must be an object`); - } - for (const [file, maxLines] of Object.entries(legacyMaxLines)) { - if (!Number.isInteger(maxLines) || maxLines <= 0) { - throw new Error(`${label} has invalid legacy budget for ${file}: ${maxLines}`); - } - } - return { defaultMaxLines: budget.defaultMaxLines, legacyMaxLines }; - } - - function isTransient(status) { - return status === 408 || status === 425 || status === 429 || (status >= 500 && status <= 599); - } - - async function withRetry(label, fn) { - let lastError; - for (let attempt = 1; attempt <= RETRY_ATTEMPTS; attempt += 1) { - try { - return await fn(); - } catch (error) { - lastError = error; - const retriable = error && (error.transient === true || /HTTP (408|425|429|5\d{2})/.test(String(error.message ?? ""))); - if (!retriable || attempt === RETRY_ATTEMPTS) throw error; - const jitter = Math.random() * RETRY_BASE_MS; - const delay = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** (attempt - 1)) + jitter; - console.error(`retry: ${label} attempt ${attempt} failed (${error.message}); sleeping ${Math.round(delay)}ms`); - await new Promise((resolve) => setTimeout(resolve, delay)); - } - } - throw lastError; - } - - async function requestJson(url, init) { - let response; - try { - response = await fetch(url, init); - } catch (error) { - const wrapped = new Error(`${url}: network error ${error?.message ?? error}`); - wrapped.transient = true; - throw wrapped; - } - if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); - return response.json(); - } - - async function getJson(url) { - return withRetry(url, () => requestJson(url, { headers })); - } - - async function graphql(query, variables) { - const body = JSON.stringify({ query, variables }); - const label = `graphql ${variables?.owner ?? ""}/${variables?.name ?? ""}@${variables?.oid ?? ""}`; - return withRetry(label, async () => { - const payload = await requestJson(GRAPHQL_URL, { - method: "POST", - headers: { ...headers, "Content-Type": "application/json" }, - body, - }); - if (Array.isArray(payload.errors) && payload.errors.length > 0) { - const messages = payload.errors.map((entry) => entry.message).join("; "); - const transient = payload.errors.some((entry) => entry.type === "RATE_LIMITED" || /timed? *out|unavailable|internal/i.test(entry.message ?? "")); - const error = new Error(`${label}: GraphQL errors: ${messages}`); - if (transient) error.transient = true; - throw error; - } - return payload.data; - }); - } - - async function getPullFiles() { - const files = []; - for (let page = 1; ; page += 1) { - const batch = await getJson(`https://api.github.com/repos/${REPO}/pulls/${PR_NUMBER}/files?per_page=100&page=${page}`); - files.push(...batch); - if (batch.length < 100) return files; - } - } - - function splitRepoName(fullName) { - const [owner, name] = fullName.split("/"); - if (!owner || !name) throw new Error(`Unexpected repository full name: ${fullName}`); - return { owner, name }; - } - - function buildBlobQuery(paths) { - const aliases = paths.map((_, index) => ` f${index}: object(expression: $e${index}) { __typename ... on Blob { text isBinary isTruncated byteSize } }`).join("\n"); - const params = paths.map((_, index) => `$e${index}: String!`).join(", "); - return `query($owner: String!, $name: String!, ${params}) {\n repository(owner: $owner, name: $name) {\n${aliases}\n }\n}`; - } - - async function fetchBlobs(repoFullName, oid, paths) { - const results = new Map(); - if (paths.length === 0) return results; - const { owner, name } = splitRepoName(repoFullName); - const rest = []; - for (let start = 0; start < paths.length; start += GRAPHQL_BATCH_SIZE) { - const chunk = paths.slice(start, start + GRAPHQL_BATCH_SIZE); - const query = buildBlobQuery(chunk); - const variables = { owner, name, oid }; - chunk.forEach((path, index) => { variables[`e${index}`] = `${oid}:${path}`; }); - const data = await graphql(query, variables); - const repository = data?.repository; - if (!repository) throw new Error(`GraphQL repository not found: ${repoFullName}`); - chunk.forEach((path, index) => { - const node = repository[`f${index}`]; - if (!node) { results.set(path, null); return; } - if (node.__typename !== "Blob") { results.set(path, null); return; } - if (node.isBinary) throw new Error(`${path} is binary; refusing to line-count`); - if (node.text === null || node.isTruncated) { rest.push(path); return; } - results.set(path, node.text); - }); - } - for (const path of rest) { - const text = await getContentViaRest(repoFullName, oid, path); - results.set(path, text); - } - return results; - } - - async function getContentViaRest(repo, ref, file) { - const encodedPath = file.split("/").map(encodeURIComponent).join("/"); - const url = `https://api.github.com/repos/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`; - return withRetry(url, async () => { - let response; - try { - response = await fetch(url, { headers }); - } catch (error) { - const wrapped = new Error(`${url}: network error ${error?.message ?? error}`); - wrapped.transient = true; - throw wrapped; - } - if (response.status === 404) return null; - if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); - const body = await response.json(); - if (body.type !== "file" || body.encoding !== "base64" || typeof body.content !== "string") { - throw new Error(`Could not decode file contents for ${file}`); - } - return Buffer.from(body.content.replace(/\s/g, ""), "base64").toString("utf8"); - }); - } - - async function main() { - const files = await getPullFiles(); - const budgetChanged = files.some(({ filename, previous_filename }) => filename === BUDGET_FILE || previous_filename === BUDGET_FILE); - const changedTests = files.filter(({ filename, status }) => status !== "removed" && TEST_FILE_RE.test(filename)); - - // Base budget lives in the trusted base repo; fetch it alone so we - // can parse legacy entries before deciding which HEAD blobs to load. - const [baseBlobs] = await Promise.all([ - fetchBlobs(REPO, BASE_SHA, [BUDGET_FILE]), - ]); - const baseText = baseBlobs.get(BUDGET_FILE); - const baseWasFallback = baseText == null; - const baseBudget = parseBudget(baseText ?? FALLBACK_BUDGET, "base budget"); - - // Assemble every HEAD path we still need in one deduplicated batch: - // budget (only if the PR changed it), each HEAD legacy file, each - // BASE legacy file that HEAD dropped, and each changed test file. - const headPaths = new Set(); - if (budgetChanged) headPaths.add(BUDGET_FILE); - // Placeholder; we resolve headBudget after fetching HEAD budget below. - let headBudget = baseBudget; - - if (budgetChanged) { - const headBudgetBlobs = await fetchBlobs(HEAD_REPO, HEAD_SHA, [BUDGET_FILE]); - const headText = headBudgetBlobs.get(BUDGET_FILE); - if (headText == null) throw new Error(`${BUDGET_FILE} must remain present and parseable at the PR head`); - headBudget = parseBudget(headText, "head budget"); - } - - for (const file of Object.keys(headBudget.legacyMaxLines)) headPaths.add(file); - for (const file of Object.keys(baseBudget.legacyMaxLines)) { - if (headBudget.legacyMaxLines[file] === undefined) headPaths.add(file); - } - for (const { filename } of changedTests) headPaths.add(filename); - - const headBlobs = await fetchBlobs(HEAD_REPO, HEAD_SHA, Array.from(headPaths)); - - if (!baseWasFallback) { - if (headBudget.defaultMaxLines > baseBudget.defaultMaxLines) { - violations.push(`defaultMaxLines increased from ${baseBudget.defaultMaxLines} to ${headBudget.defaultMaxLines}`); - } - for (const [file, baseMax] of Object.entries(baseBudget.legacyMaxLines)) { - const headMax = headBudget.legacyMaxLines[file]; - if (headMax !== undefined && headMax > baseMax) { - violations.push(`${file} legacy budget increased from ${baseMax} to ${headMax}`); - } - if (headMax === undefined) { - const text = headBlobs.get(file); - if (text != null && countLines(text) > headBudget.defaultMaxLines) { - violations.push(`${file} removed its legacy budget while still exceeding defaultMaxLines`); - } - } - } - for (const [file, headMax] of Object.entries(headBudget.legacyMaxLines)) { - if (baseBudget.legacyMaxLines[file] === undefined && headMax > headBudget.defaultMaxLines) { - violations.push(`${file} adds a new legacy budget (${headMax}) above defaultMaxLines (${headBudget.defaultMaxLines})`); - } - const text = headBlobs.get(file); - if (text == null) { - violations.push(`${file} has a legacy budget but no matching test file at the PR head`); - continue; - } - const lines = countLines(text); - if (lines > headMax) violations.push(`${file} has ${lines} line(s), above its legacy budget ${headMax}`); - if (lines < headMax) violations.push(`${file}: ${lines} line(s) < ${headMax} legacy budget; lower the budget entry`); - } - } - - for (const { filename } of changedTests) { - const text = headBlobs.get(filename); - if (text == null) throw new Error(`Changed test file ${filename} was not found at the PR head`); - const lines = countLines(text); - const maxLines = headBudget.legacyMaxLines[filename] ?? headBudget.defaultMaxLines; - if (lines > maxLines) violations.push(`${filename}: ${lines} line(s) > ${maxLines}`); - } - - if (violations.length > 0) { - console.error("FAIL: test size budget policy would be weakened or exceeded."); - for (const violation of violations) console.error(`- ${violation}`); - process.exit(1); - } - console.log(`PASS: test size budget policy is monotonic and ${changedTests.length} changed test file(s) are within budget.`); - } - - main().catch((error) => { - console.error(error); - process.exit(1); - }); - NODE + node --experimental-strip-types tools/growth-guardrails/test-size-budget.mts - name: Require changed test files not to add if statements + if: steps.tools.outputs.present == 'true' env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} @@ -391,371 +155,4 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | set -euo pipefail - - node <<'NODE' - const TEST_FILE_RE = /^(test|src|nemoclaw\/src)\/.*\.(test|spec)\.(?:[cm]?[jt]s)$/; - const GRAPHQL_URL = "https://api.github.com/graphql"; - const GRAPHQL_BATCH_SIZE = 25; - const RETRY_ATTEMPTS = 4; - const RETRY_BASE_MS = 250; - const RETRY_MAX_MS = 4000; - const { BASE_SHA, GH_TOKEN, HEAD_REPO, HEAD_SHA, PR_NUMBER, REPO } = process.env; - const headers = { Authorization: `Bearer ${GH_TOKEN}`, "X-GitHub-Api-Version": "2022-11-28" }; - - async function withRetry(label, fn) { - let lastError; - for (let attempt = 1; attempt <= RETRY_ATTEMPTS; attempt += 1) { - try { - return await fn(); - } catch (error) { - lastError = error; - const retriable = error && (error.transient === true || /HTTP (408|425|429|5\d{2})/.test(String(error.message ?? ""))); - if (!retriable || attempt === RETRY_ATTEMPTS) throw error; - const jitter = Math.random() * RETRY_BASE_MS; - const delay = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** (attempt - 1)) + jitter; - console.error(`retry: ${label} attempt ${attempt} failed (${error.message}); sleeping ${Math.round(delay)}ms`); - await new Promise((resolve) => setTimeout(resolve, delay)); - } - } - throw lastError; - } - - async function requestJson(url, init) { - let response; - try { - response = await fetch(url, init); - } catch (error) { - const wrapped = new Error(`${url}: network error ${error?.message ?? error}`); - wrapped.transient = true; - throw wrapped; - } - if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); - return response.json(); - } - - async function getJson(url) { - return withRetry(url, () => requestJson(url, { headers })); - } - - async function graphql(query, variables) { - const body = JSON.stringify({ query, variables }); - const label = `graphql ${variables.owner}/${variables.name}@${variables.oid}`; - return withRetry(label, async () => { - const payload = await requestJson(GRAPHQL_URL, { - method: "POST", - headers: { ...headers, "Content-Type": "application/json" }, - body, - }); - if (Array.isArray(payload.errors) && payload.errors.length > 0) { - const messages = payload.errors.map((entry) => entry.message).join("; "); - const transient = payload.errors.some((entry) => entry.type === "RATE_LIMITED" || /timed? *out|unavailable|internal/i.test(entry.message ?? "")); - const error = new Error(`${label}: GraphQL errors: ${messages}`); - if (transient) error.transient = true; - throw error; - } - return payload.data; - }); - } - - async function getPullFiles() { - const files = []; - for (let page = 1; ; page += 1) { - const batch = await getJson(`https://api.github.com/repos/${REPO}/pulls/${PR_NUMBER}/files?per_page=100&page=${page}`); - files.push(...batch); - if (batch.length < 100) return files; - } - } - - async function getContentViaRest(repo, ref, file) { - if (!file) return null; - const encodedPath = file.split("/").map(encodeURIComponent).join("/"); - const url = `https://api.github.com/repos/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`; - return withRetry(url, async () => { - let response; - try { - response = await fetch(url, { headers }); - } catch (error) { - const wrapped = new Error(`${url}: network error ${error?.message ?? error}`); - wrapped.transient = true; - throw wrapped; - } - if (response.status === 404) return null; - if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); - const body = await response.json(); - if (body.type !== "file" || body.encoding !== "base64" || typeof body.content !== "string") { - throw new Error(`Could not decode file contents for ${file}`); - } - return Buffer.from(body.content.replace(/\s/g, ""), "base64").toString("utf8"); - }); - } - - function splitRepoName(fullName) { - const [owner, name] = fullName.split("/"); - if (!owner || !name) throw new Error(`Unexpected repository full name: ${fullName}`); - return { owner, name }; - } - - function buildBlobQuery(paths) { - const aliases = paths.map((_, index) => ` f${index}: object(expression: $e${index}) { __typename ... on Blob { text isBinary isTruncated byteSize } }`).join("\n"); - const params = paths.map((_, index) => `$e${index}: String!`).join(", "); - return `query($owner: String!, $name: String!, ${params}) {\n repository(owner: $owner, name: $name) {\n${aliases}\n }\n}`; - } - - async function fetchBlobs(repoFullName, oid, paths) { - const results = new Map(); - if (paths.length === 0) return results; - const { owner, name } = splitRepoName(repoFullName); - const rest = []; - for (let start = 0; start < paths.length; start += GRAPHQL_BATCH_SIZE) { - const chunk = paths.slice(start, start + GRAPHQL_BATCH_SIZE); - const query = buildBlobQuery(chunk); - const variables = { owner, name, oid }; - chunk.forEach((path, index) => { variables[`e${index}`] = `${oid}:${path}`; }); - const data = await graphql(query, variables); - const repository = data?.repository; - if (!repository) throw new Error(`GraphQL repository not found: ${repoFullName}`); - chunk.forEach((path, index) => { - const node = repository[`f${index}`]; - if (!node) { results.set(path, null); return; } - if (node.__typename !== "Blob") { results.set(path, null); return; } - if (node.isBinary) throw new Error(`${path} is binary; refusing to count if statements`); - if (node.text === null || node.isTruncated) { rest.push(path); return; } - results.set(path, node.text); - }); - } - for (const path of rest) { - results.set(path, await getContentViaRest(repoFullName, oid, path)); - } - return results; - } - - function assertRepositoryName(repo, label) { - if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo ?? "")) { - throw new Error(`${label} must be an owner/name repository name`); - } - } - - function stripTriviaAndLiterals(text) { - let output = ""; - let index = 0; - let previousToken = ""; - let identifierBuffer = ""; - let lastIdentifier = ""; - - function finalizeIdentifier() { - if (identifierBuffer !== "") { - lastIdentifier = identifierBuffer; - identifierBuffer = ""; - } - } - - function appendCode(char) { - output += char; - if (/[A-Za-z0-9_$]/.test(char)) { - identifierBuffer += char; - } else { - finalizeIdentifier(); - } - if (/\S/.test(char)) previousToken = char; - } - - function consumeQuoted(quote) { - output += " "; - index += 1; - while (index < text.length) { - const char = text[index]; - if (char === "\\") { - index += 2; - } else if (char === quote) { - index += 1; - return; - } else { - index += 1; - } - } - } - - function consumeTemplate() { - output += " "; - index += 1; - while (index < text.length) { - const char = text[index]; - const next = text[index + 1]; - if (char === "\\") { - index += 2; - } else if (char === "`") { - index += 1; - return; - } else if (char === "$" && next === "{") { - output += " "; - index += 2; - consumeTemplateExpression(); - } else { - index += 1; - } - } - } - - function consumeTemplateExpression() { - let depth = 1; - while (index < text.length && depth > 0) { - const char = text[index]; - const next = text[index + 1]; - if (char === "/" && next === "/") { - output += " "; - index += 2; - while (index < text.length && !/[\r\n]/.test(text[index])) index += 1; - } else if (char === "/" && next === "*") { - output += " "; - index += 2; - while (index < text.length && !(text[index] === "*" && text[index + 1] === "/")) index += 1; - index = Math.min(index + 2, text.length); - } else if (char === '"' || char === "'") { - consumeQuoted(char); - } else if (char === "`") { - consumeTemplate(); - } else if (char === "/" && regexCanStart()) { - consumeRegex(); - } else if (char === "{") { - depth += 1; - appendCode(char); - index += 1; - } else if (char === "}") { - depth -= 1; - if (depth === 0) { - output += " "; - index += 1; - } else { - appendCode(char); - index += 1; - } - } else { - appendCode(char); - index += 1; - } - } - } - - function regexCanStart() { - const expressionKeyword = identifierBuffer || lastIdentifier; - return ( - previousToken === "" || - /[({[=,:;!&|?+\-*~^%<>]/.test(previousToken) || - /^(?:return|throw|case|delete|void|typeof|yield|await|else|do)$/.test(expressionKeyword) - ); - } - - function consumeRegex() { - output += " "; - index += 1; - let inCharacterClass = false; - while (index < text.length) { - const char = text[index]; - if (char === "\\") { - index += 2; - } else if (char === "[") { - inCharacterClass = true; - index += 1; - } else if (char === "]") { - inCharacterClass = false; - index += 1; - } else if (char === "/" && !inCharacterClass) { - index += 1; - while (/[a-z]/i.test(text[index] ?? "")) index += 1; - return; - } else { - index += 1; - } - } - } - - while (index < text.length) { - const char = text[index]; - const next = text[index + 1]; - if (char === "/" && next === "/") { - output += " "; - index += 2; - while (index < text.length && !/[\r\n]/.test(text[index])) index += 1; - } else if (char === "/" && next === "*") { - output += " "; - index += 2; - while (index < text.length && !(text[index] === "*" && text[index + 1] === "/")) index += 1; - index = Math.min(index + 2, text.length); - } else if (char === '"' || char === "'") { - consumeQuoted(char); - } else if (char === "`") { - consumeTemplate(); - } else if (char === "/" && regexCanStart()) { - consumeRegex(); - } else { - appendCode(char); - index += 1; - } - } - return output; - } - - function countIfStatements(text) { - return stripTriviaAndLiterals(text).match(/\bif\s*\(/g)?.length ?? 0; - } - - function countText(text) { - return text === null ? 0 : countIfStatements(text); - } - - async function main() { - const files = await getPullFiles(); - const changedTests = files.filter(({ filename, previous_filename }) => ( - TEST_FILE_RE.test(filename) || TEST_FILE_RE.test(previous_filename ?? "") - )); - assertRepositoryName(REPO, "REPO"); - assertRepositoryName(HEAD_REPO, "HEAD_REPO"); - - const basePaths = [...new Set(changedTests.map((file) => ( - TEST_FILE_RE.test(file.previous_filename ?? "") ? file.previous_filename : file.filename - )))]; - const headPaths = [...new Set(changedTests - .filter((file) => file.status !== "removed" && TEST_FILE_RE.test(file.filename)) - .map((file) => file.filename))]; - const [baseBlobs, headBlobs] = await Promise.all([ - fetchBlobs(REPO, BASE_SHA, basePaths), - fetchBlobs(HEAD_REPO, HEAD_SHA, headPaths), - ]); - - const details = []; - let baseTotal = 0; - let headTotal = 0; - - for (const file of changedTests) { - const basePath = TEST_FILE_RE.test(file.previous_filename ?? "") ? file.previous_filename : file.filename; - const headPath = file.status === "removed" || !TEST_FILE_RE.test(file.filename) ? null : file.filename; - const baseCount = countText(baseBlobs.get(basePath) ?? null); - const headCount = countText(headPath === null ? null : (headBlobs.get(headPath) ?? null)); - baseTotal += baseCount; - headTotal += headCount; - if (headCount > baseCount) { - details.push(`${headPath ?? file.filename}: ${headCount} if statement(s), up from ${baseCount}`); - } - } - - if (details.length > 0) { - console.error("FAIL: changed test files add if statements."); - console.error(`Changed test files contain ${headTotal} if statement(s) at PR head vs ${baseTotal} at base.`); - console.error(""); - console.error("Test bodies should stay linear. Split conditional behavior into separate test cases, use it.skipIf/it.runIf for platform or environment gates, or move non-asserting setup branches into named helpers."); - console.error(""); - console.error("Files with increased if counts:"); - for (const detail of details) console.error(`- ${detail}`); - console.error(""); - console.error("Run locally: npm run test-conditionals:scan -- --top 25"); - process.exit(1); - } - - console.log(`PASS: changed test files did not add if statements (${headTotal} at PR head vs ${baseTotal} at base).`); - } - - main().catch((error) => { - console.error(error); - process.exit(1); - }); - NODE + node --experimental-strip-types tools/growth-guardrails/test-conditionals.mts diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index e229bba1eef..f52c5ef93d9 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -516,6 +516,11 @@ "test": "registers the focused mappings at the root configuration boundary (#6692)", "category": "compatibility" }, + { + "file": "test/growth-guardrails-workflow-boundary.test.ts", + "test": "flags %s", + "category": "security" + }, { "file": "test/wechat-runtime-audit-workflow.test.ts", "test": "makes the trusted audit required in PR and main workflows", diff --git a/test/codebase-growth-guardrails-conditionals.test.ts b/test/codebase-growth-guardrails-conditionals.test.ts deleted file mode 100644 index 2b84c905f81..00000000000 --- a/test/codebase-growth-guardrails-conditionals.test.ts +++ /dev/null @@ -1,344 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { describe, expect, it } from "vitest"; - -import { scanTextForTestConditionals } from "../scripts/find-test-conditionals.mts"; - -const WORKFLOW_PATH = ".github/workflows/codebase-growth-guardrails.yaml"; -const STEP_NAME = "Require changed test files not to add if statements"; -const NODE_MARKER = "node <<'NODE'\n"; -const NODE_END_MARKER = "\n NODE"; -const ENV = { - BASE_SHA: "base-sha", - GH_TOKEN: "test-token", - HEAD_REPO: "fork/repo", - HEAD_SHA: "head-sha", - PR_NUMBER: "123", - REPO: "NVIDIA/NemoClaw", -}; - -type MockFile = { - readonly filename: string; - readonly previous_filename?: string; - readonly status?: string; -}; - -type MockContent = { - readonly repo?: string; - readonly ref: string; - readonly file: string; - readonly text: string; - readonly graphqlText?: string | null; - readonly graphqlTruncated?: boolean; -}; - -function extractConditionalsNodeScript(): string { - const workflow = fs.readFileSync(WORKFLOW_PATH, "utf8"); - const step = workflow.slice(workflow.indexOf(STEP_NAME)); - const nodeStart = step.indexOf(NODE_MARKER) + NODE_MARKER.length; - return step - .slice(nodeStart, step.indexOf(NODE_END_MARKER, nodeStart)) - .replaceAll("\n ", "\n"); -} - -function extractWorkflowCounterScript(): string { - const script = extractConditionalsNodeScript(); - const counterStart = script.indexOf("function stripTriviaAndLiterals(text)"); - const counterEnd = script.indexOf("function countText", counterStart); - return script.slice(counterStart, counterEnd); -} - -function pullFilesUrl(): string { - return `https://api.github.com/repos/${ENV.REPO}/pulls/${ENV.PR_NUMBER}/files?per_page=100&page=1`; -} - -function contentsUrl(content: MockContent): string { - const repo = content.repo ?? ENV.REPO; - const encodedPath = content.file.split("/").map(encodeURIComponent).join("/"); - return `https://api.github.com/repos/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(content.ref)}`; -} - -function encodeContent(text: string): { type: "file"; encoding: "base64"; content: string } { - return { type: "file", encoding: "base64", content: Buffer.from(text).toString("base64") }; -} - -function astIfCount(sourceText: string): number { - return scanTextForTestConditionals("test/virtual-workflow-parity.test.ts", sourceText).length; -} - -function workflowIfCount(sourceText: string): number { - const script = `${extractWorkflowCounterScript()}\nconsole.log(countIfStatements(process.argv[1]));\n`; - const result = spawnSync(process.execPath, ["-e", script, sourceText], { - cwd: process.cwd(), - encoding: "utf8", - }); - expect(result.status, result.stderr).toBe(0); - return Number(result.stdout.trim()); -} - -function runWorkflowConditionalsGuard(input: { - readonly files: readonly MockFile[]; - readonly contents: readonly MockContent[]; - readonly transientGraphqlFailures?: number; -}): ReturnType { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-growth-conditionals-")); - const scriptPath = path.join(tmpDir, "guardrail.cjs"); - const responses = new Map([ - [pullFilesUrl(), input.files], - ...input.contents.map( - (content) => [contentsUrl(content), encodeContent(content.text)] as const, - ), - ]); - const blobs = new Map( - input.contents.map( - (content) => - [ - `${content.repo ?? ENV.REPO}@${content.ref}:${content.file}`, - { - text: Object.prototype.hasOwnProperty.call(content, "graphqlText") - ? content.graphqlText - : content.text, - isTruncated: content.graphqlTruncated ?? false, - }, - ] as const, - ), - ); - const wrapper = [ - `const responses = new Map(${JSON.stringify([...responses])});`, - `const blobs = new Map(${JSON.stringify([...blobs])});`, - `let graphqlFailuresRemaining = ${input.transientGraphqlFailures ?? 0};`, - "let graphqlRequests = 0;", - "process.on('exit', () => console.log(`MOCK_GRAPHQL_REQUESTS=${graphqlRequests}`));", - "global.fetch = async (url, init = {}) => {", - " const isGraphql = String(url) === 'https://api.github.com/graphql';", - " graphqlRequests += Number(isGraphql);", - " const shouldFail = isGraphql && graphqlFailuresRemaining > 0;", - " graphqlFailuresRemaining -= Number(shouldFail);", - " const request = isGraphql && !shouldFail ? JSON.parse(String(init.body)) : {};", - " const variables = request.variables ?? {};", - " const repo = `${variables.owner}/${variables.name}`;", - " const aliases = Object.entries(variables)", - " .filter(([key]) => /^e\\d+$/.test(key))", - " .map(([key, expression]) => {", - " const index = Number(key.slice(1));", - " const blob = blobs.get(`${repo}@${expression}`);", - " return [`f${index}`, blob === undefined ? null : { __typename: 'Blob', text: blob.text, isBinary: false, isTruncated: blob.isTruncated, byteSize: Buffer.byteLength(blob.text ?? '') }];", - " });", - " const graphqlBody = { data: { repository: Object.fromEntries(aliases) } };", - " const body = responses.get(String(url));", - " const responseBody = isGraphql ? graphqlBody : body;", - " return {", - " ok: shouldFail ? false : responseBody !== undefined,", - " status: shouldFail ? 502 : responseBody === undefined ? 404 : 200,", - " json: async () => responseBody ?? {},", - " };", - "};", - extractConditionalsNodeScript(), - ].join("\n"); - - fs.writeFileSync(scriptPath, wrapper); - try { - return spawnSync(process.execPath, [scriptPath], { - cwd: process.cwd(), - encoding: "utf8", - env: { ...process.env, ...ENV }, - }); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } -} - -describe("codebase growth guardrail test conditionals step", () => { - it.each([ - { - name: "comments and strings", - source: [ - "// if (commented) return;", - "const text = 'if (string)';", - "expect(text).toContain('if');", - ].join("\n"), - }, - { - name: "regex literals", - source: "expect(/if \\(regex\\)/.test('if (regex)')).toBe(true);", - }, - { - name: "real branches and else-if chains", - source: "if (first) run(); else if (second) recover();", - }, - { - name: "nested template interpolation", - source: 'const value = `${`${(() => { if (flag) return "yes"; return "no"; })()}`}`;', - }, - { - name: "non-statement property tokens", - source: "const obj = { if: true }; expect(obj.if).toBe(true); type Shape = { if: boolean };", - }, - ])("matches local scanner count for $name", ({ source }) => { - expect(workflowIfCount(source)).toBe(astIfCount(source)); - }); - - it("fails a per-file increase even when another changed test removes an if", () => { - const result = runWorkflowConditionalsGuard({ - files: [{ filename: "test/add.test.ts" }, { filename: "test/remove.test.ts" }], - contents: [ - { file: "test/add.test.ts", ref: ENV.BASE_SHA, text: "expect(true).toBe(true);" }, - { - file: "test/add.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: "if (flag) expect(flag).toBe(true);", - }, - { - file: "test/remove.test.ts", - ref: ENV.BASE_SHA, - text: "if (flag) expect(flag).toBe(true);", - }, - { - file: "test/remove.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: "expect(true).toBe(true);", - }, - ], - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("test/add.test.ts"); - }); - - it("batches base and head blobs into one GraphQL request each", () => { - const result = runWorkflowConditionalsGuard({ - files: [{ filename: "test/first.test.ts" }, { filename: "test/second.test.ts" }], - contents: [ - { file: "test/first.test.ts", ref: ENV.BASE_SHA, text: "expect(true).toBe(true);" }, - { file: "test/second.test.ts", ref: ENV.BASE_SHA, text: "expect(true).toBe(true);" }, - { - file: "test/first.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: "expect(true).toBe(true);", - }, - { - file: "test/second.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: "expect(true).toBe(true);", - }, - ], - }); - - expect(result.status).toBe(0); - expect(result.stdout).toContain("MOCK_GRAPHQL_REQUESTS=2"); - }); - - it("retries a transient GraphQL failure", () => { - const result = runWorkflowConditionalsGuard({ - files: [{ filename: "test/retry.test.ts" }], - contents: [ - { file: "test/retry.test.ts", ref: ENV.BASE_SHA, text: "expect(true).toBe(true);" }, - { - file: "test/retry.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: "expect(true).toBe(true);", - }, - ], - transientGraphqlFailures: 1, - }); - - expect(result.status).toBe(0); - expect(result.stderr).toMatch(/retry: graphql \S+ attempt 1 failed/); - expect(result.stdout).toContain("MOCK_GRAPHQL_REQUESTS=3"); - }); - - it("uses REST contents when GraphQL returns a truncated blob", () => { - const result = runWorkflowConditionalsGuard({ - files: [{ filename: "test/truncated.test.ts" }], - contents: [ - { file: "test/truncated.test.ts", ref: ENV.BASE_SHA, text: "" }, - { - file: "test/truncated.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: "if (flag) expect(flag).toBe(true);", - graphqlText: "", - graphqlTruncated: true, - }, - ], - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("test/truncated.test.ts"); - }); - - it("does not count non-statement if property tokens", () => { - const result = runWorkflowConditionalsGuard({ - files: [{ filename: "test/non-statement.test.ts" }], - contents: [ - { file: "test/non-statement.test.ts", ref: ENV.BASE_SHA, text: "" }, - { - file: "test/non-statement.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: "const obj = { if: true }; expect(obj.if).toBe(true); type Shape = { if: boolean };", - }, - ], - }); - - expect(result.status).toBe(0); - expect(result.stdout).toContain("PASS"); - }); - - it("ignores removed files and files renamed out of test patterns", () => { - const result = runWorkflowConditionalsGuard({ - files: [ - { filename: "test/removed.test.ts", status: "removed" }, - { - filename: "src/helper.ts", - previous_filename: "test/renamed-out.test.ts", - status: "renamed", - }, - ], - contents: [ - { - file: "test/removed.test.ts", - ref: ENV.BASE_SHA, - text: "if (flag) expect(flag).toBe(true);", - }, - { - file: "test/renamed-out.test.ts", - ref: ENV.BASE_SHA, - text: "if (flag) expect(flag).toBe(true);", - }, - ], - }); - - expect(result.status).toBe(0); - expect(result.stdout).toContain("PASS"); - }); - - it("counts executable if statements inside template interpolation", () => { - const result = runWorkflowConditionalsGuard({ - files: [{ filename: "test/template.test.ts" }], - contents: [ - { file: "test/template.test.ts", ref: ENV.BASE_SHA, text: "" }, - { - file: "test/template.test.ts", - repo: ENV.HEAD_REPO, - ref: ENV.HEAD_SHA, - text: 'const value = `${(() => { if (flag) return "enabled"; return "disabled"; })()}`;', - }, - ], - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("test/template.test.ts"); - }); -}); diff --git a/test/growth-guardrails-entrypoints.test.ts b/test/growth-guardrails-entrypoints.test.ts new file mode 100644 index 00000000000..9428d4e1790 --- /dev/null +++ b/test/growth-guardrails-entrypoints.test.ts @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const REPO_ROOT = resolve(import.meta.dirname, ".."); +const ENTRYPOINT_ENV = { + BASE_SHA: "base", + GH_TOKEN: "token", + HEAD_REPO: "fork/repo", + HEAD_SHA: "head", + PR_NUMBER: "1", + REPO: "NVIDIA/NemoClaw", +} as const; + +const FETCH_PRELOAD = ` +const responses = JSON.parse(process.env.MOCK_RESPONSES ?? "[]"); +globalThis.fetch = async () => new Response(JSON.stringify(responses.shift()), { + status: 200, + headers: { "content-type": "application/json" }, +}); +`; + +function blobPayload(text: string | null): unknown { + return { + data: { + repository: { + f0: + text === null ? null : { __typename: "Blob", text, isBinary: false, isTruncated: false }, + }, + }, + }; +} + +function runEntrypoint( + relativeToolPath: string, + responses: readonly unknown[], +): ReturnType { + const directory = mkdtempSync(join(tmpdir(), "growth-guardrail-entrypoint-")); + const preload = join(directory, "fetch-preload.mjs"); + writeFileSync(preload, FETCH_PRELOAD); + const result = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + "--import", + pathToFileURL(preload).href, + resolve(REPO_ROOT, relativeToolPath), + ], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + ...process.env, + ...ENTRYPOINT_ENV, + MOCK_RESPONSES: JSON.stringify(responses), + }, + }, + ); + rmSync(directory, { recursive: true, force: true }); + return result; +} + +describe("growth-guardrails executable entrypoints (#6953)", () => { + it("prints the conditional guardrail PASS diagnostic", () => { + const result = runEntrypoint("tools/growth-guardrails/test-conditionals.mts", [[]]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain( + "PASS: changed test files did not add if statements (0 at PR head vs 0 at base).", + ); + }); + + it("prints the conditional guardrail FAIL heading and file detail", () => { + const result = runEntrypoint("tools/growth-guardrails/test-conditionals.mts", [ + [{ filename: "test/a.test.ts", status: "modified" }], + blobPayload("it('a', () => { expect(1).toBe(1); });"), + blobPayload("it('a', () => { if (condition) expect(1).toBe(1); });"), + ]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("FAIL: changed test files add if statements."); + expect(result.stderr).toContain("test/a.test.ts: 1 if statement(s), up from 0"); + }); + + it("prints the size-budget guardrail PASS diagnostic", () => { + const result = runEntrypoint("tools/growth-guardrails/test-size-budget.mts", [ + [], + blobPayload(null), + ]); + + expect(result.status).toBe(0); + expect(result.stdout).toContain( + "PASS: test size budget policy is monotonic and 0 changed test file(s) are within budget.", + ); + }); + + it("prints the size-budget guardrail FAIL heading and budget detail", () => { + const result = runEntrypoint("tools/growth-guardrails/test-size-budget.mts", [ + [{ filename: "ci/test-file-size-budget.json", status: "added" }], + blobPayload(null), + blobPayload('{"defaultMaxLines":2000,"legacyMaxLines":{}}'), + ]); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("FAIL: test size budget policy would be weakened or exceeded."); + expect(result.stderr).toContain("defaultMaxLines increased from 1500 to 2000"); + }); +}); diff --git a/test/growth-guardrails-pr-blob-client.test.ts b/test/growth-guardrails-pr-blob-client.test.ts new file mode 100644 index 00000000000..de6f4086b5f --- /dev/null +++ b/test/growth-guardrails-pr-blob-client.test.ts @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + assertRepositoryName, + createPrBlobClient, + type FetchLike, + GRAPHQL_BATCH_SIZE, + isTransientStatus, +} from "../tools/growth-guardrails/pr-blob-client.mts"; + +const DETERMINISTIC = { sleep: async () => {}, random: () => 0 } as const; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function blobData(entries: Record): unknown { + return { data: { repository: entries } }; +} + +/** A fetch stub that returns a scripted response per call and records calls. */ +function scriptedFetch(responses: Array<() => Promise>): { + fetchImpl: FetchLike; + urls: string[]; +} { + const urls: string[] = []; + let call = 0; + const fetchImpl: FetchLike = async (url) => { + urls.push(String(url)); + const responder = responses[Math.min(call, responses.length - 1)]; + call += 1; + return responder(); + }; + return { fetchImpl, urls }; +} + +describe("growth-guardrails pr-blob-client", () => { + it("classifies transient HTTP statuses", () => { + expect([408, 425, 429, 500, 503, 599].map(isTransientStatus)).toEqual([ + true, + true, + true, + true, + true, + true, + ]); + expect([200, 400, 401, 404].map(isTransientStatus)).toEqual([false, false, false, false]); + }); + + it("rejects non owner/name repository names", () => { + expect(() => assertRepositoryName("not-a-repo", "REPO")).toThrow(/owner\/name/); + expect(() => assertRepositoryName(undefined, "HEAD_REPO")).toThrow(/owner\/name/); + expect(assertRepositoryName("NVIDIA/NemoClaw", "REPO")).toBeUndefined(); + }); + + it("paginates getPullFiles until a short page", async () => { + const fullPage = Array.from({ length: 100 }, (_, i) => ({ filename: `f${i}.ts` })); + const { fetchImpl, urls } = scriptedFetch([ + async () => jsonResponse(fullPage), + async () => jsonResponse([{ filename: "last.ts" }]), + ]); + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + const files = await client.getPullFiles("NVIDIA/NemoClaw", "1"); + expect(files).toHaveLength(101); + expect(urls).toHaveLength(2); + expect(urls[1]).toContain("page=2"); + }); + + it("batches blob fetches into GRAPHQL_BATCH_SIZE-sized GraphQL queries", async () => { + const paths = Array.from({ length: GRAPHQL_BATCH_SIZE + 5 }, (_, i) => `test/f${i}.test.ts`); + const graphqlCalls: number[] = []; + const fetchImpl: FetchLike = async (_url, init) => { + const query = JSON.parse(String((init as { body?: string })?.body ?? "{}")).query as string; + const aliasCount = (query.match(/f\d+: object/g) ?? []).length; + graphqlCalls.push(aliasCount); + const entries: Record = {}; + for (let i = 0; i < aliasCount; i += 1) { + entries[`f${i}`] = { + __typename: "Blob", + text: "line\n", + isBinary: false, + isTruncated: false, + }; + } + return jsonResponse(blobData(entries)); + }; + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + const blobs = await client.fetchBlobs("NVIDIA/NemoClaw", "deadbeef", paths); + expect(graphqlCalls).toEqual([GRAPHQL_BATCH_SIZE, 5]); + expect(blobs.size).toBe(GRAPHQL_BATCH_SIZE + 5); + }); + + it("refuses to read binary blobs", async () => { + const fetchImpl: FetchLike = async () => + jsonResponse( + blobData({ f0: { __typename: "Blob", text: null, isBinary: true, isTruncated: false } }), + ); + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + await expect(client.fetchBlobs("NVIDIA/NemoClaw", "oid", ["bin.test.ts"])).rejects.toThrow( + /binary; refusing to read/, + ); + }); + + it("falls back to the REST raw media type for truncated blobs", async () => { + const contentsAccept: string[] = []; + let call = 0; + const fetchImpl: FetchLike = async (url, init) => { + call += 1; + const accept = (init?.headers as Record | undefined)?.Accept ?? ""; + contentsAccept.push(String(url).includes("/contents/") ? accept : ""); + return call === 1 + ? jsonResponse( + blobData({ + f0: { __typename: "Blob", text: null, isBinary: false, isTruncated: true }, + }), + ) + : new Response("a\nb\n", { status: 200 }); + }; + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + const blobs = await client.fetchBlobs("NVIDIA/NemoClaw", "oid", ["big.test.ts"]); + expect(blobs.get("big.test.ts")).toBe("a\nb\n"); + expect(contentsAccept).toContain("application/vnd.github.raw"); + }); + + it("retries a transient 500 then succeeds", async () => { + const { fetchImpl, urls } = scriptedFetch([ + async () => jsonResponse({ message: "boom" }, 500), + async () => jsonResponse([{ filename: "ok.ts" }]), + ]); + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + const files = await client.getPullFiles("NVIDIA/NemoClaw", "9"); + expect(files).toEqual([{ filename: "ok.ts" }]); + expect(urls).toHaveLength(2); + }); + + it("retries a transient GraphQL error payload then succeeds", async () => { + const { fetchImpl, urls } = scriptedFetch([ + async () => + jsonResponse({ + errors: [{ message: "API rate limit exceeded", type: "RATE_LIMITED" }], + }), + async () => + jsonResponse( + blobData({ + f0: { + __typename: "Blob", + text: "ok\n", + isBinary: false, + isTruncated: false, + }, + }), + ), + ]); + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + + const blobs = await client.fetchBlobs("NVIDIA/NemoClaw", "oid", ["test/a.test.ts"]); + + expect(blobs.get("test/a.test.ts")).toBe("ok\n"); + expect(urls).toHaveLength(2); + }); + + it("does not retry a non-transient GraphQL error payload", async () => { + const { fetchImpl, urls } = scriptedFetch([ + async () => + jsonResponse({ + errors: [{ message: "Resource not accessible", type: "FORBIDDEN" }], + }), + ]); + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + + await expect(client.fetchBlobs("NVIDIA/NemoClaw", "oid", ["test/a.test.ts"])).rejects.toThrow( + /GraphQL errors: Resource not accessible/, + ); + expect(urls).toHaveLength(1); + }); + + it("gives up after exhausting retries on persistent 503", async () => { + const { fetchImpl, urls } = scriptedFetch([async () => jsonResponse({ message: "down" }, 503)]); + const client = createPrBlobClient({ token: "t", fetchImpl, ...DETERMINISTIC }); + await expect(client.getPullFiles("NVIDIA/NemoClaw", "9")).rejects.toThrow(/HTTP 503/); + expect(urls).toHaveLength(4); + }); +}); diff --git a/test/growth-guardrails-test-conditionals.test.ts b/test/growth-guardrails-test-conditionals.test.ts new file mode 100644 index 00000000000..0fed4f52587 --- /dev/null +++ b/test/growth-guardrails-test-conditionals.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { scanTextForTestConditionals } from "../scripts/find-test-conditionals.mts"; +import type { PrBlobClient, PullRequestFile } from "../tools/growth-guardrails/pr-blob-client.mts"; +import { + type ConditionalChange, + countIfStatements, + evaluateConditionalViolations, + runTestConditionals, +} from "../tools/growth-guardrails/test-conditionals.mts"; + +// Fixtures are string literals (test data), so they do not add real `if` +// statements to this file under the very policy it exercises. +const NO_IF = "it('a', () => { expect(1).toBe(1); });"; +const ONE_IF = "it('a', () => { if (cond) { expect(1).toBe(1); } });"; +const TWO_IF = "it('a', () => { if (a) { expect(1).toBe(1); } if (b) { expect(2).toBe(2); } });"; + +function blobs(entries: Record): Map { + return new Map(Object.entries(entries)); +} + +type BlobFetchCall = { + readonly repo: string; + readonly oid: string; + readonly paths: readonly string[]; +}; + +function fakeClient( + pullFiles: PullRequestFile[], + table: Record, + fetchCalls: BlobFetchCall[] = [], +): PrBlobClient { + return { + getPullFiles: async () => pullFiles, + fetchBlobs: async (repo, oid, paths) => { + fetchCalls.push({ repo, oid, paths: [...paths] }); + const map = new Map(); + for (const path of paths) map.set(path, table[`${repo} ${oid} ${path}`] ?? null); + return map; + }, + }; +} + +describe("growth-guardrails test-conditionals: AST parity", () => { + it.each([ + ["no if", NO_IF, 0], + ["one if", ONE_IF, 1], + ["two ifs", TWO_IF, 2], + ])("countIfStatements matches the shared AST scanner: %s", (_label, source, expected) => { + expect(countIfStatements("test/x.test.ts", source)).toBe(expected); + expect(countIfStatements("test/x.test.ts", source)).toBe( + scanTextForTestConditionals("test/x.test.ts", source).length, + ); + }); +}); + +describe("growth-guardrails test-conditionals: pure policy", () => { + const same = (path: string): ConditionalChange => ({ + basePath: path, + headPath: path, + displayName: path, + }); + + it("flags a test file that adds an if statement", () => { + const result = evaluateConditionalViolations( + [same("test/a.test.ts")], + blobs({ "test/a.test.ts": NO_IF }), + blobs({ "test/a.test.ts": ONE_IF }), + ); + expect(result.details).toEqual(["test/a.test.ts: 1 if statement(s), up from 0"]); + expect([result.baseTotal, result.headTotal]).toEqual([0, 1]); + }); + + it("passes a test file that removes if statements", () => { + const result = evaluateConditionalViolations( + [same("test/a.test.ts")], + blobs({ "test/a.test.ts": TWO_IF }), + blobs({ "test/a.test.ts": ONE_IF }), + ); + expect(result.details).toEqual([]); + expect([result.baseTotal, result.headTotal]).toEqual([2, 1]); + }); + + it("compares across a rename using the previous path at base", () => { + const result = evaluateConditionalViolations( + [ + { + basePath: "test/old.test.ts", + headPath: "test/new.test.ts", + displayName: "test/new.test.ts", + }, + ], + blobs({ "test/old.test.ts": ONE_IF }), + blobs({ "test/new.test.ts": ONE_IF }), + ); + expect(result.details).toEqual([]); + }); + + it("flags a per-file increase even when another changed test removes an if", () => { + const result = evaluateConditionalViolations( + [same("test/adder.test.ts"), same("test/remover.test.ts")], + blobs({ "test/adder.test.ts": NO_IF, "test/remover.test.ts": TWO_IF }), + blobs({ "test/adder.test.ts": ONE_IF, "test/remover.test.ts": NO_IF }), + ); + expect(result.details).toEqual(["test/adder.test.ts: 1 if statement(s), up from 0"]); + expect([result.baseTotal, result.headTotal]).toEqual([2, 1]); + }); + + it("counts a removed test file as zero at head", () => { + const result = evaluateConditionalViolations( + [{ basePath: "test/gone.test.ts", headPath: null, displayName: "test/gone.test.ts" }], + blobs({ "test/gone.test.ts": TWO_IF }), + blobs({}), + ); + expect(result.details).toEqual([]); + expect([result.baseTotal, result.headTotal]).toEqual([2, 0]); + }); +}); + +describe("growth-guardrails test-conditionals: orchestration", () => { + const ENV = { + BASE_SHA: "base", + HEAD_REPO: "fork/repo", + HEAD_SHA: "head", + PR_NUMBER: "1", + REPO: "NVIDIA/NemoClaw", + } as const; + + it("fails a PR whose changed test adds an if statement", async () => { + const client = fakeClient([{ filename: "test/a.test.ts", status: "modified" }], { + "NVIDIA/NemoClaw base test/a.test.ts": NO_IF, + "fork/repo head test/a.test.ts": ONE_IF, + }); + const result = await runTestConditionals(client, ENV); + expect(result.ok).toBe(false); + expect(result.details).toEqual(["test/a.test.ts: 1 if statement(s), up from 0"]); + }); + + it("ignores non-test changed files", async () => { + const client = fakeClient([{ filename: "src/lib/foo.ts", status: "modified" }], {}); + const result = await runTestConditionals(client, ENV); + expect(result.ok).toBe(true); + expect([result.baseTotal, result.headTotal]).toEqual([0, 0]); + }); + + it("batches deduplicated ordinary test paths once per revision", async () => { + const fetchCalls: BlobFetchCall[] = []; + const client = fakeClient( + [ + { filename: "test/a.test.ts", status: "modified" }, + { filename: "test/b.test.ts", status: "modified" }, + { filename: "test/a.test.ts", status: "modified" }, + ], + { + "NVIDIA/NemoClaw base test/a.test.ts": NO_IF, + "NVIDIA/NemoClaw base test/b.test.ts": NO_IF, + "fork/repo head test/a.test.ts": NO_IF, + "fork/repo head test/b.test.ts": NO_IF, + }, + fetchCalls, + ); + + const result = await runTestConditionals(client, ENV); + + expect(result.ok).toBe(true); + expect(fetchCalls).toEqual([ + { + repo: "NVIDIA/NemoClaw", + oid: "base", + paths: ["test/a.test.ts", "test/b.test.ts"], + }, + { + repo: "fork/repo", + oid: "head", + paths: ["test/a.test.ts", "test/b.test.ts"], + }, + ]); + }); +}); diff --git a/test/growth-guardrails-test-size-budget.test.ts b/test/growth-guardrails-test-size-budget.test.ts new file mode 100644 index 00000000000..8a75100c469 --- /dev/null +++ b/test/growth-guardrails-test-size-budget.test.ts @@ -0,0 +1,268 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import type { PrBlobClient, PullRequestFile } from "../tools/growth-guardrails/pr-blob-client.mts"; +import { + evaluateTestSizeBudgetViolations, + runTestSizeBudget, +} from "../tools/growth-guardrails/test-size-budget.mts"; + +function blobs(entries: Record): Map { + return new Map(Object.entries(entries)); +} + +function linesOf(count: number): string { + return "x\n".repeat(count); +} + +type BlobFetchCall = { + readonly repo: string; + readonly oid: string; + readonly paths: readonly string[]; +}; + +/** Fake client: resolves blobs from a (repo\0oid\0path) table, no network. */ +function fakeClient( + pullFiles: PullRequestFile[], + table: Record, + fetchCalls: BlobFetchCall[] = [], +): PrBlobClient { + return { + getPullFiles: async () => pullFiles, + fetchBlobs: async (repo, oid, paths) => { + fetchCalls.push({ repo, oid, paths: [...paths] }); + const map = new Map(); + for (const path of paths) map.set(path, table[`${repo} ${oid} ${path}`] ?? null); + return map; + }, + }; +} + +const CLEAN_BUDGET = { defaultMaxLines: 1500, legacyMaxLines: {} } as const; +const NO_RENAMES: ReadonlyMap = new Map(); + +describe("growth-guardrails test-size-budget: pure policy", () => { + it("passes when nothing changed and no test files are touched", () => { + expect( + evaluateTestSizeBudgetViolations({ + baseBudget: CLEAN_BUDGET, + headBudget: CLEAN_BUDGET, + renames: NO_RENAMES, + headBlobs: blobs({}), + changedTests: [], + }), + ).toEqual([]); + }); + + it.each([ + [ + "defaultMaxLines increase", + { defaultMaxLines: 1600, legacyMaxLines: {} }, + /defaultMaxLines increased from 1500 to 1600/, + ], + [ + "legacy budget increase", + { defaultMaxLines: 1500, legacyMaxLines: { "test/a.test.ts": 2000 } }, + /test\/a\.test\.ts legacy budget increased from 1800 to 2000/, + ], + ])("flags a weakened budget: %s", (_label, headBudget, pattern) => { + const violations = evaluateTestSizeBudgetViolations({ + baseBudget: { defaultMaxLines: 1500, legacyMaxLines: { "test/a.test.ts": 1800 } }, + headBudget, + renames: NO_RENAMES, + headBlobs: blobs({ "test/a.test.ts": linesOf(1700) }), + changedTests: [], + }); + expect(violations.join("\n")).toMatch(pattern); + }); + + it("flags a new legacy budget above the default", () => { + const violations = evaluateTestSizeBudgetViolations({ + baseBudget: CLEAN_BUDGET, + headBudget: { defaultMaxLines: 1500, legacyMaxLines: { "test/new.test.ts": 1900 } }, + renames: NO_RENAMES, + headBlobs: blobs({ "test/new.test.ts": linesOf(1850) }), + changedTests: [], + }); + expect(violations.join("\n")).toMatch( + /adds a new legacy budget \(1900\) above defaultMaxLines \(1500\)/, + ); + }); + + it("flags a legacy entry whose file shrank below its budget", () => { + const violations = evaluateTestSizeBudgetViolations({ + baseBudget: { defaultMaxLines: 1500, legacyMaxLines: { "test/a.test.ts": 1800 } }, + headBudget: { defaultMaxLines: 1500, legacyMaxLines: { "test/a.test.ts": 1800 } }, + renames: NO_RENAMES, + headBlobs: blobs({ "test/a.test.ts": linesOf(1600) }), + changedTests: [], + }); + expect(violations.join("\n")).toMatch( + /1600 line\(s\) < 1800 legacy budget; lower the budget entry/, + ); + }); + + it("flags removing a legacy budget while the file still exceeds the default", () => { + const violations = evaluateTestSizeBudgetViolations({ + baseBudget: { defaultMaxLines: 1500, legacyMaxLines: { "test/a.test.ts": 1800 } }, + headBudget: CLEAN_BUDGET, + renames: NO_RENAMES, + headBlobs: blobs({ "test/a.test.ts": linesOf(1700) }), + changedTests: [], + }); + expect(violations.join("\n")).toMatch( + /removed its legacy budget while still exceeding defaultMaxLines/, + ); + }); + + it("flags a changed test file over the default budget", () => { + const violations = evaluateTestSizeBudgetViolations({ + baseBudget: CLEAN_BUDGET, + headBudget: CLEAN_BUDGET, + renames: NO_RENAMES, + headBlobs: blobs({ "test/big.test.ts": linesOf(1501) }), + changedTests: ["test/big.test.ts"], + }); + expect(violations).toEqual(["test/big.test.ts: 1501 line(s) > 1500"]); + }); + + it("enforces default monotonicity against the fallback baseline", () => { + const violations = evaluateTestSizeBudgetViolations({ + baseBudget: CLEAN_BUDGET, + headBudget: { defaultMaxLines: 99999, legacyMaxLines: {} }, + renames: NO_RENAMES, + headBlobs: blobs({ "test/ok.test.ts": linesOf(10) }), + changedTests: ["test/ok.test.ts"], + }); + expect(violations.join("\n")).toMatch(/defaultMaxLines increased from 1500 to 99999/); + }); + + it("carries a legacy allowance across a rename without flagging it as new", () => { + const renames = new Map([["test/new.test.ts", "test/old.test.ts"]]); + const violations = evaluateTestSizeBudgetViolations({ + baseBudget: { defaultMaxLines: 1500, legacyMaxLines: { "test/old.test.ts": 1800 } }, + headBudget: { defaultMaxLines: 1500, legacyMaxLines: { "test/new.test.ts": 1800 } }, + renames, + headBlobs: blobs({ "test/new.test.ts": linesOf(1800) }), + changedTests: [], + }); + expect(violations).toEqual([]); + }); + + it("throws when a changed test file is missing at the PR head", () => { + expect(() => + evaluateTestSizeBudgetViolations({ + baseBudget: CLEAN_BUDGET, + headBudget: CLEAN_BUDGET, + renames: NO_RENAMES, + headBlobs: blobs({}), + changedTests: ["test/missing.test.ts"], + }), + ).toThrow(/Changed test file test\/missing\.test\.ts was not found/); + }); +}); + +describe("growth-guardrails test-size-budget: orchestration", () => { + const ENV = { + BASE_SHA: "base", + HEAD_REPO: "fork/repo", + HEAD_SHA: "head", + PR_NUMBER: "1", + REPO: "NVIDIA/NemoClaw", + } as const; + + it("fetches the changed budget at head and reports a weakened default", async () => { + const client = fakeClient( + [ + { filename: "ci/test-file-size-budget.json", status: "modified" }, + { filename: "test/foo.test.ts", status: "modified" }, + ], + { + "NVIDIA/NemoClaw base ci/test-file-size-budget.json": + '{"defaultMaxLines":1500,"legacyMaxLines":{}}', + "fork/repo head ci/test-file-size-budget.json": + '{"defaultMaxLines":1600,"legacyMaxLines":{}}', + "fork/repo head test/foo.test.ts": linesOf(10), + }, + ); + const result = await runTestSizeBudget(client, ENV); + expect(result.ok).toBe(false); + expect(result.changedTestCount).toBe(1); + expect(result.violations.join("\n")).toMatch(/defaultMaxLines increased from 1500 to 1600/); + }); + + it("passes a clean PR with an unchanged budget", async () => { + const client = fakeClient([{ filename: "test/foo.test.ts", status: "modified" }], { + "NVIDIA/NemoClaw base ci/test-file-size-budget.json": + '{"defaultMaxLines":1500,"legacyMaxLines":{}}', + "fork/repo head test/foo.test.ts": linesOf(20), + }); + const result = await runTestSizeBudget(client, ENV); + expect(result.ok).toBe(true); + expect(result.violations).toEqual([]); + }); + + it("fetches the base budget and ordinary head tests in two exact batches", async () => { + const fetchCalls: BlobFetchCall[] = []; + const client = fakeClient( + [ + { filename: "test/a.test.ts", status: "modified" }, + { filename: "test/b.test.ts", status: "modified" }, + ], + { + "NVIDIA/NemoClaw base ci/test-file-size-budget.json": + '{"defaultMaxLines":1500,"legacyMaxLines":{}}', + "fork/repo head test/a.test.ts": linesOf(20), + "fork/repo head test/b.test.ts": linesOf(30), + }, + fetchCalls, + ); + + const result = await runTestSizeBudget(client, ENV); + + expect(result.ok).toBe(true); + expect(fetchCalls).toEqual([ + { + repo: "NVIDIA/NemoClaw", + oid: "base", + paths: ["ci/test-file-size-budget.json"], + }, + { + repo: "fork/repo", + oid: "head", + paths: ["test/a.test.ts", "test/b.test.ts"], + }, + ]); + }); + + it("enforces the fallback baseline when the base budget file is absent", async () => { + const client = fakeClient([{ filename: "ci/test-file-size-budget.json", status: "added" }], { + "fork/repo head ci/test-file-size-budget.json": + '{"defaultMaxLines":2000,"legacyMaxLines":{}}', + }); + const result = await runTestSizeBudget(client, ENV); + expect(result.ok).toBe(false); + expect(result.violations.join("\n")).toMatch(/defaultMaxLines increased from 1500 to 2000/); + }); + + it("passes an unchanged legacy test renamed with its budget key", async () => { + const client = fakeClient( + [ + { filename: "ci/test-file-size-budget.json", status: "modified" }, + { filename: "test/new.test.ts", previous_filename: "test/old.test.ts", status: "renamed" }, + ], + { + "NVIDIA/NemoClaw base ci/test-file-size-budget.json": + '{"defaultMaxLines":1500,"legacyMaxLines":{"test/old.test.ts":1800}}', + "fork/repo head ci/test-file-size-budget.json": + '{"defaultMaxLines":1500,"legacyMaxLines":{"test/new.test.ts":1800}}', + "fork/repo head test/new.test.ts": linesOf(1800), + }, + ); + const result = await runTestSizeBudget(client, ENV); + expect(result.ok).toBe(true); + expect(result.violations).toEqual([]); + }); +}); diff --git a/test/growth-guardrails-workflow-boundary.test.ts b/test/growth-guardrails-workflow-boundary.test.ts new file mode 100644 index 00000000000..8553e9248c0 --- /dev/null +++ b/test/growth-guardrails-workflow-boundary.test.ts @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { validateGrowthGuardrailsWorkflowBoundary } from "../tools/growth-guardrails/workflow-boundary.mts"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const WORKFLOW_PATH = path.join(ROOT, ".github/workflows/codebase-growth-guardrails.yaml"); + +function workflowSource(): string { + return fs.readFileSync(WORKFLOW_PATH, "utf8"); +} + +function validateMutation(mutate: (source: string) => string): string[] { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "growth-guardrails-boundary-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + fs.writeFileSync(workflowPath, mutate(workflowSource())); + try { + return validateGrowthGuardrailsWorkflowBoundary(workflowPath); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +describe("growth-guardrails workflow trust boundary", () => { + it("passes on the real workflow", () => { + expect(validateGrowthGuardrailsWorkflowBoundary(WORKFLOW_PATH)).toEqual([]); + }); + + // source-shape-contract: security -- The pull_request_target guardrail must bind the exact trusted execution shape and reject unsafe trigger, permission, checkout, install, action, and shell mutations + it.each([ + [ + "an untrusted pull_request trigger", + (s: string) => s.replace(" pull_request_target:", " pull_request:"), + /must not trigger on pull_request/, + ], + [ + "a write permission scope", + (s: string) => s.replace(" contents: read", " contents: write"), + /permission contents: write must be read or none/, + ], + [ + "a checkout of the PR head", + (s: string) => + s.replace( + "ref: ${{ github.event.pull_request.base.sha }}", + "ref: ${{ github.event.pull_request.head.sha }}", + ), + /actions\/checkout ref must be/, + ], + [ + "a dependency install that runs PR scripts", + (s: string) => + s.replace("npm ci --ignore-scripts --no-audit --no-fund", "npm ci --no-audit --no-fund"), + /must use --ignore-scripts/, + ], + [ + "a dropped trusted tool invocation", + (s: string) => + s.replace( + "node --experimental-strip-types tools/growth-guardrails/test-conditionals.mts", + "echo skip", + ), + /must invoke the trusted tool: .*test-conditionals\.mts/, + ], + [ + "a resurrected inline node heredoc", + (s: string) => + s.replace( + "node --experimental-strip-types tools/growth-guardrails/test-size-budget.mts", + "node <<'NODE'\n console.log(1)\n NODE", + ), + /must match the approved shape/, + ], + [ + "a job-level write permission override", + (s: string) => + s.replace( + " runs-on: ubuntu-latest\n", + " runs-on: ubuntu-latest\n permissions:\n contents: write\n", + ), + /job codebase-growth-guardrails permission contents: write must be read or none/, + ], + [ + "an appended PR-head payload execution in a trusted step", + (s: string) => + s.replace( + "node --experimental-strip-types tools/growth-guardrails/test-size-budget.mts", + 'node --experimental-strip-types tools/growth-guardrails/test-size-budget.mts\n gh api "/repos/${HEAD_REPO}/contents/payload.sh?ref=${HEAD_SHA}" --jq .content | base64 -d > "$RUNNER_TEMP/payload.sh"\n bash "$RUNNER_TEMP/payload.sh"', + ), + /must match the approved shape/, + ], + [ + "an arbitrary action step", + (s: string) => + s.replace( + " - name: Check out the trusted base revision", + " - name: Execute an untrusted action\n uses: attacker/payload@main\n\n - name: Check out the trusted base revision", + ), + /must contain exactly 7 approved steps, not 8/, + ], + [ + "a non-approved shell field", + (s: string) => + s.replace( + " run: npm ci --ignore-scripts --no-audit --no-fund", + " shell: python\n run: npm ci --ignore-scripts --no-audit --no-fund", + ), + /must match the approved shape/, + ], + [ + "an extra reusable-workflow job", + (s: string) => + `${s}\n untrusted:\n uses: attacker/payload/.github/workflows/run.yaml@main\n`, + /workflow jobs must be exactly codebase-growth-guardrails/, + ], + ])("flags %s", (_label, mutate, pattern) => { + expect(validateMutation(mutate).join("\n")).toMatch(pattern); + }); +}); diff --git a/test/pr-workflow-contract.test.ts b/test/pr-workflow-contract.test.ts index f17645f4d32..247ef15e4ae 100644 --- a/test/pr-workflow-contract.test.ts +++ b/test/pr-workflow-contract.test.ts @@ -1168,14 +1168,14 @@ describe("pull request and main workflow contracts", () => { const growthGuardrails = readYaml( ".github/workflows/codebase-growth-guardrails.yaml", ); - const guardRun = stepRuns(growthGuardrails.jobs["codebase-growth-guardrails"]).join("\n"); - - expect(guardRun).toContain("HEAD_REPO"); - expect(guardRun).toContain("HEAD_SHA"); + const guardJob = growthGuardrails.jobs["codebase-growth-guardrails"]; + const guardRun = stepRuns(guardJob).join("\n"); + const guardEnv = JSON.stringify((guardJob.steps ?? []).map((step) => step.env ?? {})); + expect(guardEnv).toContain("HEAD_REPO"); expect(guardRun).not.toContain(".raw_url"); - expect(guardRun).toContain("previous_filename"); - expect(guardRun).toContain("budgetChanged"); - expect(guardRun).toContain("has a legacy budget but no matching test file at the PR head"); + expect(guardRun).not.toContain("node <<'NODE'"); + expect(guardRun).toContain("tools/growth-guardrails/test-size-budget.mts"); + expect(guardRun).toContain("tools/growth-guardrails/test-conditionals.mts"); }); // source-shape-contract: security -- Coverage publication must exclude fork-authored reports and pin the publishing action diff --git a/tools/growth-guardrails/pr-blob-client.mts b/tools/growth-guardrails/pr-blob-client.mts new file mode 100644 index 00000000000..26acf2fc689 --- /dev/null +++ b/tools/growth-guardrails/pr-blob-client.mts @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Shared GitHub pull-request file/blob client for the codebase growth +// guardrails. Both policy evaluators (test-size budget and test-conditional +// counting) need the same batched GraphQL blob fetch, REST fallback, and +// transient-failure retry. This module owns that once instead of duplicating it +// in two workflow heredocs. +// +// Trust boundary: this runs from the trusted base checkout under +// pull_request_target. It only reads GitHub's diff metadata and blob text as +// DATA. It never executes pull-request-controlled code. + +const GRAPHQL_URL = "https://api.github.com/graphql"; +const REST_API_ROOT = "https://api.github.com"; + +export const GRAPHQL_BATCH_SIZE = 25; +export const RETRY_ATTEMPTS = 4; +export const RETRY_BASE_MS = 250; +export const RETRY_MAX_MS = 4000; + +export type FetchLike = (url: string, init?: Parameters[1]) => Promise; + +export type PullRequestFile = { + readonly filename: string; + readonly previous_filename?: string | null; + readonly status?: string; + readonly additions?: number; + readonly deletions?: number; +}; + +/** A fetched blob maps to its UTF-8 text, or null when the path is absent. */ +export type BlobMap = ReadonlyMap; + +export type PrBlobClientOptions = { + readonly token: string; + readonly fetchImpl?: FetchLike; + /** Injectable for deterministic tests; defaults to a real timer sleep. */ + readonly sleep?: (ms: number) => Promise; + /** Injectable for deterministic retry jitter in tests; defaults to Math.random. */ + readonly random?: () => number; +}; + +export type PrBlobClient = { + getPullFiles(repo: string, prNumber: string): Promise; + fetchBlobs(repoFullName: string, oid: string, paths: readonly string[]): Promise; +}; + +type RetriableError = Error & { transient?: boolean }; + +function defaultSleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function isTransientStatus(status: number): boolean { + return status === 408 || status === 425 || status === 429 || (status >= 500 && status <= 599); +} + +function splitRepoName(fullName: string): { owner: string; name: string } { + const [owner, name] = fullName.split("/"); + if (!owner || !name) throw new Error(`Unexpected repository full name: ${fullName}`); + return { owner, name }; +} + +function buildBlobQuery(paths: readonly string[]): string { + const aliases = paths + .map( + (_, index) => + ` f${index}: object(expression: $e${index}) { __typename ... on Blob { text isBinary isTruncated byteSize } }`, + ) + .join("\n"); + const params = paths.map((_, index) => `$e${index}: String!`).join(", "); + return `query($owner: String!, $name: String!, ${params}) {\n repository(owner: $owner, name: $name) {\n${aliases}\n }\n}`; +} + +export function createPrBlobClient(options: PrBlobClientOptions): PrBlobClient { + const fetchImpl = options.fetchImpl ?? (globalThis.fetch as FetchLike); + const sleep = options.sleep ?? defaultSleep; + const random = options.random ?? Math.random; + const headers = { + Authorization: `Bearer ${options.token}`, + "X-GitHub-Api-Version": "2022-11-28", + }; + + async function withRetry(label: string, fn: () => Promise): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= RETRY_ATTEMPTS; attempt += 1) { + try { + return await fn(); + } catch (error) { + lastError = error; + const err = error as RetriableError; + const retriable = + err && + (err.transient === true || /HTTP (408|425|429|5\d{2})/.test(String(err.message ?? ""))); + if (!retriable || attempt === RETRY_ATTEMPTS) throw error; + const jitter = random() * RETRY_BASE_MS; + const delay = Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** (attempt - 1)) + jitter; + console.error( + `retry: ${label} attempt ${attempt} failed (${err.message}); sleeping ${Math.round(delay)}ms`, + ); + await sleep(delay); + } + } + throw lastError; + } + + async function requestJson(url: string, init?: Parameters[1]): Promise { + let response: Response; + try { + response = await fetchImpl(url, init); + } catch (error) { + const wrapped: RetriableError = new Error( + `${url}: network error ${(error as Error)?.message ?? error}`, + ); + wrapped.transient = true; + throw wrapped; + } + if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); + return response.json(); + } + + function getJson(url: string): Promise { + return withRetry(url, () => requestJson(url, { headers })); + } + + async function graphql( + query: string, + variables: Record, + ): Promise> { + const body = JSON.stringify({ query, variables }); + const label = `graphql ${variables.owner ?? ""}/${variables.name ?? ""}@${variables.oid ?? ""}`; + return withRetry(label, async () => { + const payload = (await requestJson(GRAPHQL_URL, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body, + })) as { + data?: Record; + errors?: Array<{ message?: string; type?: string }>; + }; + if (Array.isArray(payload.errors) && payload.errors.length > 0) { + const messages = payload.errors.map((entry) => entry.message).join("; "); + const transient = payload.errors.some( + (entry) => + entry.type === "RATE_LIMITED" || + /timed? *out|unavailable|internal/i.test(entry.message ?? ""), + ); + const error: RetriableError = new Error(`${label}: GraphQL errors: ${messages}`); + if (transient) error.transient = true; + throw error; + } + return payload.data ?? {}; + }); + } + + async function getContentViaRest( + repo: string, + ref: string, + file: string, + ): Promise { + if (!file) return null; + const encodedPath = file.split("/").map(encodeURIComponent).join("/"); + const url = `${REST_API_ROOT}/repos/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`; + // This fallback only runs for blobs GraphQL truncated, i.e. large files + // (1-100 MB). The default /contents object shape drops `content` above 1 MB, + // so request the raw media type and read the body text directly. + const rawHeaders = { ...headers, Accept: "application/vnd.github.raw" }; + return withRetry(url, async () => { + let response: Response; + try { + response = await fetchImpl(url, { headers: rawHeaders }); + } catch (error) { + const wrapped: RetriableError = new Error( + `${url}: network error ${(error as Error)?.message ?? error}`, + ); + wrapped.transient = true; + throw wrapped; + } + if (response.status === 404) return null; + if (!response.ok) throw new Error(`${url}: HTTP ${response.status}`); + return response.text(); + }); + } + + async function getPullFiles(repo: string, prNumber: string): Promise { + const files: PullRequestFile[] = []; + for (let page = 1; ; page += 1) { + const batch = (await getJson( + `${REST_API_ROOT}/repos/${repo}/pulls/${prNumber}/files?per_page=100&page=${page}`, + )) as PullRequestFile[]; + files.push(...batch); + if (batch.length < 100) return files; + } + } + + async function fetchBlobs( + repoFullName: string, + oid: string, + paths: readonly string[], + ): Promise { + const results = new Map(); + if (paths.length === 0) return results; + const { owner, name } = splitRepoName(repoFullName); + const rest: string[] = []; + for (let start = 0; start < paths.length; start += GRAPHQL_BATCH_SIZE) { + const chunk = paths.slice(start, start + GRAPHQL_BATCH_SIZE); + const query = buildBlobQuery(chunk); + const variables: Record = { owner, name, oid }; + chunk.forEach((path, index) => { + variables[`e${index}`] = `${oid}:${path}`; + }); + const data = await graphql(query, variables); + const repository = data.repository as Record | undefined; + if (!repository) throw new Error(`GraphQL repository not found: ${repoFullName}`); + chunk.forEach((path, index) => { + const node = repository[`f${index}`]; + if (!node || node.__typename !== "Blob") { + results.set(path, null); + return; + } + if (node.isBinary) throw new Error(`${path} is binary; refusing to read`); + if (node.text == null || node.isTruncated) { + rest.push(path); + return; + } + results.set(path, node.text); + }); + } + for (const path of rest) { + results.set(path, await getContentViaRest(repoFullName, oid, path)); + } + return results; + } + + return { getPullFiles, fetchBlobs }; +} + +type BlobNode = { + readonly __typename?: string; + readonly text?: string | null; + readonly isBinary?: boolean; + readonly isTruncated?: boolean; + readonly byteSize?: number; +}; + +export function assertRepositoryName(repo: string | undefined, label: string): void { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo ?? "")) { + throw new Error(`${label} must be an owner/name repository name`); + } +} diff --git a/tools/growth-guardrails/test-conditionals.mts b/tools/growth-guardrails/test-conditionals.mts new file mode 100644 index 00000000000..c3e44f3c919 --- /dev/null +++ b/tools/growth-guardrails/test-conditionals.mts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Trusted policy evaluator: a changed test file may not add `if` statements to +// its body. Runs from the base checkout under pull_request_target and reads PR +// blobs as DATA only — blob text is PARSED with the TypeScript AST, never +// executed. +// +// The `if`-statement count reuses scanTextForTestConditionals from the local +// scanner (scripts/find-test-conditionals.mts) so the workflow and +// `npm run test-conditionals:scan` agree by construction. No second lexer lives +// in workflow YAML. + +import { scanTextForTestConditionals } from "../../scripts/find-test-conditionals.mts"; +import { + assertRepositoryName, + type BlobMap, + createPrBlobClient, + type PrBlobClient, +} from "./pr-blob-client.mts"; + +const TEST_FILE_RE = /^(test|src|nemoclaw\/src)\/.*\.(test|spec)\.(?:[cm]?[jt]s)$/; + +/** Count `if` statements in test source using the shared TypeScript AST scanner. */ +export function countIfStatements(file: string, text: string): number { + return scanTextForTestConditionals(file, text).length; +} + +function countText(file: string, text: string | null | undefined): number { + return text == null ? 0 : countIfStatements(file, text); +} + +export type ConditionalChange = { + /** Path to read at the base revision (previous name on a rename). */ + readonly basePath: string; + /** Path to read at the head revision, or null when removed/renamed-away. */ + readonly headPath: string | null; + /** Path used in violation output. */ + readonly displayName: string; +}; + +export type ConditionalEvaluation = { + readonly details: string[]; + readonly baseTotal: number; + readonly headTotal: number; +}; + +/** Pure policy: compare base vs head `if` counts per changed test file. */ +export function evaluateConditionalViolations( + changes: readonly ConditionalChange[], + baseBlobs: BlobMap, + headBlobs: BlobMap, +): ConditionalEvaluation { + const details: string[] = []; + let baseTotal = 0; + let headTotal = 0; + + for (const change of changes) { + const baseCount = countText(change.basePath, baseBlobs.get(change.basePath) ?? null); + const headCount = + change.headPath === null + ? 0 + : countText(change.headPath, headBlobs.get(change.headPath) ?? null); + baseTotal += baseCount; + headTotal += headCount; + if (headCount > baseCount) { + details.push( + `${change.headPath ?? change.displayName}: ${headCount} if statement(s), up from ${baseCount}`, + ); + } + } + + return { details, baseTotal, headTotal }; +} + +export type ConditionalEnv = { + readonly BASE_SHA: string; + readonly HEAD_REPO: string; + readonly HEAD_SHA: string; + readonly PR_NUMBER: string; + readonly REPO: string; +}; + +export type ConditionalResult = ConditionalEvaluation & { readonly ok: boolean }; + +/** Orchestrates fetch + evaluate. The client is injectable for tests. */ +export async function runTestConditionals( + client: PrBlobClient, + env: ConditionalEnv, +): Promise { + assertRepositoryName(env.REPO, "REPO"); + assertRepositoryName(env.HEAD_REPO, "HEAD_REPO"); + + const files = await client.getPullFiles(env.REPO, env.PR_NUMBER); + const changedTests = files.filter( + ({ filename, previous_filename }) => + TEST_FILE_RE.test(filename) || TEST_FILE_RE.test(previous_filename ?? ""), + ); + + const changes: ConditionalChange[] = changedTests.map((file) => { + const basePath = TEST_FILE_RE.test(file.previous_filename ?? "") + ? (file.previous_filename as string) + : file.filename; + const headPath = + file.status === "removed" || !TEST_FILE_RE.test(file.filename) ? null : file.filename; + return { basePath, headPath, displayName: file.filename }; + }); + + const basePaths = [...new Set(changes.map((change) => change.basePath))]; + const headPaths = [ + ...new Set(changes.map((change) => change.headPath).filter((p): p is string => p !== null)), + ]; + + const [baseBlobs, headBlobs] = await Promise.all([ + client.fetchBlobs(env.REPO, env.BASE_SHA, basePaths), + client.fetchBlobs(env.HEAD_REPO, env.HEAD_SHA, headPaths), + ]); + + const evaluation = evaluateConditionalViolations(changes, baseBlobs, headBlobs); + return { ...evaluation, ok: evaluation.details.length === 0 }; +} + +function readEnv(): ConditionalEnv & { GH_TOKEN: string } { + const { BASE_SHA, GH_TOKEN, HEAD_REPO, HEAD_SHA, PR_NUMBER, REPO } = process.env; + if (!BASE_SHA || !GH_TOKEN || !HEAD_REPO || !HEAD_SHA || !PR_NUMBER || !REPO) { + throw new Error( + "Missing required environment: BASE_SHA GH_TOKEN HEAD_REPO HEAD_SHA PR_NUMBER REPO", + ); + } + return { BASE_SHA, GH_TOKEN, HEAD_REPO, HEAD_SHA, PR_NUMBER, REPO }; +} + +async function main(): Promise { + const env = readEnv(); + const client = createPrBlobClient({ token: env.GH_TOKEN }); + const result = await runTestConditionals(client, env); + if (!result.ok) { + console.error("FAIL: changed test files add if statements."); + console.error( + `Changed test files contain ${result.headTotal} if statement(s) at PR head vs ${result.baseTotal} at base.`, + ); + console.error(""); + console.error( + "Test bodies should stay linear. Split conditional behavior into separate test cases, use it.skipIf/it.runIf for platform or environment gates, or move non-asserting setup branches into named helpers.", + ); + console.error(""); + console.error("Files with increased if counts:"); + for (const detail of result.details) console.error(`- ${detail}`); + console.error(""); + console.error("Run locally: npm run test-conditionals:scan -- --top 25"); + process.exit(1); + } + console.log( + `PASS: changed test files did not add if statements (${result.headTotal} at PR head vs ${result.baseTotal} at base).`, + ); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/tools/growth-guardrails/test-size-budget.mts b/tools/growth-guardrails/test-size-budget.mts new file mode 100644 index 00000000000..f64500d0d55 --- /dev/null +++ b/tools/growth-guardrails/test-size-budget.mts @@ -0,0 +1,218 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Trusted policy evaluator: changed test files must stay within the size budget, +// and the budget itself must stay monotonic (a PR may not weaken it). Runs from +// the base checkout under pull_request_target and reads PR blobs as DATA only. +// +// Line counting and budget parsing are reused from the local scanner +// (scripts/check-test-file-size-budget.mts) so the workflow and the local check +// agree by construction — no second implementation lives in workflow YAML. + +import { + countLines, + parseBudget, + type TestFileSizeBudget, +} from "../../scripts/check-test-file-size-budget.mts"; +import { + type BlobMap, + createPrBlobClient, + type PrBlobClient, + type PullRequestFile, +} from "./pr-blob-client.mts"; + +const BUDGET_FILE = "ci/test-file-size-budget.json"; +const FALLBACK_BUDGET = '{"defaultMaxLines":1500,"legacyMaxLines":{}}'; +const TEST_FILE_RE = /^(test|src|nemoclaw\/src)\/.*\.(test|spec)\.(ts|js|mts|mjs|cts|cjs)$/; + +export type BudgetEvaluationInput = { + readonly baseBudget: TestFileSizeBudget; + readonly headBudget: TestFileSizeBudget; + readonly headBlobs: BlobMap; + readonly changedTests: readonly string[]; + /** + * Maps a renamed test's head path to its base path, so a moved legacy + * allowance is compared against the base-path entry instead of read as new. + */ + readonly renames: ReadonlyMap; +}; + +function legacyOf(budget: TestFileSizeBudget): Readonly> { + return budget.legacyMaxLines ?? {}; +} + +/** + * Pure budget policy. Returns human-readable violation strings (empty = PASS). + * Throws only for a structural impossibility (a changed test missing at head). + * + * When the base budget file is absent the caller passes the 1500-line fallback + * as baseBudget, so default-limit monotonicity is always enforced against it. + */ +export function evaluateTestSizeBudgetViolations(input: BudgetEvaluationInput): string[] { + const { baseBudget, headBudget, headBlobs, changedTests, renames } = input; + const baseLegacy = legacyOf(baseBudget); + const headLegacy = legacyOf(headBudget); + const violations: string[] = []; + + if (headBudget.defaultMaxLines > baseBudget.defaultMaxLines) { + violations.push( + `defaultMaxLines increased from ${baseBudget.defaultMaxLines} to ${headBudget.defaultMaxLines}`, + ); + } + + // Each head legacy entry is compared against its base allowance, following a + // rename so a moved-but-unchanged budget is not misread as newly added. + for (const [file, headMax] of Object.entries(headLegacy)) { + const baseMax = baseLegacy[renames.get(file) ?? file]; + if (baseMax === undefined && headMax > headBudget.defaultMaxLines) { + violations.push( + `${file} adds a new legacy budget (${headMax}) above defaultMaxLines (${headBudget.defaultMaxLines})`, + ); + } + if (baseMax !== undefined && headMax > baseMax) { + violations.push(`${file} legacy budget increased from ${baseMax} to ${headMax}`); + } + const text = headBlobs.get(file); + if (text == null) { + violations.push(`${file} has a legacy budget but no matching test file at the PR head`); + continue; + } + const lines = countLines(text); + if (lines > headMax) + violations.push(`${file} has ${lines} line(s), above its legacy budget ${headMax}`); + if (lines < headMax) { + violations.push( + `${file}: ${lines} line(s) < ${headMax} legacy budget; lower the budget entry`, + ); + } + } + + // A base legacy entry dropped at head (and not carried over by a rename) must + // not be removed while its file still exceeds the default. + const carriedBases = new Set(Object.keys(headLegacy).map((file) => renames.get(file) ?? file)); + for (const file of Object.keys(baseLegacy)) { + if (headLegacy[file] !== undefined || carriedBases.has(file)) continue; + const text = headBlobs.get(file); + if (text != null && countLines(text) > headBudget.defaultMaxLines) { + violations.push(`${file} removed its legacy budget while still exceeding defaultMaxLines`); + } + } + + for (const filename of changedTests) { + const text = headBlobs.get(filename); + if (text == null) throw new Error(`Changed test file ${filename} was not found at the PR head`); + const lines = countLines(text); + const maxLines = headLegacy[filename] ?? headBudget.defaultMaxLines; + if (lines > maxLines) violations.push(`${filename}: ${lines} line(s) > ${maxLines}`); + } + + return violations; +} + +export type BudgetEnv = { + readonly BASE_SHA: string; + readonly HEAD_REPO: string; + readonly HEAD_SHA: string; + readonly PR_NUMBER: string; + readonly REPO: string; +}; + +export type BudgetResult = { + readonly ok: boolean; + readonly violations: readonly string[]; + readonly changedTestCount: number; +}; + +/** Orchestrates fetch + evaluate. The client is injectable for tests. */ +export async function runTestSizeBudget( + client: PrBlobClient, + env: BudgetEnv, +): Promise { + const files = await client.getPullFiles(env.REPO, env.PR_NUMBER); + const budgetChanged = files.some( + ({ filename, previous_filename }) => + filename === BUDGET_FILE || previous_filename === BUDGET_FILE, + ); + const changedTests = files + .filter( + ({ filename, status }: PullRequestFile) => + status !== "removed" && TEST_FILE_RE.test(filename), + ) + .map(({ filename }) => filename); + + // Renamed test files map their head path back to their base path so a moved + // legacy budget is compared against the base-path allowance, not read as new. + const renames = new Map(); + for (const { filename, previous_filename } of files) { + if (previous_filename && previous_filename !== filename) + renames.set(filename, previous_filename); + } + + // Base budget lives in the trusted base repo; fetch it alone so we can parse + // legacy entries before deciding which HEAD blobs to load. When it is absent + // the 1500-line fallback becomes the baseline the head budget must not weaken. + const baseBlobs = await client.fetchBlobs(env.REPO, env.BASE_SHA, [BUDGET_FILE]); + const baseText = baseBlobs.get(BUDGET_FILE); + const baseBudget = parseBudget(baseText ?? FALLBACK_BUDGET, "base budget"); + + let headBudget = baseBudget; + if (budgetChanged) { + const headBudgetBlobs = await client.fetchBlobs(env.HEAD_REPO, env.HEAD_SHA, [BUDGET_FILE]); + const headText = headBudgetBlobs.get(BUDGET_FILE); + if (headText == null) + throw new Error(`${BUDGET_FILE} must remain present and parseable at the PR head`); + headBudget = parseBudget(headText, "head budget"); + } + + // Assemble every HEAD path we still need in one deduplicated batch: each HEAD + // legacy file, each BASE legacy file that HEAD dropped, and each changed test. + const headPaths = new Set(); + for (const file of Object.keys(legacyOf(headBudget))) headPaths.add(file); + for (const file of Object.keys(legacyOf(baseBudget))) { + if (legacyOf(headBudget)[file] === undefined) headPaths.add(file); + } + for (const filename of changedTests) headPaths.add(filename); + + const headBlobs = await client.fetchBlobs(env.HEAD_REPO, env.HEAD_SHA, Array.from(headPaths)); + + const violations = evaluateTestSizeBudgetViolations({ + baseBudget, + headBudget, + headBlobs, + changedTests, + renames, + }); + + return { ok: violations.length === 0, violations, changedTestCount: changedTests.length }; +} + +function readEnv(): BudgetEnv & { GH_TOKEN: string } { + const { BASE_SHA, GH_TOKEN, HEAD_REPO, HEAD_SHA, PR_NUMBER, REPO } = process.env; + if (!BASE_SHA || !GH_TOKEN || !HEAD_REPO || !HEAD_SHA || !PR_NUMBER || !REPO) { + throw new Error( + "Missing required environment: BASE_SHA GH_TOKEN HEAD_REPO HEAD_SHA PR_NUMBER REPO", + ); + } + return { BASE_SHA, GH_TOKEN, HEAD_REPO, HEAD_SHA, PR_NUMBER, REPO }; +} + +async function main(): Promise { + const env = readEnv(); + const client = createPrBlobClient({ token: env.GH_TOKEN }); + const result = await runTestSizeBudget(client, env); + if (!result.ok) { + console.error("FAIL: test size budget policy would be weakened or exceeded."); + for (const violation of result.violations) console.error(`- ${violation}`); + process.exit(1); + } + console.log( + `PASS: test size budget policy is monotonic and ${result.changedTestCount} changed test file(s) are within budget.`, + ); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/tools/growth-guardrails/workflow-boundary.mts b/tools/growth-guardrails/workflow-boundary.mts new file mode 100644 index 00000000000..8cedbdec48e --- /dev/null +++ b/tools/growth-guardrails/workflow-boundary.mts @@ -0,0 +1,292 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Trust-boundary assertions for the codebase growth guardrails workflow. +// +// The workflow runs under pull_request_target, so it executes in the base-repo +// context with a token. It must therefore stay data-only with respect to the +// pull request: inspect PR metadata and blob text, but never check out or +// execute pull-request-controlled code. This module parses the workflow and +// returns a violation string for every broken invariant (empty array = OK), so +// a regression that weakens the boundary fails a unit test rather than shipping. + +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; + +import YAML from "yaml"; + +/** The only ref a checkout in this workflow may use: the trusted base commit. */ +export const TRUSTED_BASE_REF = "${{ github.event.pull_request.base.sha }}"; + +/** The trusted tool entrypoints that replace the former inline node heredocs. */ +export const REQUIRED_TOOL_INVOCATIONS = [ + "node --experimental-strip-types tools/growth-guardrails/test-size-budget.mts", + "node --experimental-strip-types tools/growth-guardrails/test-conditionals.mts", +] as const; + +const HEAD_REF_MARKERS = [ + "github.event.pull_request.head", + "github.head_ref", + "refs/pull/", +] as const; + +const TRUSTED_JOB_ID = "codebase-growth-guardrails"; +const APPROVED_WORKFLOW_ENVELOPE_SHA256 = + "d3dcbf1a277f7d41fa3ff0357935027be900d866c5b808bb9b31e851d5719968"; +const APPROVED_JOB_ENVELOPE_SHA256 = + "a674580895f5761361f13fc49eecab030694c28a403c98cba38ee24fdfac15f1"; + +// Hash the complete parsed step object, not only `run`. This binds names, env, +// conditions, action refs and inputs, and execution fields such as `shell`, +// while allowing comments and YAML formatting to change without weakening the +// trust boundary. +const APPROVED_STEP_SHAPES = [ + { + name: "Block newly added JavaScript files", + sha256: "c2291c5ea47f093845b8e6bc6a93698fd9fe3f617b5c03265c2803439749ea38", + }, + { + name: "Require src/lib/onboard.ts to be net-neutral or smaller", + sha256: "92aba85cd31e30bfaa9cdd05dde6962f14373b1d60a9121bfcea48146ca0348b", + }, + { + name: "Check out the trusted base revision", + sha256: "ca92ffc6907f8ef3ddcf98eacb19dee0de5b7a76b0520f0ada097fa88b0af3b2", + }, + { + name: "Detect guardrail tools on the base revision", + sha256: "f10b24b97320e991cc060be9f5e98bda2705abd8d0814da123cfd5ca8672a442", + }, + { + name: "Install trusted dependencies", + sha256: "bf5757db70862f1e068748855d97ab5ae6a4a43ebd7ed812baa9f45269bdf6c3", + }, + { + name: "Require changed test files to stay within size budget", + sha256: "eec9020e81cf9d972592cc3129f6656ec0d46618623fb368674e8e823d8c1457", + }, + { + name: "Require changed test files not to add if statements", + sha256: "9aa363162eb2dc6740d9a27c52e6171b2a5587017eb039773a0f4ee0b32f8cde", + }, +] as const; + +type WorkflowStep = { + readonly [key: string]: unknown; + readonly name?: string; + readonly uses?: string; + readonly run?: string; + readonly with?: Record; +}; + +type WorkflowJob = { + readonly [key: string]: unknown; + readonly steps?: readonly WorkflowStep[]; + readonly permissions?: Record; +}; + +type WorkflowDoc = { + readonly [key: string]: unknown; + readonly on?: Record; + readonly permissions?: Record; + readonly jobs?: Record; +}; + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (value !== null && typeof value === "object") { + const object = value as Readonly>; + return Object.fromEntries( + Object.keys(object) + .sort() + .map((key) => [key, canonicalize(object[key])]), + ); + } + return value; +} + +function canonicalSha256(value: unknown): string { + const serialized = JSON.stringify(canonicalize(value)); + if (serialized === undefined) throw new Error("cannot hash an undefined workflow shape"); + return createHash("sha256").update(serialized).digest("hex"); +} + +function withoutKey( + object: Readonly>, + excludedKey: string, +): Record { + return Object.fromEntries(Object.entries(object).filter(([key]) => key !== excludedKey)); +} + +function allSteps(wf: WorkflowDoc): WorkflowStep[] { + return Object.values(wf.jobs ?? {}).flatMap((job) => job.steps ?? []); +} + +export function validateGrowthGuardrailsWorkflowBoundary(workflowPath: string): string[] { + const violations: string[] = []; + const wf = YAML.parse(readFileSync(workflowPath, "utf8")) as WorkflowDoc; + + // 1. Trigger must be pull_request_target (base context) and never the + // untrusted pull_request head context. + const on = wf.on ?? {}; + if (!("pull_request_target" in on)) { + violations.push("workflow must trigger on pull_request_target"); + } + if ("pull_request" in on) { + violations.push( + "workflow must not trigger on pull_request (runs untrusted head code with a token)", + ); + } + + // 2. Permissions must stay read-only: a write scope would let PR-influenced + // logic mutate the repo. + const permissions = wf.permissions ?? {}; + if (Object.keys(permissions).length === 0) { + violations.push("workflow must declare explicit read-only permissions"); + } + for (const [scope, value] of Object.entries(permissions)) { + if (value !== "read" && value !== "none") { + violations.push(`permission ${scope}: ${String(value)} must be read or none, not write`); + } + } + // A job-level permissions block overrides the workflow default, so a write + // scope there would reopen the boundary even with read-only top-level perms. + for (const [jobId, job] of Object.entries(wf.jobs ?? {})) { + for (const [scope, value] of Object.entries(job.permissions ?? {})) { + if (value !== "read" && value !== "none") { + violations.push( + `job ${jobId} permission ${scope}: ${String(value)} must be read or none, not write`, + ); + } + } + } + + // 3. Bind the complete execution shape. A substring allowlist is not a trust + // boundary: a permitted command can be followed by another executor, and + // an extra action or reusable-workflow job has no `run` text to inspect. + // The envelopes reject new execution fields, while the ordered full-step + // hashes reject missing, duplicate, extra, reordered, or modified steps. + const workflowEnvelopeHash = canonicalSha256(withoutKey(wf, "jobs")); + if (workflowEnvelopeHash !== APPROVED_WORKFLOW_ENVELOPE_SHA256) { + violations.push( + `workflow envelope must match the approved shape (sha256: ${workflowEnvelopeHash})`, + ); + } + + const jobs = wf.jobs ?? {}; + const jobIds = Object.keys(jobs); + if (jobIds.length !== 1 || jobIds[0] !== TRUSTED_JOB_ID) { + violations.push( + `workflow jobs must be exactly ${TRUSTED_JOB_ID}, not ${jobIds.join(", ") || "none"}`, + ); + } + + const trustedJob = jobs[TRUSTED_JOB_ID]; + if (trustedJob) { + const jobEnvelopeHash = canonicalSha256(withoutKey(trustedJob, "steps")); + if (jobEnvelopeHash !== APPROVED_JOB_ENVELOPE_SHA256) { + violations.push( + `job ${TRUSTED_JOB_ID} envelope must match the approved shape (sha256: ${jobEnvelopeHash})`, + ); + } + + const jobSteps = trustedJob.steps ?? []; + if (jobSteps.length !== APPROVED_STEP_SHAPES.length) { + violations.push( + `job ${TRUSTED_JOB_ID} must contain exactly ${APPROVED_STEP_SHAPES.length} approved steps, not ${jobSteps.length}`, + ); + } + + const stepNames = jobSteps.flatMap((step) => (step.name ? [step.name] : [])); + const duplicateNames = [ + ...new Set(stepNames.filter((name, index) => stepNames.indexOf(name) !== index)), + ]; + if (duplicateNames.length > 0) { + violations.push( + `job ${TRUSTED_JOB_ID} must not contain duplicate step names: ${duplicateNames.join(", ")}`, + ); + } + + for (const [index, approved] of APPROVED_STEP_SHAPES.entries()) { + const step = jobSteps[index]; + if (!step) continue; + if (step.name !== approved.name) { + violations.push( + `job ${TRUSTED_JOB_ID} step ${index + 1} must be ${approved.name}, not ${step.name ?? "unnamed"}`, + ); + } + const stepHash = canonicalSha256(step); + if (stepHash !== approved.sha256) { + violations.push( + `job ${TRUSTED_JOB_ID} step ${step.name ?? index + 1} must match the approved shape (sha256: ${stepHash})`, + ); + } + } + } + + const steps = allSteps(wf); + + // 4. Any checkout must pin the trusted base commit and never the PR head. + const checkoutSteps = steps.filter((step) => (step.uses ?? "").startsWith("actions/checkout")); + if (checkoutSteps.length === 0) { + violations.push("workflow must check out the trusted base ref to run the guardrail tools"); + } + for (const step of checkoutSteps) { + const ref = step.with?.ref; + if (typeof ref !== "string") { + violations.push("actions/checkout must pin an explicit ref (the trusted base sha)"); + continue; + } + if (ref !== TRUSTED_BASE_REF) { + violations.push(`actions/checkout ref must be ${TRUSTED_BASE_REF}, not ${ref}`); + } + if (HEAD_REF_MARKERS.some((marker) => ref.includes(marker))) { + violations.push(`actions/checkout must not reference the PR head ref (${ref})`); + } + } + + // 5. Each policy must still be invoked through its pinned trusted tool. + const runScripts = steps.flatMap((step) => (step.run ? [step.run] : [])); + const allRun = runScripts.join("\n"); + for (const invocation of REQUIRED_TOOL_INVOCATIONS) { + if (!allRun.includes(invocation)) { + violations.push(`workflow must invoke the trusted tool: ${invocation}`); + } + } + + // 6. Preserve specific diagnostics for dependency installs and PR-head Git + // operations in addition to the exact execution-shape check above. + for (const script of runScripts) { + for (const line of script.split("\n")) { + if (/\bnpm (ci|install)\b/.test(line) && !line.includes("--ignore-scripts")) { + violations.push(`dependency install must use --ignore-scripts: ${line.trim()}`); + } + if ( + /\bgit (checkout|fetch|merge)\b/.test(line) && + HEAD_REF_MARKERS.some((marker) => line.includes(marker)) + ) { + violations.push(`must not check out PR head code in a run step: ${line.trim()}`); + } + } + } + + return violations; +} + +async function main(): Promise { + const workflowPath = process.argv[2] ?? ".github/workflows/codebase-growth-guardrails.yaml"; + const violations = validateGrowthGuardrailsWorkflowBoundary(workflowPath); + if (violations.length > 0) { + console.error(`FAIL: ${workflowPath} violates the growth-guardrails trust boundary:`); + for (const violation of violations) console.error(`- ${violation}`); + process.exit(1); + } + console.log(`PASS: ${workflowPath} satisfies the growth-guardrails trust boundary.`); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +}