-
Notifications
You must be signed in to change notification settings - Fork 3.1k
feat(skills/merge-gate): add PR Review Advisor as a hard gate in check-gates.ts #5601
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
8e012dc
feat(skills/merge-gate): add PR Review Advisor as a hard gate in chec…
prekshivyas a5326ea
fix(skills/merge-gate): address PRA-3/4/5/6 in check-gates advisor gate
prekshivyas 26aa97e
Merge branch 'main' into fix/merge-gate-pra-check
prekshivyas b363569
fix(skills/merge-gate): validate advisor Actions run provenance in PR…
prekshivyas 16b7781
Merge remote-tracking branch 'upstream/fix/merge-gate-pra-check' into…
prekshivyas 3bfcd09
Merge branch 'main' into fix/merge-gate-pra-check
prekshivyas File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
191 changes: 191 additions & 0 deletions
191
.agents/skills/nemoclaw-maintainer-day/scripts/pra-gate.ts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| /** | ||
| * Pure PR Review Advisor gate logic — no shell calls, fully unit-testable. | ||
| * | ||
| * Exported and used by check-gates.ts. Separated so tests can exercise the | ||
| * parsing and provenance validation without mocking `gh`. | ||
| */ | ||
|
|
||
| // --------------------------------------------------------------------------- | ||
| // Types | ||
| // --------------------------------------------------------------------------- | ||
|
|
||
| export interface PraComment { | ||
| id: number; | ||
| user?: { login?: string }; | ||
| body?: string; | ||
| updated_at?: string; | ||
| } | ||
|
|
||
| export interface PraRun { | ||
| name?: string; | ||
| head_sha?: string; | ||
| event?: string; | ||
| run_attempt?: number; | ||
| run_started_at?: string; | ||
| created_at?: string; | ||
| updated_at?: string; | ||
| } | ||
|
|
||
| export interface PraMeta { | ||
| headSha: string; | ||
| recommendation: string; | ||
| runId: number; | ||
| runAttempt: number; | ||
| commentId: number; | ||
| } | ||
|
|
||
| export interface PrAdvisorGateResult { | ||
| pass: boolean; | ||
| details: string; | ||
| recommendation?: string; | ||
| openRequired?: number; | ||
| } | ||
|
|
||
| // --------------------------------------------------------------------------- | ||
| // Constants | ||
| // --------------------------------------------------------------------------- | ||
|
|
||
| // Explicit allowlist: only these recommendation values mean "OK to merge". | ||
| // Anything else — including unknown values — fails the gate. | ||
| // Source: SUMMARY_RECOMMENDATIONS in tools/pr-review-advisor/analyze.mts. | ||
| // "approved" is not a valid advisor recommendation; only "merge_as_is" is. | ||
| export const PRA_PASS_RECOMMENDATIONS = new Set(["merge_as_is"]); | ||
|
|
||
| // Full metadata line: all five fields must be present for a trusted comment. | ||
| const PRA_FULL_META_RE = | ||
| /head_sha:\s*([0-9a-f]+);\s*recommendation:\s*([a-z_]+);\s*run_id:\s*(\d+);\s*run_attempt:\s*(\d+);\s*comment_id:\s*(\d+)/i; | ||
|
|
||
| const PRA_REQUIRED_RE = /\*\*Open items:\*\*[^|]*?(\d+)\s+required/; | ||
|
|
||
| // --------------------------------------------------------------------------- | ||
| // Pure functions | ||
| // --------------------------------------------------------------------------- | ||
|
|
||
| /** | ||
| * Parse the embedded HTML metadata from a PRA comment body. | ||
| * Returns null when metadata is absent or any required field is missing. | ||
| */ | ||
| export function parsePraMeta(body: string): PraMeta | null { | ||
| const m = PRA_FULL_META_RE.exec(body); | ||
| if (!m) return null; | ||
| return { | ||
| headSha: m[1].toLowerCase(), | ||
| recommendation: m[2].toLowerCase(), | ||
| runId: parseInt(m[3], 10), | ||
| runAttempt: parseInt(m[4], 10), | ||
| commentId: parseInt(m[5], 10), | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * Evaluate a single PRA comment against the current PR head SHA. | ||
| * Validates provenance (comment_id and head_sha) before trusting the | ||
| * recommendation so a spoofed or stale comment cannot bypass the gate. | ||
| */ | ||
| export function evalPraComment(comment: PraComment, headSha: string): PrAdvisorGateResult { | ||
| const body = comment.body ?? ""; | ||
| const meta = parsePraMeta(body); | ||
|
|
||
| if (!meta) { | ||
| return { | ||
| pass: false, | ||
| details: "PR Review Advisor marker present but metadata incomplete — fail-closed", | ||
| }; | ||
| } | ||
|
|
||
| if (meta.commentId !== comment.id) { | ||
| return { | ||
| pass: false, | ||
| details: "PR Review Advisor comment_id mismatch — fail-closed (possible spoof)", | ||
| }; | ||
| } | ||
|
|
||
| const normalizedHead = headSha.toLowerCase(); | ||
| if (meta.headSha !== normalizedHead) { | ||
| return { | ||
| pass: false, | ||
| details: `PR Review Advisor is stale (sha ${meta.headSha.slice(0, 7)} ≠ head ${normalizedHead.slice(0, 7)}) — re-run CI`, | ||
| }; | ||
| } | ||
|
|
||
| const rec = meta.recommendation; | ||
| if (PRA_PASS_RECOMMENDATIONS.has(rec)) { | ||
| return { pass: true, details: `PR Review Advisor: ${rec}`, recommendation: rec }; | ||
| } | ||
|
|
||
| const requiredMatch = PRA_REQUIRED_RE.exec(body); | ||
| const openRequired = requiredMatch ? parseInt(requiredMatch[1], 10) : undefined; | ||
|
|
||
| return { | ||
| pass: false, | ||
| details: `PR Review Advisor: ${rec}${openRequired !== undefined ? ` (${openRequired} required item(s))` : ""}`, | ||
| recommendation: rec, | ||
| openRequired, | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * Parse NDJSON output from `gh api --paginate --jq ".[]"`. | ||
| * Each line is one JSON comment object; malformed lines are skipped. | ||
| */ | ||
| export function parsePraCommentNdjson(raw: string): PraComment[] { | ||
| const comments: PraComment[] = []; | ||
| for (const line of raw.split("\n")) { | ||
| const trimmed = line.trim(); | ||
| if (!trimmed) continue; | ||
| try { | ||
| comments.push(JSON.parse(trimmed) as PraComment); | ||
| } catch { | ||
| // skip malformed lines | ||
| } | ||
| } | ||
| return comments; | ||
| } | ||
|
|
||
| /** | ||
| * Return the latest github-actions[bot] comment that contains the PRA marker. | ||
| * Only github-actions[bot] is trusted; user-posted comments are ignored. | ||
| */ | ||
| export function selectLatestTrustedPraComment(comments: PraComment[]): PraComment | null { | ||
| const trusted = comments.filter( | ||
| (c) => | ||
| c.user?.login === "github-actions[bot]" && | ||
| (c.body ?? "").includes("nemoclaw-pr-review-advisor"), | ||
| ); | ||
| return trusted.length > 0 ? trusted[trusted.length - 1] : null; | ||
| } | ||
|
|
||
| // --------------------------------------------------------------------------- | ||
| // Run provenance | ||
| // --------------------------------------------------------------------------- | ||
|
|
||
| function isTimestampWithin(value: string, start: string, end: string): boolean { | ||
| const t = Date.parse(value); | ||
| const s = Date.parse(start); | ||
| const e = Date.parse(end); | ||
| if (![t, s, e].every(Number.isFinite)) return false; | ||
| return t >= s && t <= e; | ||
| } | ||
|
|
||
| /** | ||
| * Verify that a GitHub Actions run corresponds to the trusted PR Review / Advisor | ||
| * workflow for this PR head. Mirrors isTrustedAdvisorRun() in | ||
| * tools/pr-review-advisor/analyze.mts. | ||
| * | ||
| * Pure function — the caller is responsible for fetching the run data. | ||
| */ | ||
| export function validateAdvisorRun(run: PraRun, meta: PraMeta, commentUpdatedAt: string): boolean { | ||
| const startedAt = run.run_started_at ?? run.created_at; | ||
| const endedAt = run.updated_at; | ||
| if (!startedAt || !endedAt) return false; | ||
| return ( | ||
| run.name === "PR Review / Advisor" && | ||
| run.event === "pull_request" && | ||
| (run.head_sha ?? "").toLowerCase() === meta.headSha && | ||
| (run.run_attempt ?? -1) === meta.runAttempt && | ||
| isTimestampWithin(commentUpdatedAt, startedAt, endedAt) | ||
| ); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🧩 Analysis chain
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 69
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 41
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 125
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 41
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 735
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 1022
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 118
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 3369
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 480
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 714
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 121
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 120
🏁 Script executed:
Repository: NVIDIA/NemoClaw
Length of output: 4701
Remove early return on empty output; distinguish API failures from zero comments
Line 283 treats empty
rawas an API error, but--jq ".[]"also produces empty output for zero comments. This blocks valid PRs with no issue comments. The subsequent logic (parsePraCommentNdjson → selectLatestTrustedPraComment) already handles the empty case correctly, so the early check should not fail-close on empty output.Suggested fix
const raw = run("gh", [ "api", `repos/${repo}/issues/${number}/comments`, "--paginate", "--jq", - ".[]", + 'if length==0 then "__EMPTY__" else .[] end', ]); if (!raw) { return { pass: false, details: "Could not fetch PR comments (API error — fail-closed)" }; } + if (raw === "__EMPTY__") { + return { pass: true, details: "No PR Review Advisor comment found" }; + } const allComments = parsePraCommentNdjson(raw);🤖 Prompt for AI Agents