diff --git a/.agents/skills/nemoclaw-maintainer-cross-issue-sweep/SKILL.md b/.agents/skills/nemoclaw-maintainer-cross-issue-sweep/SKILL.md index fa93cd6a0cb..60d3a15dd57 100644 --- a/.agents/skills/nemoclaw-maintainer-cross-issue-sweep/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cross-issue-sweep/SKILL.md @@ -109,7 +109,7 @@ See `templates/report.md` for the format. ## Composition with other skills -The pr-comparator (`nemoclaw-maintainer-pr-comparator`) calls this skill as a sub-step when comparing competing PRs. Adjacent-fix counts feed Tier 3 tiebreakers; contradicting hits factor into Tier 2 quality scoring. +This skill is a separate, optional follow-up to `nemoclaw-maintainer-pr-comparator`. The comparator does not call it or include its findings in the deterministic score. Run the sweep explicitly when a maintainer wants adjacent-fix or contradiction evidence alongside the comparator verdict, and report that evidence separately. ## What this skill does NOT do (deferred) diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md index 100c5b0878d..3146ffce6ad 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md @@ -11,7 +11,7 @@ user_invocable: true Use the release scripts for normal release operations. Do not run raw `git tag`, `git push`, `gh api`, or version-bump commands by hand for the normal release flow. -The release is one annotated semver tag on an already-merged `origin/main` commit. The GitHub workflow moves `latest`; release admins promote `lkg` manually after validation. After the tag is cut, finish version-label housekeeping for remaining open issues/PRs and draft release notes for the maintainer to post. +The release is one annotated semver tag on an already-merged `origin/main` commit. The GitHub workflow moves `latest`; release admins promote `lkg` manually after validation. After the tag and `latest` are verified, automatically move remaining open issues/PRs from the released version label to the next patch label, then draft release notes for the maintainer to post. ## Hard Rules @@ -33,7 +33,7 @@ Release Progress: - [ ] Step 2: Show plan and exact confirmation phrase - [ ] Step 3: Cut the semver tag from the confirmed plan - [ ] Step 4: Wait for workflow-managed latest -- [ ] Step 5: Housekeep remaining open issues/PRs +- [ ] Step 5: Bump remaining open issues/PRs - [ ] Step 6: Generate release-note data and draft Markdown - [ ] Step 7: Hand off announcement steps ``` @@ -108,14 +108,16 @@ The script waits until `vX.Y.Z^{}` and `latest^{}` both peel to the planned comm If it fails, report the failed workflow/status. Do not manually move `latest`. -### Step 5: Housekeep Remaining Open Issues/PRs +### Step 5: Bump Remaining Open Issues/PRs -Move any remaining open issues or PRs labeled with the released version to the next patch label: +Move every remaining open issue or PR carrying the released version to the next patch label: ```bash node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer-day/scripts/bump-stragglers.ts ``` +This is automatic post-tag housekeeping covered by the release plan and exact confirmation in Step 2. The script creates the next patch label when needed, removes the released-version label, and adds the next-version label to every open straggler. Do not run it before Step 4 verifies both the semver tag and workflow-managed `latest`. + Then verify the released version has no open stragglers: ```bash @@ -125,7 +127,6 @@ gh pr list --repo NVIDIA/NemoClaw --state open --label --limi Summarize: -- shipped/closed items that remain associated with ``; - open issues/PRs bumped to ``; - any items that need manual maintainer attention. @@ -174,4 +175,4 @@ Return: - `latest` workflow fails or times out: report the workflow/status; do not move `latest` manually. - `latest` workflow rejects a rollback: keep `latest` unchanged, inspect the plan target commit, and regenerate the plan for the current `origin/main` tip if appropriate. - `lkg` changed: stop and escalate to a release admin. -- Housekeeping finds open items that should still ship in the released version: stop and ask the maintainer whether to leave the label or bump them. +- Post-tag housekeeping fails: report the error and list items still carrying the released label. After the failure is fixed, rerun the same bump command; already-moved items no longer match the source label. diff --git a/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md b/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md index a0b264844d4..aad4590244c 100644 --- a/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md +++ b/.agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md @@ -1,3 +1,6 @@ + + + # Merge Gate Workflow Run the last maintainer check before approval. Never merge automatically. @@ -6,11 +9,12 @@ Run the last maintainer check before approval. Never merge automatically. For the full priority list see [PR-REVIEW-PRIORITIES.md](PR-REVIEW-PRIORITIES.md). A PR is approval-ready only when **all** hard gates pass: -1. **CI green** — all required checks in `statusCheckRollup`. -2. **No conflicts** — `mergeStateStatus` clean. -3. **No major CodeRabbit** — ignore style nits; block on correctness/security bugs. -4. **PR Review Advisor: merge_as_is** — `check-gates.ts` checks this automatically. The gate passes only when the latest advisor comment has `recommendation: merge_as_is`. All other recommendation values — including `blocked`, `needs_rework`, `merge_after_fixes`, `superseded`, `info_only`, and any unknown value — fail the gate. The referenced Actions run is validated (name, event, head SHA, run attempt, timestamp) before the recommendation is trusted. Correctness, security, acceptance, and test-depth findings block until addressed or explicitly judged false-positive by a maintainer. -5. **Risky code tested** — see [RISKY-AREAS.md](RISKY-AREAS.md). Confirm tests exist (added or pre-existing). +1. **Contributor compliance** — the PR body contains the contributor's `Signed-off-by:` declaration and every PR commit appears as `Verified` in GitHub. Reject noncompliant PRs; maintainers do not repair contributor history. +2. **CI green** — all required checks in `statusCheckRollup`. +3. **No conflicts** — `mergeStateStatus` clean. +4. **No major CodeRabbit** — ignore style nits; block on correctness/security bugs. +5. **PR Review Advisor: merge_as_is** — `check-gates.ts` checks this automatically. The gate passes only when the latest advisor comment has `recommendation: merge_as_is`. All other recommendation values — including `blocked`, `needs_rework`, `merge_after_fixes`, `superseded`, `info_only`, and any unknown value — fail the gate. The referenced Actions run is validated (name, event, head SHA, run attempt, timestamp) before the recommendation is trusted. Correctness, security, acceptance, and test-depth findings block until addressed or explicitly judged false-positive by a maintainer. +6. **Risky code tested** — see [RISKY-AREAS.md](RISKY-AREAS.md). Confirm tests exist (added or pre-existing). ## Step 1: Run the Gate Checker @@ -25,6 +29,7 @@ This checks all gates programmatically and returns structured JSON with `allPass The script handles the deterministic checks. You handle judgment calls: - **Missing required checks:** The script verifies that `checks`, `commit-lint`, and `dco-check` are present in the status rollup. If any are missing, **workflows have not been triggered** — this happens on fork PRs from first-time contributors that need "Approve and run" clicked in the Actions tab. Go to the PR's Checks tab, approve the workflows, wait for all checks to complete, then re-run the gate checker. **Never approve a PR with missing checks.** +- **Contributor compliance failed:** Reject the PR and ask the contributor to provide the PR-body DCO declaration or replace unverified commits with a clean verified history. Do not approve, merge, amend, sign, or force-push on the contributor's behalf. - **Conflicts (DIRTY):** Do NOT approve — GitHub invalidates approvals when new commits are pushed. Salvage first (rebase), wait for CI, then re-run the gate checker. Follow [SALVAGE-PR.md](SALVAGE-PR.md). - **CI failing but narrow:** Follow the salvage workflow in [SALVAGE-PR.md](SALVAGE-PR.md). - **CI pending:** Wait and re-check. Do not approve while checks are still running. diff --git a/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md b/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md index 81585c0a629..2d497b070fe 100644 --- a/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md +++ b/.agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md @@ -1,15 +1,19 @@ + + + # PR Review Priorities Ordered list of what NemoClaw maintainers look for in a pull request. Higher items block approval; lower items inform queue ranking. ## Hard gates (all must pass to approve) -1. **Security correctness** — no sandbox escape, SSRF, credential exposure, policy bypass, or installer trust violation. PRs touching risky areas (see [RISKY-AREAS.md](RISKY-AREAS.md)) get a deep security pass before anything else. -2. **CI green** — all required checks in `statusCheckRollup` must pass. -3. **No merge conflicts** — `mergeStateStatus` must be clean. -4. **No unresolved major/critical CodeRabbit findings** — correctness and safety findings block; style nits do not. Use judgment on borderline cases. -5. **No unresolved actionable PR Review Advisor findings** — correctness, security, acceptance-coverage, and test-depth findings block unless explicitly judged false-positive. Ask the user before acting on ambiguous or design-changing advice. -6. **Tests for touched risky code** — risky areas must have test coverage, either added in the PR or pre-existing. No exceptions. +1. **Contributor compliance** — the PR body has the contributor's DCO declaration and every commit appears as `Verified` in GitHub. Maintainers reject noncompliant PRs and do not repair contributor history. +2. **Security correctness** — no sandbox escape, SSRF, credential exposure, policy bypass, or installer trust violation. PRs touching risky areas (see [RISKY-AREAS.md](RISKY-AREAS.md)) get a deep security pass before anything else. +3. **CI green** — all required checks in `statusCheckRollup` must pass. +4. **No merge conflicts** — `mergeStateStatus` must be clean. +5. **No unresolved major/critical CodeRabbit findings** — correctness and safety findings block; style nits do not. Use judgment on borderline cases. +6. **No unresolved actionable PR Review Advisor findings** — correctness, security, acceptance-coverage, and test-depth findings block unless explicitly judged false-positive. Ask the user before acting on ambiguous or design-changing advice. +7. **Tests for touched risky code** — risky areas must have test coverage, either added in the PR or pre-existing. No exceptions. ## Quality expectations (block if violated, but fixable via salvage) @@ -30,10 +34,10 @@ The team follows a daily ship cycle. All maintainer skills operate within this r 1. **Morning** (`/nemoclaw-maintainer-morning`) — triage the backlog, pick items for the day, label them with the target version (e.g., `v0.0.8`). 2. **During the day** (`/nemoclaw-maintainer-day`) — land PRs using the maintainer loop. Version labels make progress visible on dashboards. -3. **Evening** (`/nemoclaw-maintainer-evening`) — check what shipped, bump open items to the next version (`v0.0.9`), generate a QA-focused summary, cut the tag, and prepare release notes for posting. +3. **Evening** (`/nemoclaw-maintainer-evening`) — check what shipped, identify open stragglers, generate a QA-focused summary, cut the tag, automatically bump stragglers to the next patch, and prepare release notes for posting. 4. **Overnight** — QA team (different timezone) tests the tag. Any issues they file enter the next morning's triage like any other issue. -Version labels are living markers: they always mean "ship in this version." If an item doesn't make the cut, the label moves to the next patch version. +Version labels activate release work; they are not readiness claims. If an open item misses the tag, its label moves to the next patch during post-tag housekeeping. ## Explicitly not priorities diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts index fb762bbc896..f7ef91bec92 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts @@ -4,30 +4,30 @@ /** * Deterministic merge-gate checker for a single NemoClaw PR. * - * Checks all 5 required gates and outputs structured JSON. + * Checks all required gates and outputs structured JSON. * Claude uses the output to decide: approve, route to salvage, or report blockers. * * Usage: node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts [--repo OWNER/REPO] */ import { + evalPraComment, + type PrAdvisorGateResult, + type PraRun, + parsePraCommentNdjson, + parsePraMeta, + selectLatestTrustedPraComment, + validateAdvisorRun, +} from "./pra-gate.ts"; +import { + ghJson, isRiskyFile, isTestFile, - run, - ghJson, parseStringArg, REQUIRED_CHECK_NAMES, + run, type StatusCheck, } from "./shared.ts"; -import { - parsePraCommentNdjson, - parsePraMeta, - selectLatestTrustedPraComment, - evalPraComment, - validateAdvisorRun, - type PraRun, - type PrAdvisorGateResult, -} from "./pra-gate.ts"; // --------------------------------------------------------------------------- // Types @@ -60,6 +60,10 @@ interface GateOutput { coderabbit: GateResult & { unresolvedThreads?: CodeRabbitThread[] }; riskyCodeTested: GateResult & { riskyFiles?: string[]; hasTests?: boolean }; prAdvisor: PrAdvisorGateResult; + contributorCompliance: GateResult & { + dcoDeclarationPresent?: boolean; + unverifiedCommits?: Array<{ sha: string; reason: string }>; + }; }; } @@ -351,6 +355,112 @@ function checkRiskyCodeTested( }; } +// --------------------------------------------------------------------------- +// Gate 6: Contributor compliance +// --------------------------------------------------------------------------- + +const DCO_DECLARATION = /^Signed-off-by:\s+.+\s+<[^<>\s]+@[^<>\s]+>\s*$/mu; + +interface CommitVerificationRecord { + sha: string; + verified: boolean; + reason: string; +} + +function normalizeCommitVerification(value: unknown): CommitVerificationRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return { sha: "(unknown)", verified: false, reason: "malformed_commit_verification_data" }; + } + + const record = value as Record; + if ( + typeof record.sha !== "string" || + typeof record.verified !== "boolean" || + typeof record.reason !== "string" + ) { + return { + sha: typeof record.sha === "string" ? record.sha : "(unknown)", + verified: false, + reason: "malformed_commit_verification_data", + }; + } + + return { sha: record.sha, verified: record.verified, reason: record.reason }; +} + +function checkContributorCompliance( + repo: string, + number: number, + body: string, +): GateResult & { + dcoDeclarationPresent?: boolean; + unverifiedCommits?: Array<{ sha: string; reason: string }>; +} { + const dcoDeclarationPresent = DCO_DECLARATION.test(body ?? ""); + const raw = run("gh", [ + "api", + `repos/${repo}/pulls/${number}/commits`, + "--paginate", + "--jq", + '.[] | {sha, verified: (.commit.verification.verified // false), reason: (.commit.verification.reason // "unknown")}', + ]); + + if (!raw) { + return { + pass: false, + details: "Could not verify PR commit signatures (API error — fail-closed)", + dcoDeclarationPresent, + }; + } + + const commits: CommitVerificationRecord[] = []; + try { + for (const line of raw.split("\n")) { + const trimmed = line.trim(); + if (trimmed) commits.push(normalizeCommitVerification(JSON.parse(trimmed) as unknown)); + } + } catch { + return { + pass: false, + details: "Could not parse PR commit signature data — fail-closed", + dcoDeclarationPresent, + }; + } + + if (commits.length === 0) { + return { + pass: false, + details: "No PR commits returned while checking contributor compliance — fail-closed", + dcoDeclarationPresent, + }; + } + + const unverifiedCommits = commits + .filter((commit) => commit.verified !== true) + .map(({ sha, reason }) => ({ sha, reason })); + if (!dcoDeclarationPresent || unverifiedCommits.length > 0) { + const failures = [ + ...(dcoDeclarationPresent ? [] : ["PR body lacks a valid Signed-off-by declaration"]), + ...(unverifiedCommits.length > 0 + ? [`${unverifiedCommits.length} commit(s) are not GitHub Verified`] + : []), + ]; + return { + pass: false, + details: failures.join("; "), + dcoDeclarationPresent, + unverifiedCommits, + }; + } + + return { + pass: true, + details: `DCO declaration present; all ${commits.length} commit(s) are GitHub Verified`, + dcoDeclarationPresent, + unverifiedCommits: [], + }; +} + // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- @@ -372,11 +482,12 @@ function main(): void { "--repo", repo, "--json", - "number,title,url,files,statusCheckRollup,mergeStateStatus,headRefOid", + "number,title,url,body,files,statusCheckRollup,mergeStateStatus,headRefOid", ]) as { number: number; title: string; url: string; + body: string; files: Array<{ path: string; status: string }>; statusCheckRollup: StatusCheck[]; mergeStateStatus: string; @@ -393,13 +504,20 @@ function main(): void { const coderabbit = checkCodeRabbit(repo, prNumber); const riskyCodeTested = checkRiskyCodeTested(prData.files ?? []); const prAdvisor = checkPrAdvisor(repo, prNumber, prData.headRefOid ?? ""); + const contributorCompliance = checkContributorCompliance(repo, prNumber, prData.body ?? ""); const output: GateOutput = { pr: prNumber, url: prData.url, title: prData.title, - allPass: ci.pass && conflicts.pass && coderabbit.pass && riskyCodeTested.pass && prAdvisor.pass, - gates: { ci, conflicts, coderabbit, riskyCodeTested, prAdvisor }, + allPass: + ci.pass && + conflicts.pass && + coderabbit.pass && + riskyCodeTested.pass && + prAdvisor.pass && + contributorCompliance.pass, + gates: { ci, conflicts, coderabbit, riskyCodeTested, prAdvisor, contributorCompliance }, }; console.log(JSON.stringify(output, null, 2)); diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts index 5315d931659..ad3ab72e9d9 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/shared.ts @@ -122,8 +122,10 @@ export const SCORE_NEAR_MISS = 30; export const SCORE_SECURITY_ACTIONABLE = 20; /** PR carries the "security" GitHub label */ export const SCORE_LABEL_SECURITY = 15; -/** PR carries a "priority: high" GitHub label */ -export const SCORE_LABEL_PRIORITY_HIGH = 10; +/** PR has Urgent Project Priority */ +export const SCORE_PROJECT_PRIORITY_URGENT = 15; +/** PR has High Project Priority */ +export const SCORE_PROJECT_PRIORITY_HIGH = 10; /** PR has been stale > 7 days — mild priority bump to prevent rot */ export const SCORE_STALE_AGE = 5; diff --git a/.agents/skills/nemoclaw-maintainer-day/scripts/triage.ts b/.agents/skills/nemoclaw-maintainer-day/scripts/triage.ts index 27c81171c0b..ecea9f9f489 100644 --- a/.agents/skills/nemoclaw-maintainer-day/scripts/triage.ts +++ b/.agents/skills/nemoclaw-maintainer-day/scripts/triage.ts @@ -17,22 +17,23 @@ import { resolve } from "node:path"; import { isRiskyFile, - run, - parseStringArg, + PENALTY_BROAD_CI_RED, + PENALTY_CODERABBIT_MAJOR, + PENALTY_DRAFT_OR_CONFLICT, + PENALTY_MERGE_BLOCKED, parseIntArg, + parseStringArg, REQUIRED_CHECK_NAMES, - type StatusCheck, + run, + SCORE_LABEL_SECURITY, SCORE_MERGE_NOW, - SCORE_REVIEW_READY, SCORE_NEAR_MISS, + SCORE_PROJECT_PRIORITY_HIGH, + SCORE_PROJECT_PRIORITY_URGENT, + SCORE_REVIEW_READY, SCORE_SECURITY_ACTIONABLE, - SCORE_LABEL_SECURITY, - SCORE_LABEL_PRIORITY_HIGH, SCORE_STALE_AGE, - PENALTY_DRAFT_OR_CONFLICT, - PENALTY_CODERABBIT_MAJOR, - PENALTY_BROAD_CI_RED, - PENALTY_MERGE_BLOCKED, + type StatusCheck, } from "./shared.ts"; // --------------------------------------------------------------------------- @@ -73,6 +74,7 @@ interface ClassifiedPr { createdAt: string; draft: boolean; labels: string[]; + projectPriority: string | null; } interface QueueItem { @@ -90,6 +92,7 @@ interface QueueItem { nextAction: string; ageHours: number; labels: string[]; + projectPriority: string | null; } interface HotCluster { @@ -131,7 +134,7 @@ function ghApi(path: string): unknown { // Data fetching // --------------------------------------------------------------------------- -function fetchOpenPrs(repo: string, approvedOnly: boolean): PrData[] { +function fetchOpenPrs(repo: string): PrData[] { // Use gh api --paginate with REST for lightweight pagination (no GraphQL timeout). // --jq outputs one JSON object per PR per page; we collect them as NDJSON then parse. const out = run( @@ -170,9 +173,6 @@ function fetchOpenPrs(repo: string, approvedOnly: boolean): PrData[] { prs.push(JSON.parse(trimmed) as PrData); } } - if (approvedOnly) { - return prs.filter((pr) => pr.reviewDecision === "APPROVED"); - } return prs; } catch { return []; @@ -299,6 +299,7 @@ function classifyPr(pr: PrData): ClassifiedPr { createdAt: pr.createdAt, draft, labels: (pr.labels ?? []).map((l) => l.name), + projectPriority: null, }; } @@ -310,6 +311,67 @@ function fetchPrFiles(repo: string, number: number): string[] { return data.map((f) => f.filename); } +function fetchProjectPriorities(repo: string): Map { + if (repo !== "NVIDIA/NemoClaw") return new Map(); + + const out = run("gh", [ + "api", + "graphql", + "--paginate", + "-f", + `query=query($endCursor: String) { + organization(login: "NVIDIA") { + projectV2(number: 199) { + items(first: 100, after: $endCursor) { + nodes { + content { + ... on PullRequest { number repository { nameWithOwner } } + } + fieldValues(first: 100) { + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { + name + field { ... on ProjectV2SingleSelectField { name } } + } + } + } + } + pageInfo { hasNextPage endCursor } + } + } + } + }`, + "--jq", + '.data.organization.projectV2.items.nodes[] | {number: .content.number, repository: .content.repository.nameWithOwner, priority: ([.fieldValues.nodes[] | select(.field.name == "Priority") | .name][0] // null)}', + ]); + if (!out) return new Map(); + + try { + const priorities = new Map(); + for (const line of out.split("\n")) { + if (!line.trim()) continue; + const item = JSON.parse(line) as { + number?: number; + repository?: string; + priority?: string | null; + }; + if ( + item.repository === repo && + typeof item.number === "number" && + typeof item.priority === "string" + ) { + priorities.set(item.number, item.priority); + } + } + return priorities; + } catch { + process.stderr.write( + "Could not parse Project 199 item data; continuing without priority boosts.\n", + ); + return new Map(); + } +} + function loadState(): StateFile | null { const stateDir = resolve(".nemoclaw-maintainer"); const statePath = resolve(stateDir, "state.json"); @@ -356,10 +418,11 @@ function scoreItem( nextAction = bucket === "merge-now" ? "security-sweep → merge-gate" : "security-sweep → review"; } - // GitHub label boosts + // Canonical routing-label and Project Priority boosts const labelSet = new Set(item.labels.map((l) => l.toLowerCase())); if (labelSet.has("security")) score += SCORE_LABEL_SECURITY; - if (labelSet.has("priority: high")) score += SCORE_LABEL_PRIORITY_HIGH; + if (item.projectPriority === "Urgent") score += SCORE_PROJECT_PRIORITY_URGENT; + if (item.projectPriority === "High") score += SCORE_PROJECT_PRIORITY_HIGH; if (item.updatedAt) { const age = Date.now() - new Date(item.updatedAt).getTime(); @@ -421,27 +484,50 @@ function main(): void { // 1. Fetch all open PRs via REST (lightweight, paginated, no GraphQL timeout) process.stderr.write("Fetching all open PRs via REST...\n"); - const prs = fetchOpenPrs(repo, false); + const prs = fetchOpenPrs(repo); if (prs.length === 0) { console.error("No open PRs found. GitHub API may be experiencing issues."); process.exit(1); } process.stderr.write(`Found ${prs.length} open PRs. Filtering non-draft candidates...\n`); - // 2. Filter to non-draft, then enrich top candidates with CI + review data. + // 2. Fetch Project Priority before choosing the capped enrichment set so an + // Urgent or High PR cannot be hidden outside the REST list's initial order. + const projectPriorities = fetchProjectPriorities(repo); + const priorityRank = new Map([ + ["Urgent", 2], + ["High", 1], + ]); + + // 3. Filter to non-draft, prioritize Urgent/High, then enrich top candidates + // with CI + review data. // NOTE: Enrichment is capped at limit*3 by design — each enrichPr() call is // a separate GitHub API request, so we intentionally limit the blast radius. // Un-enriched PRs will classify as "blocked" (empty checks), which is the // safe default. This is NOT a bug. - const candidates = prs.filter((pr) => !pr.isDraft); + const candidates = prs + .filter((pr) => !pr.isDraft) + .sort( + (a, b) => + (priorityRank.get(projectPriorities.get(b.number) ?? "") ?? 0) - + (priorityRank.get(projectPriorities.get(a.number) ?? "") ?? 0), + ); const enrichCount = Math.min(candidates.length, limit * 3); process.stderr.write(`Enriching ${enrichCount} of ${candidates.length} candidates...\n`); for (let i = 0; i < enrichCount; i++) { enrichPr(repo, candidates[i]); } - // 3. Classify all PRs (un-enriched ones will be blocked due to empty checks) - const classified = prs.map(classifyPr); + // 4. Apply --approved-only after enrichment, when reviewDecision is known, + // then classify all retained PRs. Un-enriched PRs have an empty review + // decision and are excluded in approved-only mode; otherwise they classify + // as blocked due to empty checks. + const classified = prs + .filter((pr) => !approvedOnly || pr.reviewDecision === "APPROVED") + .map(classifyPr); + for (const item of classified) { + item.projectPriority = projectPriorities.get(item.number) ?? null; + } process.stderr.write( `Classified: ${classified.filter((c) => c.mergeNow).length} merge-now, ` + `${classified.filter((c) => c.reviewReady).length} review-ready, ` + @@ -492,6 +578,7 @@ function main(): void { ? Math.floor((Date.now() - new Date(item.createdAt).getTime()) / 3_600_000) : 0, labels: item.labels, + projectPriority: item.projectPriority, }); } diff --git a/.agents/skills/nemoclaw-maintainer-evening/SKILL.md b/.agents/skills/nemoclaw-maintainer-evening/SKILL.md index 5da3743934d..4e2902d2a37 100644 --- a/.agents/skills/nemoclaw-maintainer-evening/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-evening/SKILL.md @@ -1,12 +1,12 @@ --- name: nemoclaw-maintainer-evening -description: Runs the end-of-day maintainer handoff for NemoClaw. Checks version target progress, bumps stragglers to the next patch version, generates a QA handoff summary, and cuts the release tag. Use at the end of the workday. Trigger keywords - evening, end of day, EOD, wrap up, ship it, cut tag, handoff, done for the day. +description: Runs the end-of-day maintainer handoff for NemoClaw. Checks version target progress, records stragglers for an automatic post-tag bump, generates a QA handoff summary, and cuts the release tag. Use at the end of the workday. Trigger keywords - evening, end of day, EOD, wrap up, ship it, cut tag, handoff, done for the day. user_invocable: true --- # NemoClaw Maintainer Evening -Wrap up the day: check progress, bump stragglers, summarize for QA, cut the tag, and prepare release notes for posting. +Wrap up the day: check progress, identify stragglers, summarize for QA, cut the tag, automatically bump stragglers to the next patch, and prepare release notes for posting. See [PR-REVIEW-PRIORITIES.md](../nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md) for the daily cadence. @@ -19,13 +19,18 @@ node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer The first script determines the target version. The second shows shipped vs open. Present the progress summary to the user. -## Step 2: Bump Stragglers +## Step 2: Review Post-Tag Stragglers ```bash -node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer-day/scripts/bump-stragglers.ts +gh pr list --repo NVIDIA/NemoClaw --state open --label --limit 100 \ + --json number,title,url,labels +gh issue list --repo NVIDIA/NemoClaw --state open --label --limit 100 \ + --json number,title,url,labels ``` -This creates the next version label if needed, then moves all open items from the current version to the next. Tell the user what got bumped. +List open labeled PRs and issues as the post-tag housekeeping plan. Tell the maintainer that, after the tag and workflow-managed `latest` are verified, `cut-release-tag` will automatically move all of them to the next patch label. + +If an item should leave the daily release flow instead of moving forward, remove it from the released-version label before asking for the release confirmation phrase. ## Step 3: Generate Handoff Summary @@ -37,7 +42,7 @@ This lists commits since the last tag, identifies risky areas touched, and sugge ## Step 4: Cut the Tag and Publish Release Notes -Load `cut-release-tag`. The version is already known — default to patch bump, but still show the commit, changelog, housekeeping plan, and release notes draft for confirmation. NemoClaw releases are tag-based: tag `main`, let the workflow move `latest`, verify remaining open issues/PRs have been bumped, and prepare the release notes announcement for the maintainer to post. +Load `cut-release-tag`. The version is already known — default to patch bump, but still show the commit, changelog, post-tag bump plan, and release notes draft for confirmation. NemoClaw releases are tag-based: tag `main`, let the workflow move `latest`, automatically bump remaining open issues/PRs to the next patch label, and prepare the release notes announcement for the maintainer to post. ## Step 5: Confirm and Share @@ -61,4 +66,5 @@ node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer - Never cut a tag or hand off release notes without user confirmation. - If nothing was labeled or nothing shipped, ask whether to skip the tag today. -- Version labels are living markers: they always mean "ship in this version." If an item slips, the label moves forward. +- A PR version label activates release work; it is not a readiness claim. +- If an open item misses the tag, post-tag housekeeping moves its target to the next patch version. diff --git a/.agents/skills/nemoclaw-maintainer-find-review-pr/SKILL.md b/.agents/skills/nemoclaw-maintainer-find-review-pr/SKILL.md index 071b74b3091..b04c8dedb18 100644 --- a/.agents/skills/nemoclaw-maintainer-find-review-pr/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-find-review-pr/SKILL.md @@ -1,27 +1,35 @@ --- name: nemoclaw-maintainer-find-review-pr -description: Finds open GitHub PRs with security and priority-high labels, links each to its issue, detects duplicates (multiple PRs fixing the same issue), and presents a table of review candidates. Use when looking for the next PR to review. Trigger keywords - find pr, find review, next pr, pr to review, duplicate pr, security pr. +description: Finds open security-labeled GitHub PRs with Urgent or High Project Priority, links each to its issue, detects duplicates, and presents a table of review candidates. Use when looking for the next PR to review. Trigger keywords - find pr, find review, next pr, pr to review, duplicate pr, security pr. user_invocable: true --- # Find PR to Review -Search for open PRs labeled `security` + `priority: high`, associate each with its linked issue, detect duplicates (multiple PRs targeting the same issue), and present a clean summary so you can decide what to review or close. +Search for open PRs labeled `security` whose Project Priority is `Urgent` or `High`, associate each with its linked issue, detect duplicates (multiple PRs targeting the same issue), and present a clean summary so you can decide what to review or close. ## Prerequisites - `gh` (GitHub CLI) must be installed and authenticated. +- The active `gh` token must have `read:project` scope. - You must be in a GitHub repository (or the user must specify `OWNER/REPO`). ## Step 1: Fetch candidate PRs -List all open PRs that carry **both** the `security` and `priority: high` labels: +Read the NemoClaw Development Tracker and select open PRs that carry the canonical `security` label and have Project Priority `Urgent` or `High`: ```bash -gh pr list --label security --label "priority: high" --state open --limit 50 --json number,title,author,headRefName,labels,body,createdAt +gh project item-list 199 --owner NVIDIA --limit 1000 --format json \ + | jq '[.items[] + | select(.content.repository == "NVIDIA/NemoClaw") + | select(.content.type == "PullRequest") + | select(((.labels // []) | index("security")) != null) + | select(.priority == "Urgent" or .priority == "High") + | {number: .content.number, title: .content.title, url: .content.url, + priority: .priority, status: .status}]' ``` -If the result is empty, report that there are no matching PRs and stop. +Discard entries whose PR is no longer open. Fetch PR body, author, branch, labels, and creation time for the remaining numbers with `gh pr view`. If the result is empty, report that there are no matching PRs and stop. ## Step 2: Extract linked issues @@ -87,22 +95,23 @@ Present non-duplicate PRs in a table: ```markdown ### Review candidates (no duplicates) -| PR | Issue | Title | Author | Age | -|----|-------|-------|--------|-----| -| #1476 | #577 | disable remote uninstall fallback | user1 | 2d | -| #1121 | #804 | Landlock read-only /sandbox | user2 | 6d | +| PR | Issue | Priority | Title | Author | Age | +|----|-------|----------|-------|--------|-----| +| #1476 | #577 | Urgent | disable remote uninstall fallback | user1 | 2d | +| #1121 | #804 | High | Landlock read-only /sandbox | user2 | 6d | ``` ### Summary line End with a one-line recommendation of which PR to review first, preferring: -1. Older PRs (waiting longest) -2. PRs with passing checks -3. PRs with smaller diff size (easier to review) +1. Project Priority (`Urgent` before `High`) +2. Older PRs (waiting longest) +3. PRs with passing checks +4. PRs with smaller diff size (easier to review) ## Notes - Do NOT automatically close any PRs. Only present findings and recommendations. - If the user specifies additional filters (e.g., a specific scope label like `OpenShell`), apply them. -- If the user asks for a different priority label, adjust accordingly. +- If the user asks for a different priority, filter the Project Priority field. Never use or create a priority label. diff --git a/.agents/skills/nemoclaw-maintainer-morning/SKILL.md b/.agents/skills/nemoclaw-maintainer-morning/SKILL.md index 16ac63cecce..daaf4a6be86 100644 --- a/.agents/skills/nemoclaw-maintainer-morning/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-morning/SKILL.md @@ -16,7 +16,7 @@ See [PR-REVIEW-PRIORITIES.md](../nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer-day/scripts/version-target.ts ``` -This fetches tags, computes the next patch version, and finds open items still carrying older version labels. Surface stragglers first — the team needs to decide: relabel to today's target, or defer further. +This fetches tags, computes the next patch version, and finds open items still carrying older version labels. Surface stragglers first — they indicate post-tag housekeeping was interrupted or an item slipped across multiple cycles. Decide whether to relabel them to today's target or defer them out of the daily release flow. ## Step 2: Triage @@ -26,9 +26,9 @@ Run the triage script to rank the full backlog: node --experimental-strip-types --no-warnings .agents/skills/nemoclaw-maintainer-day/scripts/triage.ts --approved-only ``` -If too few results, run without `--approved-only`. The script calls `gh-pr-merge-now --json`, enriches candidates with risky-area detection, and applies the scoring model documented in [PR-REVIEW-PRIORITIES.md](../nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md). +If too few results, run without `--approved-only`. The script fetches open PRs through `gh`, reads Project 199 Priority, enriches candidates with review, CI, file, and risky-area data, and applies the scoring model documented in [PR-REVIEW-PRIORITIES.md](../nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md). -Also use `find-review-pr` to surface PRs with `security` + `priority: high` labels. Merge these into the candidate pool. +Also use `find-review-pr` to surface `security` PRs whose Project Priority is `Urgent` or `High`. Merge these into the candidate pool. ## Step 3: Label Version Targets @@ -61,5 +61,6 @@ Include: total items targeted, how many are PRs vs issues, how many are already ## Notes - This skill runs once at the start of the day. Use `/nemoclaw-maintainer-day` during the day to execute. -- The target version label is the source of truth for "what we're shipping today." +- On a PR, the target version label activates daily release work; actual release inclusion requires that PR to be merged with the label at cutoff. +- On an issue, the target version label is tracking or "needs PR" coordination only. - Stragglers from previous versions should be addressed first — they already slipped once. diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md b/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md index e5c6ffbc11d..eec60554251 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/daily-flow.md @@ -68,5 +68,5 @@ Agents may recommend labels, assignments, Project field changes, comments, merge - A PR daily version label activates daily release work; it is not a readiness claim. - Release inclusion requires a PR to be both merged and carrying the relevant daily version label at release cutoff. - Issue daily version labels are tracking or coordination signals only. -- Open PRs with daily version labels carry forward until the label is removed. +- Open PRs and issues that miss a tagged release carry forward by automatically moving from the released version label to the next patch label after the tag and `latest` are verified. - Durable release history belongs in releases, release notes, or manifests, not in long-lived labels. diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/examples.md b/.agents/skills/nemoclaw-maintainer-policies/references/examples.md index c56d5784706..d7fe2dfad6d 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/examples.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/examples.md @@ -469,8 +469,9 @@ Dry run: Anti-examples: - Do not treat `v0.0.8` as a readiness claim. -- Do not change the version label just because the day ended. -- Do not delete an old version label while an open PR still depends on it. +- Do not move stragglers before the semver tag and workflow-managed `latest` are verified. +- Do not leave an open straggler on the released label after authorized post-tag housekeeping succeeds. +- Do not bulk-edit daily version labels outside the named release automation or another explicit authorization context. ## Agent-Owned Label Examples diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/project-workflow.md b/.agents/skills/nemoclaw-maintainer-policies/references/project-workflow.md index 4c5b3fdcd71..f776011e741 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/project-workflow.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/project-workflow.md @@ -68,7 +68,7 @@ Daily `v0.0.x` labels have different meanings by item kind: - On PRs, the label activates the PR for daily release work. Merged PRs carrying the daily label are candidates for the daily release cutoff. - On issues, the label is an attention, regression-tracking, or "needs PR for this daily release" signal. It does not include the issue in the release by itself. -Open labeled PRs carry forward until the label is removed. Open issues may keep or lose daily labels according to maintainer judgment and the current daily slate. +Open labeled PRs and issues that miss a tagged release are automatically moved to the next patch label during post-tag housekeeping. Remove a version label without replacement only when the item is deferred, superseded, closed, or no longer part of the daily release cycle. ## Issue Templates diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md index 5ca51c17741..750880212ed 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md @@ -13,8 +13,8 @@ Daily release labels coordinate release work. They do not classify issues and th - Applying a daily version label is not a readiness claim. - Release includes PRs that both carry the daily version label and are merged by cutoff. - Issue version labels are tracking signals only; an issue label does not include work in the release without a merged labeled PR. -- Open PRs with version labels carry forward automatically. -- A PR leaves the cycle only when the version label is removed. +- Open PRs and issues that miss a tagged release carry forward automatically by moving from the released version label to the next patch label. +- A PR or issue leaves the daily release cycle only when its version label is removed without a replacement. - Version labels are pruned after seven days only after durable release history is preserved and no open PR still carries or depends on the old label. ## Cutoff @@ -25,19 +25,22 @@ At cutoff: 1. List merged PRs carrying the target version label. 2. Confirm each is intended for the release. -3. List open PRs still carrying daily version labels as carry-forward work. +3. List open PRs and issues still carrying the target label as post-tag stragglers. 4. Generate QA handoff from merged PRs. 5. Cut the release tag only with explicit maintainer confirmation. +6. After the tag and workflow-managed `latest` are verified, automatically move every open straggler to the next patch label. ## Carry Forward -Open PRs with version labels remain active carry-forward work by default. Do not change the version label just because the day ended. Keep the label while the PR is still intended for the daily release flow, and remove it when the PR should leave that flow. +Open PRs and issues that miss the cutoff remain active carry-forward work, but their target changes after the release succeeds. Post-tag housekeeping creates the next patch label if needed, removes the released-version label from every open straggler, and adds the next patch label. + +Run the automatic bump only after both the semver tag and workflow-managed `latest` resolve to the confirmed release commit. The release confirmation must include the housekeeping plan, so the post-tag label writes remain inside the authorized release operation. Maintainers may: - Add the current version label when they want the PR visible in the current day queue. -- Remove an old version label when the PR is deferred, superseded, closed, or no longer part of the daily cycle. -- Keep an old version label until release history and dashboards no longer need it. +- Remove a version label without replacement when an item is deferred, superseded, closed, or no longer part of the daily cycle. +- Rerun post-tag housekeeping after a partial failure; already-moved items no longer match the released source label, so the operation is safely resumable. ## Pruning @@ -45,7 +48,7 @@ Old version labels may be deleted only when all conditions are true: 1. The label is older than seven days. 2. Durable release history has been preserved in tags, release notes, Agent Feed artifacts, or equivalent reports. -3. No open PR still carries or depends on the old label. +3. No open PR or issue still carries or depends on the old label after post-tag housekeeping. 4. The current authorization context explicitly allows label pruning. Pruning is a cleanup operation, not part of ordinary daily triage. diff --git a/.agents/skills/nemoclaw-maintainer-pr-comparator/SKILL.md b/.agents/skills/nemoclaw-maintainer-pr-comparator/SKILL.md index 9e5454dfc69..74f296a3d8d 100644 --- a/.agents/skills/nemoclaw-maintainer-pr-comparator/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-pr-comparator/SKILL.md @@ -67,7 +67,7 @@ scripts/collect-gates.sh scripts/check-coderabbit-threads.sh ``` -Five gates, all mandatory. See `checks/tier-0-gates.md` for the full list and interpretation. +Six gates, all mandatory. See `checks/tier-0-gates.md` for the full list and interpretation. ### Step 5: Tier 1 correctness diff --git a/.agents/skills/nemoclaw-maintainer-pr-comparator/checks/tier-0-gates.md b/.agents/skills/nemoclaw-maintainer-pr-comparator/checks/tier-0-gates.md index 3ce1d45b439..afa062dd7a1 100644 --- a/.agents/skills/nemoclaw-maintainer-pr-comparator/checks/tier-0-gates.md +++ b/.agents/skills/nemoclaw-maintainer-pr-comparator/checks/tier-0-gates.md @@ -1,14 +1,18 @@ + + + # Tier 0 — Plumbing Gates -Mandatory prerequisites. Any gate failure means the PR cannot be merged in its current state. Five gates total. Run `scripts/collect-gates.sh ` to evaluate gates 1-4 mechanically; run `scripts/check-coderabbit-threads.sh ` for gate 5. +Mandatory prerequisites. Any gate failure means the PR cannot be merged in its current state. Six gates total. Run `scripts/collect-gates.sh ` to evaluate gates 1-5 mechanically; run `scripts/check-coderabbit-threads.sh ` for gate 6. ## Contents - Gate 1: PR state OPEN - Gate 2: CI green on latest head SHA - Gate 3: Mergeable, no conflicts -- Gate 4: Branch protection satisfied -- Gate 5: Automated reviewer threads resolved +- Gate 4: Contributor compliance satisfied +- Gate 5: Branch protection satisfied +- Gate 6: Automated reviewer threads resolved ## Gate 1: PR state OPEN @@ -36,13 +40,19 @@ The CI rollup must show all required checks passing on the **latest** head SHA, - `DIRTY` — staged changes block merge - `BLOCKED` — required checks failing or reviews missing -## Gate 4: Branch protection satisfied +## Gate 4: Contributor compliance satisfied + +The PR body must include a valid contributor `Signed-off-by:` declaration, and every commit in the PR must appear as `Verified` in GitHub. Check both conditions directly; a passing CI job is not a substitute for commit verification. + +**Why this is a hard kill:** contributor compliance is a self-serve eligibility requirement. Maintainers reject noncompliant PRs and do not amend, sign, force-push, approve, or merge them on the contributor's behalf. + +## Gate 5: Branch protection satisfied -`reviewDecision: APPROVED`, plus all branch-protection requirements (CODEOWNERS, DCO, required hooks). The skill defers to branch protection — it does NOT separately verify CODEOWNERS membership or DCO sign-off. +`reviewDecision: APPROVED`, plus all branch-protection requirements such as CODEOWNERS and required hooks. The skill may defer CODEOWNERS membership to branch protection, but Gate 4 always checks DCO and GitHub commit verification directly. **Why defer:** Branch protection rules are the source of truth. Re-implementing the check in the skill would drift from repo policy. If your repo doesn't enforce CODEOWNERS via branch protection, set `codeowners_enforced_via_branch_protection: false` in `repo-policy.md` and add explicit team checks. -## Gate 5: Automated reviewer threads resolved +## Gate 6: Automated reviewer threads resolved All threads created by automated reviewers (e.g., CodeRabbit) must be in `resolved: true` state. **Zero unresolved threads is the bar.** @@ -56,6 +66,6 @@ For each gate, the skill records: - Pass/fail - Evidence (head SHA, check names, mergeable state, thread IDs) -- Whether the failure is **trivial** (auto-fixable: rebase, push sign-off) or **substantive** (real work: CI red, conflicts, missing approvals) +- Whether the failure is **ineligible** (missing PR-body DCO or any unverified commit), **trivial** (for example, a missing issue link), or **substantive** (CI red, conflicts, or missing approvals) -The trivial/substantive split feeds degraded mode (see `tiebreakers.md`). +The ineligible/trivial/substantive classification feeds degraded mode (see `tiebreakers.md`). Ineligible PRs are rejected rather than ranked for salvage. diff --git a/.agents/skills/nemoclaw-maintainer-pr-comparator/repo-policy.md b/.agents/skills/nemoclaw-maintainer-pr-comparator/repo-policy.md index 8d5b392fb44..effaa3078cc 100644 --- a/.agents/skills/nemoclaw-maintainer-pr-comparator/repo-policy.md +++ b/.agents/skills/nemoclaw-maintainer-pr-comparator/repo-policy.md @@ -1,3 +1,6 @@ + + + # Repo Policy Configurable defaults that adapt the skill to a specific repository. Edit this file when adopting the skill in a new repo. The skill reads these values to know what to gate on. @@ -24,8 +27,8 @@ If your repo does NOT enforce CODEOWNERS via branch protection, set this to `fal ## Commit compliance policy NemoClaw default: a DCO sign-off declaration is required in the PR description, and GitHub verified commit signatures are required for every PR commit. -DCO is enforced by the `dco-check` workflow. -Verified signatures are enforced by branch protection and maintainer review. +DCO is enforced by the `dco-check` workflow and checked directly in the PR body by the comparator and merge gate. +Verified signatures are checked directly for every PR commit by the comparator and merge gate; branch protection remains a separate gate. ```yaml dco_required: true diff --git a/.agents/skills/nemoclaw-maintainer-pr-comparator/scripts/collect-gates.sh b/.agents/skills/nemoclaw-maintainer-pr-comparator/scripts/collect-gates.sh index 6d19bd210ff..75897087afd 100755 --- a/.agents/skills/nemoclaw-maintainer-pr-comparator/scripts/collect-gates.sh +++ b/.agents/skills/nemoclaw-maintainer-pr-comparator/scripts/collect-gates.sh @@ -3,8 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # # Collect Tier 0 gate state for a PR and emit JSON for downstream scoring. -# Covers gates 1-4 (state, CI on latest SHA, mergeable, branch protection). -# Gate 5 (CodeRabbit threads) is handled by check-coderabbit-threads.sh. +# Covers gates 1-5 (state, CI on latest SHA, mergeable, contributor compliance, +# branch protection). Gate 6 (CodeRabbit threads) is handled by +# check-coderabbit-threads.sh. # # Usage: collect-gates.sh [--repo OWNER/REPO] @@ -17,15 +18,32 @@ fi pr="$1" shift || true + +if [[ ! "$pr" =~ ^[0-9]+$ ]]; then + jq -n --arg pr "$pr" '{pr: $pr, error: "invalid_pr_number"}' + exit 0 +fi + +emit_error() { + jq -n --argjson pr "$pr" --arg error "$1" '{pr: $pr, error: $error}' +} + repo_args=() +repo_name="" if [ "${1:-}" = "--repo" ] && [ -n "${2:-}" ]; then repo_args=(--repo "$2") + repo_name="$2" +else + repo_name=$(gh repo view --json nameWithOwner -q .nameWithOwner 2>/dev/null) || { + emit_error "repo_resolution_failed" + exit 0 + } fi raw=$(gh pr view "$pr" "${repo_args[@]}" \ - --json number,state,headRefOid,statusCheckRollup,mergeable,mergeStateStatus,reviewDecision \ + --json number,state,body,headRefOid,statusCheckRollup,mergeable,mergeStateStatus,reviewDecision \ 2>/dev/null) || { - printf '{"pr":%s,"error":"fetch_failed"}\n' "$pr" + emit_error "fetch_failed" exit 0 } @@ -34,17 +52,82 @@ state=$(printf '%s' "$raw" | jq -r .state) gate_state_open=$([ "$state" = "OPEN" ] && echo true || echo false) # Gate 2: CI green on latest head SHA. statusCheckRollup contains the latest run. -# Count failures and pendings; gate passes only when all are SUCCESS or SKIPPED. -ci_failure_count=$(printf '%s' "$raw" | jq '[(.statusCheckRollup // [])[] | select((.conclusion // .state) == "FAILURE" or (.conclusion // .state) == "CANCELLED" or (.conclusion // .state) == "TIMED_OUT")] | length') -ci_pending_count=$(printf '%s' "$raw" | jq '[(.statusCheckRollup // [])[] | select(.status == "IN_PROGRESS" or .status == "QUEUED" or .status == "PENDING")] | length') -gate_ci_green=$([ "$ci_failure_count" = "0" ] && [ "$ci_pending_count" = "0" ] && echo true || echo false) +# Fail closed when required checks are missing, including an empty rollup. +required_checks='["checks","commit-lint","dco-check"]' +observed_checks=$(printf '%s' "$raw" | jq -c '[(.statusCheckRollup // [])[] | (.name // .context // empty)] | unique') +missing_checks=$(jq -cn --argjson required "$required_checks" --argjson observed "$observed_checks" '$required - $observed') +missing_check_count=$(printf '%s' "$missing_checks" | jq 'length') +# Keep this allowlist aligned with check-gates.ts checkCi(): every other completed +# CheckRun conclusion or terminal StatusContext state fails closed. +ci_failing_checks=$(printf '%s' "$raw" | jq -c '[ + (.statusCheckRollup // [])[] + | if .state != null then + (.state | ascii_upcase) as $state + | select($state != "SUCCESS" and $state != "PENDING" and $state != "EXPECTED") + | "\(.context // .name // "(unknown)"): \($state)" + else + (.status // "" | ascii_upcase) as $status + | (.conclusion // "" | ascii_upcase) as $conclusion + | select($status == "COMPLETED") + | select($conclusion != "SUCCESS" and $conclusion != "NEUTRAL" and $conclusion != "SKIPPED") + | "\(.name // .context // "(unknown)"): \($conclusion)" + end +]') +ci_pending_checks=$(printf '%s' "$raw" | jq -c '[ + (.statusCheckRollup // [])[] + | if .state != null then + (.state | ascii_upcase) as $state + | select($state == "PENDING" or $state == "EXPECTED" or $state == "") + | (.context // .name // "(unknown)") + else + (.status // "" | ascii_upcase) as $status + | select($status != "COMPLETED") + | (.name // .context // "(unknown)") + end +]') +ci_failure_count=$(printf '%s' "$ci_failing_checks" | jq 'length') +ci_pending_count=$(printf '%s' "$ci_pending_checks" | jq 'length') +gate_ci_green=$( + [ "$ci_failure_count" = "0" ] && [ "$ci_pending_count" = "0" ] && [ "$missing_check_count" = "0" ] && echo true || echo false +) # Gate 3: mergeable mergeable=$(printf '%s' "$raw" | jq -r .mergeable) merge_state=$(printf '%s' "$raw" | jq -r .mergeStateStatus) gate_mergeable=$([ "$mergeable" = "MERGEABLE" ] && [ "$merge_state" = "CLEAN" ] && echo true || echo false) -# Gate 4: branch protection (proxy via reviewDecision = APPROVED) +# Gate 4: contributor compliance (PR-body DCO + every commit GitHub Verified) +if printf '%s' "$raw" | jq -r '.body // ""' | grep -Eq '^Signed-off-by:[[:space:]]+.+[[:space:]]+<[^<>[:space:]]+@[^<>[:space:]]+>[[:space:]]*$'; then + dco_declaration_present=true +else + dco_declaration_present=false +fi + +commits_fetch_failed=false +commit_parse_failed=false +commits_raw=$(gh api "repos/$repo_name/pulls/$pr/commits" --paginate \ + --jq '.[] | {sha, verified: (.commit.verification.verified // false), reason: (.commit.verification.reason // "unknown")}' \ + 2>/dev/null) || commits_fetch_failed=true + +if [ "$commits_fetch_failed" = "true" ] || [ -z "$commits_raw" ]; then + commit_count=0 + unverified_commits='[]' + gate_contributor_compliance=false +elif commits_json=$(printf '%s\n' "$commits_raw" | jq -s '.' 2>/dev/null); then + commit_count=$(printf '%s' "$commits_json" | jq 'length') + unverified_commits=$(printf '%s' "$commits_json" | jq '[.[] | select(.verified != true) | {sha, reason}]') + unverified_count=$(printf '%s' "$unverified_commits" | jq 'length') + gate_contributor_compliance=$( + [ "$dco_declaration_present" = "true" ] && [ "$unverified_count" = "0" ] && echo true || echo false + ) +else + commit_count=0 + unverified_commits='[]' + commit_parse_failed=true + gate_contributor_compliance=false +fi + +# Gate 5: branch protection (proxy via reviewDecision = APPROVED) review_decision=$(printf '%s' "$raw" | jq -r .reviewDecision) gate_branch_protection=$([ "$review_decision" = "APPROVED" ] && echo true || echo false) @@ -55,30 +138,61 @@ head_sha=$(printf '%s' "$raw" | jq -r .headRefOid) # Trivial: stale base only (everything else here is substantive). classify_failures=() [ "$gate_state_open" = "false" ] && classify_failures+=("substantive:not_open") -[ "$gate_ci_green" = "false" ] && classify_failures+=("substantive:ci_failures=$ci_failure_count,pending=$ci_pending_count") +[ "$gate_ci_green" = "false" ] && classify_failures+=("substantive:ci_failures=$ci_failure_count,pending=$ci_pending_count,missing=$(printf '%s' "$missing_checks" | jq -r 'join(",")')") [ "$gate_mergeable" = "false" ] && classify_failures+=("substantive:mergeable=$mergeable,state=$merge_state") +[ "$gate_contributor_compliance" = "false" ] && classify_failures+=("ineligible:contributor_compliance") [ "$gate_branch_protection" = "false" ] && classify_failures+=("substantive:review=$review_decision") -failures_json=$(printf '%s\n' "${classify_failures[@]:-}" | grep -v '^$' | jq -Rs 'split("\n") | map(select(length > 0))' 2>/dev/null || echo '[]') - -cat < 0))') + +jq -n \ + --argjson pr "$pr" \ + --arg head_sha "$head_sha" \ + --argjson gate_state_open "$gate_state_open" \ + --argjson gate_ci_green "$gate_ci_green" \ + --argjson gate_mergeable "$gate_mergeable" \ + --argjson gate_contributor_compliance "$gate_contributor_compliance" \ + --argjson gate_branch_protection "$gate_branch_protection" \ + --arg state "$state" \ + --argjson ci_failure_count "$ci_failure_count" \ + --argjson ci_pending_count "$ci_pending_count" \ + --argjson ci_failing_checks "$ci_failing_checks" \ + --argjson ci_pending_checks "$ci_pending_checks" \ + --argjson missing_checks "$missing_checks" \ + --arg mergeable "$mergeable" \ + --arg merge_state "$merge_state" \ + --argjson dco_declaration_present "$dco_declaration_present" \ + --argjson commit_count "$commit_count" \ + --argjson unverified_commits "$unverified_commits" \ + --argjson commit_fetch_failed "$commits_fetch_failed" \ + --argjson commit_parse_failed "$commit_parse_failed" \ + --arg review_decision "$review_decision" \ + --argjson failures "$failures_json" \ + '{ + pr: $pr, + head_sha: $head_sha, + gates: { + state_open: $gate_state_open, + ci_green_latest_sha: $gate_ci_green, + mergeable: $gate_mergeable, + contributor_compliance: $gate_contributor_compliance, + branch_protection: $gate_branch_protection + }, + details: { + state: $state, + ci_failure_count: $ci_failure_count, + ci_pending_count: $ci_pending_count, + ci_failing_checks: $ci_failing_checks, + ci_pending_checks: $ci_pending_checks, + ci_missing_required_checks: $missing_checks, + mergeable: $mergeable, + merge_state_status: $merge_state, + dco_declaration_present: $dco_declaration_present, + commit_count: $commit_count, + unverified_commits: $unverified_commits, + commit_fetch_failed: $commit_fetch_failed, + commit_parse_failed: $commit_parse_failed, + review_decision: $review_decision + }, + failures: $failures + }' diff --git a/.agents/skills/nemoclaw-maintainer-pr-comparator/templates/verdict.md b/.agents/skills/nemoclaw-maintainer-pr-comparator/templates/verdict.md index 9eac016f1d5..19bb4a8969e 100644 --- a/.agents/skills/nemoclaw-maintainer-pr-comparator/templates/verdict.md +++ b/.agents/skills/nemoclaw-maintainer-pr-comparator/templates/verdict.md @@ -1,3 +1,6 @@ + + + # Verdict Template Render the final scorecard with `scripts/render-verdict.py`. Below is the human-readable shape it produces. @@ -71,18 +74,20 @@ Every judgment in the trace must include: If the verdict is **degraded mode** ("Neither mergeable yet"), substitute the verdict block: ```markdown -### Verdict: Neither mergeable yet — PR #A is closer +### Verdict: Neither mergeable yet — PR #B is closer -**PR #A — fix to merge:** +**PR #A — ineligible:** - Substantive: Rebase against current main (3 conflicts in ``) -- Trivial: Push DCO sign-off +- Ineligible: contributor gate failed; the author must fix each failing requirement before re-review + - Missing PR-body DCO declaration: update the PR body + - Missing GitHub Verified commit history: replace the branch with compliant history -**PR #B — issues to address:** +**PR #B — fix to merge:** - Substantive: 5 unresolved CodeRabbit threads at `` - Substantive: macos-e2e check failing on test "" at `` ### Suggested action -1. Coordinate with PR #A author: rebase + sign-off (~30 min) -2. After PR #A is mergeable, re-run this skill to confirm winner +1. Ask the PR #A author to fix each failing contributor gate; do not repair or approve the PR on their behalf +2. Salvage PR #B by resolving the substantive failures, then re-run this skill to confirm the winner ``` diff --git a/.agents/skills/nemoclaw-maintainer-pr-comparator/tiebreakers.md b/.agents/skills/nemoclaw-maintainer-pr-comparator/tiebreakers.md index 0a28cfb438a..9ac0847e805 100644 --- a/.agents/skills/nemoclaw-maintainer-pr-comparator/tiebreakers.md +++ b/.agents/skills/nemoclaw-maintainer-pr-comparator/tiebreakers.md @@ -1,3 +1,6 @@ + + + # Tier 3 — Ranking and Degraded Mode Final decision logic. Two paths: happy mode when at least one PR passes all Tier 0 gates, degraded mode when none do. @@ -31,17 +34,19 @@ If after all five tiebreakers no PR wins: recommend "merge A, cherry-pick releva Don't give up — pick the closest-to-ready and recommend salvage steps. 1. Classify each Tier 0 failure per PR: - - **Trivial** (auto-fixable): missing sign-off, missing issue link, stale base, force-pushed since last review + - **Trivial** (author-fixable without changing commit compliance): missing issue link, stale base, force-pushed since last review + - **Ineligible**: missing PR-body DCO declaration or any commit that is not GitHub Verified. Reject rather than salvage; the contributor must provide a clean compliant history. - **Substantive** (real work): CI red, mergeability conflicts, missing CODEOWNERS approvals, unresolved CodeRabbit threads 2. Distance-to-ready ranking: - - Fewer substantive failures wins + - Any PR with an **Ineligible** failure ranks below every eligible PR; if all candidates are ineligible, return a rejection-only verdict + - Among eligible PRs, fewer substantive failures wins - Tie → fewer trivial failures wins - Tie → higher Tier 1-2 weighted score wins (correctness beneath the broken plumbing) 3. Output: - Per-PR Tier 0 failure list - Per-PR Tier 1-2 scorecard (so the winner has objective merit beneath the gates) - Verdict: "Neither mergeable yet. PR A is closer — fix [substantive list]. PR B has [issues]." - - Salvage steps per PR (rebase command, sign-off command, CR thread links, etc.) + - Salvage steps per eligible PR (rebase command, CR thread links, etc.) ## Behavior-coverage matrix diff --git a/.agents/skills/nemoclaw-maintainer-triage/SKILL.md b/.agents/skills/nemoclaw-maintainer-triage/SKILL.md index a13f0df64ce..4fefbfccba7 100644 --- a/.agents/skills/nemoclaw-maintainer-triage/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-triage/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-triage -description: AI-assisted label triage for NVIDIA/NemoClaw issues and PRs. Reads triage-instructions.md at runtime for consistent label guidance. Supports single-item mode (give it a number) and batch mode (fetches up to 50 unlabeled open items). On approval, applies labels and an optional triage comment via gh CLI, then logs the session to the daily-rhythm activity folder. Trigger keywords - triage, label issues, suggest labels, batch triage, triage issue, triage PR, label this, what labels. +description: AI-assisted triage for NVIDIA/NemoClaw issues and PRs using native Issue Type, Project fields, and the canonical label taxonomy. Supports single-item and batch modes, presents a dry run, and applies only the accepted write set. Trigger keywords - triage, label issues, suggest labels, batch triage, triage issue, triage PR, label this, what labels. user_invocable: true --- @@ -9,133 +9,86 @@ user_invocable: true # NemoClaw Maintainer — Triage -AI-assisted label suggestion for issues and PRs. Reads live triage instructions, suggests labels and a triage comment, applies on approval, and logs the session. +Triage issues and PRs through the canonical NemoClaw workflow. Native Issue Type owns issue classification, Project fields own priority and lifecycle, and labels own routing and immediate action queues. ---- - -## Step 1: Read Triage Instructions +## Step 1: Load Canonical Policy -Before suggesting any labels, read the live instructions from [references/triage-instructions.md](references/triage-instructions.md). +Before evaluating an item, read these files in order: -Do not triage from memory. The instructions contain the label guide, tone rules, skip list, and output format. They may have been updated since your last session. +1. [workflow-policy.md](../nemoclaw-maintainer-policies/references/workflow-policy.md) +2. [triage-instructions.md](../nemoclaw-maintainer-policies/references/triage-instructions.md) +3. [label-taxonomy.json](../nemoclaw-maintainer-policies/references/label-taxonomy.json) +4. [examples.md](../nemoclaw-maintainer-policies/references/examples.md) ---- +Do not use a skill-local label guide. The policy package is the only source of truth for Issue Type, Project fields, labels, confidence, authorization, and output shape. ## Step 2: Determine Mode -**Single-item mode** — user provides a specific issue or PR number: +**Single-item mode** — the user provides an issue or PR number: ```bash -# For an issue: -gh issue view --repo NVIDIA/NemoClaw --json number,title,body,labels,url,author - -# For a PR: -gh pr view --repo NVIDIA/NemoClaw --json number,title,body,labels,url,author +gh issue view --repo NVIDIA/NemoClaw --json number,title,body,labels,url,author,projectItems +gh pr view --repo NVIDIA/NemoClaw --json number,title,body,labels,url,author,files,isDraft,mergeStateStatus,projectItems,statusCheckRollup ``` -**Batch mode** — user says "batch", "all unlabeled", or provides no number: +Use the command matching the item kind. For issues, also read the native Issue Type through the GitHub GraphQL API. For Project Priority and Status, use live Project 199 data rather than inferring state from labels. -```bash -# Fetch unlabeled open issues (no labels applied yet): -gh issue list --repo NVIDIA/NemoClaw --limit 50 --json number,title,body,labels,url,author \ - | jq '[.[] | select(.labels | length == 0)]' - -# Fetch unlabeled open PRs: -gh pr list --repo NVIDIA/NemoClaw --limit 50 --json number,title,body,labels,url,author \ - | jq '[.[] | select(.labels | length == 0)]' -``` - -In batch mode, work through items one at a time — present each suggestion and wait for approval before moving to the next. - ---- - -## Step 3: Suggest Labels and Comment - -For each item, apply the rules from `triage-instructions.md` and present: - -**Action:** `label` · **Suggested labels:** `bug`, `Platform: MacOS` -**Reason:** One sentence from the instructions. -**Triage comment (optional):** -> Comment text here. - -Ask: "Apply these labels? (yes / skip / edit labels / no comment)" - -Options: - -- **yes** — apply as shown -- **skip** — move to next item without applying -- **edit labels** — user specifies different labels, then apply -- **no comment** — apply labels only, skip posting the comment - ---- - -## Step 4: Apply on Approval - -Apply labels: - -```bash -# Issue: -gh issue edit --repo NVIDIA/NemoClaw --add-label "bug,Platform: MacOS" - -# PR: -gh pr edit --repo NVIDIA/NemoClaw --add-label "enhancement: inference" -``` - -Post comment (if approved): +**Batch mode** — collect both normal inbox items and unlabeled items: ```bash -gh issue comment --repo NVIDIA/NemoClaw --body "Comment text here." -# or for PRs: -gh pr comment --repo NVIDIA/NemoClaw --body "Comment text here." +gh issue list --repo NVIDIA/NemoClaw --state open --label "needs: triage" --limit 50 --json number,title,body,labels,url,author +gh issue list --repo NVIDIA/NemoClaw --state open --limit 50 --json number,title,body,labels,url,author +gh pr list --repo NVIDIA/NemoClaw --state open --label "needs: triage" --limit 50 --json number,title,body,labels,url,author,isDraft,mergeStateStatus +gh pr list --repo NVIDIA/NemoClaw --state open --limit 50 --json number,title,body,labels,url,author,isDraft,mergeStateStatus ``` ---- +From the unfiltered results, retain items with no labels, merge them with the `needs: triage` results, and deduplicate by item kind and number. Work through the resulting set one item at a time. -## Step 5: Log to Activity +## Step 3: Present the Dry Run -After each approved item, append to `~/development/daily-rhythm/activity/nemoclaw-triage-log.md`. +Use the JSON-compatible payload defined by canonical `triage-instructions.md`. Include: -Use the absolute path — this file lives in the daily-rhythm activity folder so it persists to GitLab over time. +- native Issue Type for issues; +- Project Priority and Status recommendations; +- only canonical labels from `label-taxonomy.json`; +- labels to remove, including a completed `needs: triage` inbox marker; +- confidence, rationale, questions, and `human_review_required`; +- the exact proposed public comment, when one is useful. -```markdown -### [ISSUE|PR] NVIDIA/NemoClaw# — -**Date:** YYYY-MM-DD -**Labels applied:** bug, Platform: MacOS -**Comment posted:** yes | no +Prefer no label over a guessed label. Never substitute labels for Issue Type, Priority, Status, or resolution. Never propose an unknown label, and never propose `PRR` during normal triage. ---- -``` +In batch mode, present each dry run and wait for an explicit `apply`, `skip`, or edited write set before moving to the next item. -Create the file if it doesn't exist, with this header: +## Step 4: Apply Only the Accepted Write Set -```markdown -# NemoClaw — Triage Log +An accepted dry run authorizes only the exact fields, labels, and comment the maintainer accepted. Resolve live Issue Type IDs, Project field IDs, and Project option IDs immediately before writing; do not hardcode mutable IDs in this skill. -A running record of label triage actions on NVIDIA/NemoClaw issues and PRs. -Persisted via daily-rhythm to GitLab. +Apply writes in this order: ---- -``` +1. Set native Issue Type and accepted Project fields. +2. Add and remove canonical labels. +3. Remove `needs: triage` when the inbox action is complete. +4. Post the exact accepted comment, if any. -At the end of a batch session, append a session summary before the individual entries: +If the accepted plan contains a low-confidence inference, an unknown label, or a write outside the current authorization context, stop and return a corrected dry run instead of writing. -```markdown -## YYYY-MM-DD — Triage Session -**Items triaged:** N -**Labels applied:** N labels across N items +## Step 5: Report ---- -``` +For every applied item, report: -Never stage or commit this file to the NemoClaw repo. +- Issue Type before and after, when applicable; +- Project Priority and Status before and after; +- labels added and removed; +- whether a comment was posted; +- any proposed write that was skipped and why. ---- +Do not write an external activity log unless the invoking maintainer explicitly asks for one. -## Response Time Note +## Batch Ordering -When triaging in batch mode, prioritize items in this order: +Prioritize candidates using policy evidence, not labels that duplicate Project Priority: -1. Items with outage, data loss, or critical breakage signals in title or body (candidate for `priority: high`) -2. Items opened by company-affiliated or known community contributors -3. Issues open > 5 business days with no label (first-response window at risk) -4. Everything else by recency +1. Security-sensitive or outage/data-loss reports that may warrant Project Priority `Urgent` or `High`. +2. Action-blocked items requiring a precise author or maintainer response. +3. Items waiting longest for an initial actionable triage decision. +4. Remaining items by recency. diff --git a/.agents/skills/nemoclaw-maintainer-triage/references/triage-instructions.md b/.agents/skills/nemoclaw-maintainer-triage/references/triage-instructions.md deleted file mode 100644 index 091c28b28ad..00000000000 --- a/.agents/skills/nemoclaw-maintainer-triage/references/triage-instructions.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -orphan: true -title: "NemoClaw Triage Instructions" -description: "AI-assisted label triage instructions for NVIDIA/NemoClaw issues and PRs. Single source of truth for the nemoclaw-maintainer-triage CLI skill and the nvoss-velocity dashboard." -keywords: triage, labels, issues, pull requests, maintainer -topics: [maintainer, triage, labels] -tags: [maintainer, triage] -content_type: reference -difficulty: advanced -audience: maintainers -status: active ---- - -<!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. --> -<!-- SPDX-License-Identifier: Apache-2.0 --> - -# NemoClaw Triage Instructions - -This document is the single source of truth for AI-assisted label triage on NVIDIA/NemoClaw issues and PRs. -It is read at runtime by the `nemoclaw-maintainer-triage` CLI skill and fetched at generation time by the nvoss-velocity dashboard. - ---- - -## Role - -You are a GitHub issue and PR labeler for NemoClaw, NVIDIA's open-source agentic AI assistant framework. - -For each item: - -1. Assign 1–5 labels from the provided list that best match the content. Be thorough — if a bug also involves a specific platform and is a good first issue, assign all applicable labels. Only skip a label if it genuinely does not apply. -2. Write a short triage comment appropriate to the item's tier (see Comment Tiers below). - ---- - -## Output Format - -Return ONLY valid JSON — no markdown fences, no explanation: - -```json -{"results": [{"number": 123, "labels": ["bug", "good first issue"], "reason": "One sentence explaining label choices.", "comment": "Comment text."}]} -``` - -Fields: - -- `number` — the issue or PR number -- `labels` — array of label names, exactly as provided in the label list -- `reason` — one concise sentence explaining why these labels apply -- `comment` — triage comment text (see Comment Tiers) - ---- - -## Label Assignment Rules - -- Use only label names exactly as provided in the label list -- Assign 1–5 labels per item — apply every label that genuinely fits -- If a specific `enhancement: *` sub-label is assigned, do NOT also assign the bare `enhancement` label — the sub-label is sufficient -- If genuinely unclear, assign `question` - ---- - -## Skip Labels - -Never assign these — they require human judgment: - -- `duplicate` -- `invalid` -- `wontfix` -- `priority: medium` -- `priority: low` -- `status: triage` -- `NV QA` - -`priority: high` is allowed ONLY when the issue clearly blocks critical functionality, causes data loss, or describes a production outage — not based on the author's frustration or urgency language alone. - ---- - -## Label Guide - -Use these descriptions to match labels to issue/PR content: - -- `bug`: User reports something broken — unexpected error, crash, exception, traceback, "not working", "fails", "broken", unexpected behavior -- `enhancement`: Generic enhancement — use only if none of the specific `enhancement: *` sub-types clearly apply -- `enhancement: feature`: Request for a new capability — "would be great if", "feature request", "add support for", "please add" -- `enhancement: inference`: Inference routing, model support, provider configuration -- `enhancement: security`: Security controls, policies, audit logging -- `enhancement: policy`: Network policy, egress rules, sandbox policy -- `enhancement: ui`: CLI UX, output formatting, terminal display -- `enhancement: platform`: Cross-platform support (pair with a `Platform: *` label) -- `enhancement: provider`: Cloud or inference provider support (pair with a `Provider: *` label) -- `enhancement: performance`: Speed, resource usage, memory, latency -- `enhancement: reliability`: Stability, error handling, recovery, retries -- `enhancement: testing`: Test coverage, CI/CD quality, test infrastructure -- `enhancement: MCP`: MCP protocol support, tool integration -- `enhancement: CI/CD`: Pipeline, build system, automation -- `enhancement: documentation`: Docs improvements, examples, guides -- `question`: Asking how to do something — "how do I", "is it possible", "does X support" -- `documentation`: Missing or incorrect docs, README errors, API doc gaps -- `good first issue`: Small well-scoped fix, doc typo, clear simple change — easy entry point for new contributors -- `help wanted`: Clear fix or improvement that needs a community contribution -- `security`: Auth issues, API key exposure, CVE, vulnerability, unauthorized access -- `status: needs-info`: Issue or PR has no description, no reproduction steps, or so little detail the team cannot act on it -- `priority: high`: Issue blocks critical functionality, causes data loss, or describes a production outage — apply only when the report clearly describes severe, reproducible impact -- `Platform: MacOS`: Issue specific to macOS, Mac OS X, or Apple Silicon (M1/M2/M3/M4). Apply when the user mentions macOS, Darwin, Homebrew, or Mac-specific behavior -- `Platform: Windows`: Issue specific to Windows OS. Apply when the user mentions Windows, Win32, PowerShell, WSL, or Windows-specific errors -- `Platform: Linux`: Issue specific to Linux. Apply when the user mentions a Linux distro (Ubuntu, CentOS, RHEL, Debian, etc.) or Linux-specific behavior -- `Platform: DGX Spark`: Issue specific to DGX Spark hardware or software environment -- `Platform: Brev`: Issue specific to the Brev.dev cloud environment -- `Platform: ARM64`: Issue specific to ARM64 / aarch64 architecture -- `Integration: Slack`: Issue or feature involving the Slack integration or Slack bridge -- `Integration: Discord`: Issue or feature involving the Discord integration -- `integration: dcode`: Issue or feature involving LangChain Deep Code, Deep Code, `langchain-deepagents-code`, or `dcode` -- `Integration: Telegram`: Issue or feature involving the Telegram integration -- `Integration: GitHub`: Issue or feature involving GitHub-specific behavior (not the repo itself) -- `Provider: NVIDIA`: Issue or feature specific to NVIDIA inference endpoints or NIM -- `Provider: OpenAI`: Issue or feature specific to OpenAI API or models -- `Provider: Anthropic`: Issue or feature specific to Anthropic / Claude models -- `Provider: Azure`: Issue or feature specific to Azure OpenAI or Azure cloud -- `Provider: AWS`: Issue or feature specific to AWS Bedrock or AWS cloud -- `Provider: GCP`: Issue or feature specific to Google Cloud / Vertex AI - ---- - -## Comment Tiers - -Items are classified as `quality_tier` or `standard_tier` before generation. This is passed in the item metadata. - -- **quality_tier** (influencer author, company-affiliated author, or body > 800 chars): Write 2–3 sentences. Start with "Thanks," then naturally reference specific details from the body. Avoid "I've taken a look at", "I've reviewed", "it appears to", "I can see that" — these sound bot-generated. Write like a human maintainer giving a warm, specific response. -- **standard_tier**: Write 1 sentence acknowledging the report and mentioning the labels applied. - ---- - -## Tone Rules (strictly enforced) - -- Use "could" not "should"; use "may" not "will" — this is a first response, not a commitment -- Never say "Thanks for fixing" — say "Thanks for the proposed fix" or "Thanks for submitting this" -- Never say "Thanks for adding" — say "Thanks for the suggested addition" -- Never claim the submission accomplishes something before review -- Do not say "I'll" or "we'll" -- For issues (bugs, questions, enhancements): use "this identifies a..." or "this reports a..." -- For PRs: use "this proposes a way to..." -- For security-related items: never confirm a vulnerability is real; use neutral language -- Do NOT open with praise about detail or thoroughness. Only reference the quality of the report if the body is genuinely exceptional — multiple reproduction steps, version info, logs, and clear expected vs actual behavior. For most reports, skip the praise entirely and go straight to the triage acknowledgment. -- Do not add generic closing filler phrases -- If a "Spam signal:" line is present in the item metadata, assign only `status: needs-info` and ask for more detail politely -- If a "Note: Author also opened..." line is present, briefly acknowledge if the relationship is plausible diff --git a/.agents/skills/nemoclaw-maintainer-verify-stale/SKILL.md b/.agents/skills/nemoclaw-maintainer-verify-stale/SKILL.md index 1712aa2203e..14153a7e959 100644 --- a/.agents/skills/nemoclaw-maintainer-verify-stale/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-verify-stale/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-verify-stale -description: "Verifies whether stale NVIDIA/NemoClaw bug reports still reproduce on the latest tag. Use when maintainers ask to verify stale issues, reproduce old bugs on latest, drain the bug backlog, or apply fixed-on-latest, verify-inconclusive, or status: wont-fix. Runs candidate filtering, local/Brev reproduction, by-design detection, confidence scoring, redacted comments, and tag-only labeling; never auto-closes." +description: "Verifies whether stale NVIDIA/NemoClaw bug reports still reproduce on the latest tag. Use when maintainers ask to verify stale issues, reproduce old bugs on latest, or drain the bug backlog. Uses native Issue Type, canonical labels, local/Brev reproduction, by-design detection, confidence scoring, approved Project updates, and redacted verdict comments; never auto-closes." user_invocable: true --- @@ -9,7 +9,7 @@ user_invocable: true # NemoClaw Maintainer — Verify Stale Issues -Automates the maintainer loop: choose an old bug, verify whether it still reproduces on the latest NemoClaw tag, then post an evidence-backed comment and label. It is tag-only: never close issues automatically. +Automates the maintainer loop: choose an old issue whose native Issue Type is `Bug`, verify whether it still reproduces on the latest NemoClaw tag, then prepare an evidence-backed Project/comment write set for maintainer approval. It never closes issues automatically and never substitutes labels for Issue Type, lifecycle, or resolution. ## Progress checklist @@ -26,23 +26,24 @@ Verify-stale progress: - [ ] Validate the reproducer on baseline, then verify latest - [ ] Check by-design/static-analysis branch when behavior was removed - [ ] Score, redact, draft, and self-verify comment links -- [ ] Re-check issue state, post comment/label, update tracker when required +- [ ] Re-check issue state, apply the accepted Project/comment write set - [ ] Append activity log entry ``` ## Workflow 1. **Select candidates and versions.** Read [reference/candidate-selection.md](reference/candidate-selection.md). Use it for single-issue mode, batch mode, latest-tag detection, filters, idempotency, active-discussion handling, and reported-version parsing. -2. **Classify and prepare.** Read [reference/environment-and-reproducer.md](reference/environment-and-reproducer.md). Use it for CPU/GPU/provider/bug-class classification, safe API-key handling, reproducer extraction, dependency checks, Brev auth, label checks, and local-first verification. +2. **Classify and prepare.** Read [reference/environment-and-reproducer.md](reference/environment-and-reproducer.md). Use it for CPU/GPU/provider/bug-class classification, safe API-key handling, reproducer extraction, dependency checks, Brev auth, and local-first verification. 3. **Stop for approval before cost.** In batch mode, present one issue's plan and wait for maintainer approval before provisioning Brev. 4. **Provision and install.** If local-first does not settle the issue, read [reference/brev-provisioning.md](reference/brev-provisioning.md). Use it for Brev reuse/provisioning, reset, baseline/latest installs, dependency bootstrap, and `brev exec` footguns. 5. **Run the verification rubric.** Read [reference/reproduction-rubrics.md](reference/reproduction-rubrics.md). Use it to validate baseline behavior, retry with a synthesized reproducer if needed, run latest, handle architectural drift, and branch for performance or rebuild-cycle bugs. -6. **Check intentional changes.** If the symptom targets removed/deprecated behavior, read [reference/by-design.md](reference/by-design.md). Use static evidence to apply `status: wont-fix` only when the by-design branch self-verifies. -7. **Score, comment, label, and log.** Read [reference/scoring-comments-and-logging.md](reference/scoring-comments-and-logging.md). Use it for confidence scoring, redaction, concise templates, issue-state race checks, Project 199 movement, infra failures, and activity logging. +6. **Check intentional changes.** If the symptom targets removed/deprecated behavior, read [reference/by-design.md](reference/by-design.md). Use static evidence to recommend Project Status `Won't Fix`, then request explicit approval for the exact Project/comment write set. +7. **Score, propose, apply, and log.** Read [reference/scoring-comments-and-logging.md](reference/scoring-comments-and-logging.md). Use it for confidence scoring, redaction, concise templates, authorization, issue-state race checks, approved Project 199 movement, infra failures, and activity logging. ## Non-negotiables -- Never auto-close an issue. Apply labels and ask a maintainer/reporter to confirm. +- Never auto-close an issue. Verdict names belong in comments and logs, not labels. +- Never write a Project field, assignment, or public comment before the maintainer accepts that exact write set. - Never put API keys on a command line. Use the file-based pattern in `environment-and-reproducer.md`. - Never post unredacted transcripts, issue excerpts, synthesized scripts, internal hostnames, email addresses, or tokens. - Never post a comment with broken markdown links or tag-drifting `file:line` citations. Re-run cited commands and link-check at least one rendered link per comment section. @@ -57,5 +58,5 @@ Verify-stale progress: | Environment classification, credentials, reproducer, preconditions, local-first | [reference/environment-and-reproducer.md](reference/environment-and-reproducer.md) | | Brev box reuse/provision, reset, installs, dependency bootstrap | [reference/brev-provisioning.md](reference/brev-provisioning.md) | | Baseline/latest matching, synth-repro, drift, performance, rebuild-cycle | [reference/reproduction-rubrics.md](reference/reproduction-rubrics.md) | -| Static by-design/wont-fix branch | [reference/by-design.md](reference/by-design.md) | -| Score, redact, comment, label, tracker move, infra handling, log | [reference/scoring-comments-and-logging.md](reference/scoring-comments-and-logging.md) | +| Static by-design branch and proposed `Won't Fix` Project state | [reference/by-design.md](reference/by-design.md) | +| Score, redact, authorize, comment, Project update, infra handling, log | [reference/scoring-comments-and-logging.md](reference/scoring-comments-and-logging.md) | diff --git a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/brev-provisioning.md b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/brev-provisioning.md index c30a0af4e56..bb5c159c692 100644 --- a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/brev-provisioning.md +++ b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/brev-provisioning.md @@ -235,7 +235,7 @@ Bootstrap **once before Step 8b's baseline run** and reuse for Step 8d's latest | Sandbox structure (build, file permissions, mounts, layout) | ✓ Always | Provider not exercised | | Networking / policy (port forwards, NAT, egress rules, channels guards) | ✓ Always | Provider not exercised | | Generic inference flow (does an agent turn complete, does the proxy route correctly) | ✓ Usually | Ollama can fail in the same shape as NIM/Gemini for most flow bugs | -| Provider-specific behavior (`Provider: NVIDIA` symptom, NIM-only error handling, `Provider: Gemini` quirks) | ✗ No | Different code paths; substitution doesn't exercise the bug | +| Provider-specific behavior (`provider: nvidia` symptom, NIM-only error handling, Gemini-specific quirks) | ✗ No | Different code paths; substitution doesn't exercise the bug | | Model-specific behavior (`gemini-flash-3-preview` doesn't handle prompt X, `nemotron-3-nano:4b` works fine) | ✗ No | Wrong model = wrong outputs | | Ollama-shape-specific (#2519 "Ollama-local 401" — local-vs-networked Ollama config) | △ Sometimes | A generic Ollama install may or may not reproduce; may need specific configuration | | Performance / latency on specific silicon | ✗ No | Hardware substitution caveat (Step 10) and Step 8e perf rubric apply | diff --git a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/by-design.md b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/by-design.md index 677567cb2df..2026f6679de 100644 --- a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/by-design.md +++ b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/by-design.md @@ -3,7 +3,7 @@ # verify-stale — By-Design Detection Reference -Use whenever the reproducer points at removed, intentionally changed, or deprecated behavior. This branch can short-circuit Brev cost and label `status: wont-fix`, but every claim needs verifiable evidence. +Use whenever the reproducer points at removed, intentionally changed, or deprecated behavior. This branch can short-circuit Brev cost and recommend Project Status `Won't Fix`, but every claim needs verifiable evidence and the state change requires explicit maintainer approval. ## Contents @@ -18,7 +18,7 @@ Use whenever the reproducer points at removed, intentionally changed, or depreca ## Step 8.5: Detect "Behavior Changed by Design" -Before scoring, check whether the symptom is intentional. Some bugs are filed against behavior that was **deliberately changed or removed** in a merged PR — running the standard rubric on these produces misleading verdicts. The symptom "still reproduces" but the right answer is "won't fix, see PR #X." Issue #2791 is the prototype: `config set` was removed in PR #2227, the reporter tested a version that already had it gone, and a standard rubric run would have buried that context under a low-confidence `verify-inconclusive` label. +Before scoring, check whether the symptom is intentional. Some bugs are filed against behavior that was **deliberately changed or removed** in a merged PR — running the standard rubric on these produces misleading verdicts. The symptom "still reproduces" but the right answer is "won't fix, see PR #X." Issue #2791 is the prototype: `config set` was removed in PR #2227, the reporter tested a version that already had it gone, and a standard rubric run would have buried that context under a low-confidence `verify-inconclusive` verdict. This step is split into substeps so the rigor is mechanical, not optional. Every claim in the final comment must be backed by a verifiable evidence block — a comment URL with quoted phrase, a commit SHA with diff range, or a grep command with its actual output. Hand-wavy claims fail Step 8.5d's self-verification pass and force a bail to `verify-inconclusive`. @@ -103,10 +103,12 @@ The cost of an incorrect "I checked and X is gone" claim in a public comment, or - **Skip the Step 9 score table** entirely. The "exit 0 + expected output" axis doesn't apply when the expected output is no longer the contract. - **Skip Brev provisioning** if the signal fires before Step 7 — a remote run would just confirm what static analysis already proved. (Signals 2 and 3 can run as soon as the reported version is parsed in Step 4.) -- **Apply label `status: wont-fix`** (the existing repo label — quote it on the CLI: `gh issue edit <num> --add-label "status: wont-fix"`). It's already in the Step 3 issue-type skip list, so a labelled issue is automatically excluded from future runs without needing a separate idempotency clause. +- **Prepare a dry run** containing Project Status `Won't Fix`, the exact public comment, the durable `by-design` marker, and `human_review_required: true`. +- **Request explicit maintainer approval** for that exact write set. Do not substitute a status label or write before approval. +- **On approval, update Project Status first, then post the accepted comment.** If approval is withheld, report the evidence without mutating GitHub. - **Use the by-design comment template below** instead of the standard Step 10 template. - **@-mention the reporter** so they can object if the framing is wrong. -- **Never auto-close.** A maintainer pulls the trigger, same as the other label paths. +- **Never auto-close.** A maintainer separately decides whether to close after reviewing the evidence and reporter response. ### By-design comment template @@ -164,7 +166,7 @@ The new workflow is `<one-sentence: how to do what the user was trying to do>`. `<NVBugs cross-ref line — see below>` -<!-- nemoclaw-verify-stale v1 YYYY-MM-DD --> +<!-- nemoclaw-verify-stale v1 verdict=by-design YYYY-MM-DD --> ```` **NVBugs cross-ref line.** If `NVBUGS_REF` was set in Step 4, append: diff --git a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/candidate-selection.md b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/candidate-selection.md index a62baf96b38..4b042b54181 100644 --- a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/candidate-selection.md +++ b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/candidate-selection.md @@ -20,17 +20,63 @@ Use after loading `SKILL.md` to choose an issue and establish its reported NemoC ```bash gh issue view <number> --repo NVIDIA/NemoClaw \ - --json number,title,body,labels,url,author,createdAt,comments + --json number,title,body,labels,url,author,createdAt ``` +Also fetch the native Issue Type and current Project 199 fields with GraphQL. Resolve fields by their live names rather than hardcoding mutable IDs: + +```bash +gh api graphql -F number=<number> -f query='query($number: Int!) { + repository(owner: "NVIDIA", name: "NemoClaw") { + issue(number: $number) { + issueType { name } + projectItems(first: 20) { + nodes { + project { number } + fieldValues(first: 50) { + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { + name + field { ... on ProjectV2SingleSelectField { name } } + } + } + } + } + } + } + } +}' +``` + +A candidate must have native Issue Type `Bug`; labels never substitute for this check. From the Project 199 item, read the `Priority` and `Status` single-select values. + **Batch mode** — user says "batch", "weekly", or provides no number. Cap at **15 issues** for *processing* per run, enforced as a slice after Step 3/4 filters narrow the pool. The cap exists because batch is sequential (Step 7 reuse-or-provision keeps it on 1–2 Brev boxes total) and the wallclock budget is ~2–3 hours per 15-issue run; running larger forces the maintainer to either drop the per-plan approval gate or spread the batch across multiple sessions. -The discovery query needs to see the entire open-bug pool — the per-run processing cap is downstream. Use `--limit 1000` so the skill doesn't silently drop issues beyond the page (the candidate triage run found 129 open bugs; an earlier `--limit 100` would have missed 29 of them). +The discovery query needs to see the entire open-issue pool because native Issue Type, not a `bug` label, identifies bug reports. Use paginated GraphQL and retain only nodes whose `issueType.name` is `Bug`: ```bash -gh issue list --repo NVIDIA/NemoClaw --state open --limit 1000 \ - --label bug \ - --json number,title,body,labels,url,author,createdAt,comments +gh api graphql --paginate -f query='query($endCursor: String) { + repository(owner: "NVIDIA", name: "NemoClaw") { + issues(first: 100, after: $endCursor, states: OPEN) { + nodes { + number title body url createdAt + author { login } + issueType { name } + labels(first: 100) { nodes { name } } + } + pageInfo { hasNextPage endCursor } + } + } +}' --jq '.data.repository.issues.nodes[] | select(.issueType.name == "Bug")' +``` + +Read Project Priority and Status from live Project 199 data. Do not infer either field from labels. + +Before applying the idempotency or active-discussion filters to each candidate, fetch its complete comment history through the paginated REST endpoint. Nested GraphQL comment connections are not paginated by the outer issue query and can silently truncate old markers: + +```bash +COMMENTS=$(gh api "repos/NVIDIA/NemoClaw/issues/$ISSUE_NUMBER/comments?per_page=100" \ + --paginate --jq '.[]' | jq -s '.') ``` In batch mode, work through items one at a time. Present each verification plan and wait for approval before any Brev provisioning. @@ -61,46 +107,66 @@ This is the version the skill will verify against. Record it — every comment m Apply these rules in order. Drop any issue that fails a rule. -**Issue-type allowlist:** must have `bug` label. -**Issue-type skip:** drop if any label exactly matches `documentation`, `status: wont-fix`, `status: needs-info`, `security`, OR is `enhancement` / starts with the prefix `enhancement:` (the repo carries several prefixed variants — `enhancement: feature`, `enhancement: MCP`, `enhancement: testing`, `enhancement: ui`, `enhancement: provider`, `enhancement: platform`, `enhancement: policy`, `enhancement: inference`, `enhancement: integration`, `enhancement: performance`, `enhancement: skill` — and exact-match misses them all; surfaced from #1752; enumerate via `gh label list -L 200 --search enhancement` if you need the live set). Use the canonical repo label names — bare `wontfix` / `needs-info` are NOT the repo's labels (verified via `gh label list`); the actual labels carry a `status:` prefix and a hyphen. +**Issue-type allowlist:** native Issue Type must be `Bug`. + +**Project Status skip:** drop items already in `Done`, `Won't Fix`, or `Duplicate`. These are Project Status values, not labels. -**Platform skip (Brev-reproducible only in v1):** drop if any of `Platform: Windows/WSL`, `Platform: MacOS`, `Platform: macOS`, `Platform: Jetson AGX Thor/Orin`. Brev has no equivalent hardware for Jetson (embedded/edge ARM with integrated GPU is not in the Brev SKU catalog), so any Brev verification of a Jetson-only bug would produce a misleading "fixed-on-x86" verdict. Keep `Platform: Ubuntu`, `Platform: DGX Spark`, `Platform: GB10`, `Platform: All`, or no platform label. `Platform: DGX Spark` and `Platform: GB10` stay in scope but Step 10 requires a "Hardware substitution" caveat in the comment naming the Brev SKU we used as a substitute (Brev x86 GPU SKUs are not faithful to GB10 / Grace Hopper silicon for performance-shape or memory-architecture-shape bugs). +**Security skip:** drop items carrying the canonical `security` label. Potential vulnerability reports require the dedicated security workflow and neutral handling, not public stale-verification commentary. + +**Platform skip (Brev-reproducible only in v1):** drop `platform: windows`, `platform: wsl`, `platform: macos`, and `platform: jetson`. Brev has no equivalent hardware for those targets, so verification would produce a misleading cross-platform verdict. Keep `platform: ubuntu`, `platform: dgx-spark`, `platform: gb10`, or no platform label. DGX Spark and GB10 remain in scope only with the Step 10 hardware-substitution caveat. **TUI / interactive-UI skip:** drop if the issue title contains `TUI`, `dashboard UI`, `chat UI`, `keystroke`, or `key press`, OR if the body describes interactive UI behavior (key sequences, mouse interactions, browser-side UI state) without a non-interactive reproducer (no `NEMOCLAW_NON_INTERACTIVE=1` or equivalent env var pattern). `brev exec` does not allocate a real TTY by default, so TUI reproducers hang or silently fail at the first prompt; v1 documents this as out-of-scope rather than emitting a wrong verdict. v1.1 may add a `script(1)` / `expect` / `tmux send-keys` harness to lift this skip. -**Integration skip (deferred to v2):** drop if any of `Integration: Slack`, `Integration: Discord`, `Integration: Telegram`, `Integration: Hermes`, `Integration: OpenClaw`, `Integration: WeChat`. These need third-party credentials a fresh Brev box cannot provide. +**Integration skip (deferred to v2):** drop `integration: slack`, `integration: discord`, `integration: telegram`, `integration: hermes`, `integration: openclaw`, and `integration: wechat`. These need third-party credentials a fresh Brev box cannot provide. -**Component allowlist (must have at least one):** `NemoClaw CLI`, `Sandbox`, `OpenShell`, `Docker`, `Getting Started`, or any `Platform:` label that survived the platform skip. +Do not require retired component labels. Native Issue Type, version evidence, canonical routing labels, and reproducer suitability determine eligibility. **Idempotency:** drop if **either** of these is true: -- The issue carries a `fixed-on-latest` or `verify-inconclusive` label. These labels are persistent; rerun verification only when a maintainer explicitly targets the issue or removes the label. The by-design path uses the existing repo `status: wont-fix` label, which is already covered by the issue-type skip rule above — no separate idempotency clause needed for that path. -- A comment matching `<!-- nemoclaw-verify-stale v\d+ YYYY-MM-DD -->` was posted **within the last 7 days**. The regex matches any marker version (`v1`, `v2`, …) so future skill versions can re-verify older-marked issues by tightening the regex (e.g. require a specific marker version). The marker carries a date so the candidate filter can apply a TTL — useful for the still-reproduces case (Step 9), where no label is applied and we want next week's run to re-verify rather than skip forever. +- Any comment contains a final marker for `fixed-on-latest`, `verify-inconclusive`, or `by-design`. These markers are durable; rerun only when a maintainer explicitly targets the issue. +- A `still-reproduces` marker was posted within the last seven days. Its TTL allows a later weekly run to catch a newly landed fix. + +Use markers shaped like: -Implementation — match the marker against each comment's `createdAt`. Use `gh issue view --json comments` (single-issue mode already fetches this; batch mode's `gh issue list` also returns the comment array per issue): +```text +<!-- nemoclaw-verify-stale v1 verdict=fixed-on-latest YYYY-MM-DD --> +<!-- nemoclaw-verify-stale v1 verdict=verify-inconclusive YYYY-MM-DD --> +<!-- nemoclaw-verify-stale v1 verdict=by-design YYYY-MM-DD --> +<!-- nemoclaw-verify-stale v1 verdict=still-reproduces YYYY-MM-DD --> +``` + +Implementation — match markers against each comment's body and creation time: ```bash # Cutoff for the 7-day TTL. macOS and Linux date(1) syntax differ; try both. SEVEN_DAYS_AGO=$(date -u -v-7d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null \ || date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ) -# Returns the timestamp of the most recent marker comment within the TTL, or empty. -RECENT_MARKER=$(gh issue view "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --json comments \ - --jq --arg cutoff "$SEVEN_DAYS_AGO" ' - .comments[] - | select(.body | test("<!-- nemoclaw-verify-stale v\\d+ \\d{4}-\\d{2}-\\d{2} -->")) - | select(.createdAt > $cutoff) - | .createdAt' \ +# Final markers do not expire. +FINAL_MARKER=$(printf '%s' "$COMMENTS" \ + | jq -r ' + .[] + | select(.body | test("<!-- nemoclaw-verify-stale v\\d+ verdict=(fixed-on-latest|verify-inconclusive|by-design) \\d{4}-\\d{2}-\\d{2} -->")) + | .created_at' \ + | head -1) + +# still-reproduces markers expire after seven days. +RECENT_STILL_REPRODUCES=$(printf '%s' "$COMMENTS" \ + | jq -r --arg cutoff "$SEVEN_DAYS_AGO" ' + .[] + | select(.body | test("<!-- nemoclaw-verify-stale v\\d+ verdict=still-reproduces \\d{4}-\\d{2}-\\d{2} -->")) + | select(.created_at > $cutoff) + | .created_at' \ | head -1) -if [ -n "$RECENT_MARKER" ]; then - echo "Skip: marker posted $RECENT_MARKER (within 7-day TTL)" +if [ -n "$FINAL_MARKER" ] || [ -n "$RECENT_STILL_REPRODUCES" ]; then + echo "Skip: prior final verdict or recent still-reproduces verification found" # In single-issue mode: exit 0 with a friendly message. # In batch mode: continue to the next candidate. fi ``` -Run this check for every candidate that survived the label-based filters above; drop those whose `RECENT_MARKER` is non-empty. +Run this check for every candidate that survived the canonical field and label filters above. **Unanswered-maintainer-question handling.** Find the most recent maintainer (`MEMBER`, `OWNER`, `COLLABORATOR`) comment that **looks like a question** (`?`, polite imperative like "please confirm/share/clarify", or starter like "could you / can you / do you") AND that the reporter has not replied to since. Pure triage acknowledgments (`"✨ Thanks for reporting…"`) are skipped. The age of the qualifying comment determines skip-or-proceed: @@ -114,26 +180,26 @@ REPORTER=$(gh issue view "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --json author -- # Question-detection patterns are chained as separate test() calls so each # heuristic is independently readable and a future addition (e.g. "what about", # "why does") is a one-line append rather than a regex-alternation patch. -UNANSWERED_MAINT=$(gh issue view "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --json comments \ - --jq --arg reporter "$REPORTER" --arg cutoff "$SEVEN_DAYS_AGO" ' - (.comments - | map(select((.authorAssociation == "MEMBER" or .authorAssociation == "OWNER" or .authorAssociation == "COLLABORATOR") +UNANSWERED_MAINT=$(printf '%s' "$COMMENTS" \ + | jq --arg reporter "$REPORTER" --arg cutoff "$SEVEN_DAYS_AGO" ' + (. + | map(select((.author_association == "MEMBER" or .author_association == "OWNER" or .author_association == "COLLABORATOR") and (.body | test("\\?") # literal "?" or test("(?i)\\bplease (confirm|share|provide|clarify|tell|verify|check|let me know|let us know)") # polite imperative or test("(?i)\\b(could|can|would) you\\b") # modal interrogative or test("(?i)\\bdo you (have|know|see|use)\\b")))) # "do you ..." - | sort_by(.createdAt) | last) as $maint + | sort_by(.created_at) | last) as $maint | if $maint == null then null else - ((.comments - | map(select(.author.login == $reporter and .createdAt > $maint.createdAt)) + ((. + | map(select(.user.login == $reporter and .created_at > $maint.created_at)) | length) as $replies | if $replies > 0 then null else { - createdAt: $maint.createdAt, - url: $maint.url, - login: $maint.author.login, - recent: ($maint.createdAt > $cutoff) + createdAt: $maint.created_at, + url: $maint.html_url, + login: $maint.user.login, + recent: ($maint.created_at > $cutoff) } end) end') @@ -206,7 +272,7 @@ If no version survives, drop the issue from the candidate set — we cannot esta **Variable format for downstream steps.** Set `REPORTED_VERSION` to the **full tag string** (e.g., `REPORTED_VERSION="v0.0.32"`), not just the patch number. Step 8a's installer expects the full tag via the `NEMOCLAW_INSTALL_TAG` env var. -**Batch cap enforcement.** In batch mode, after Step 3 label filters and the Step 4 version+candidate-rule filters narrow the pool, sort surviving candidates by `(-versions_behind, -age_days)` so the most stale come first, then **slice to the top 15**: +**Batch cap enforcement.** In batch mode, after Step 3 field/label filters and the Step 4 version+candidate-rule filters narrow the pool, sort surviving candidates by `(-versions_behind, -age_days)` so the most stale come first, then **slice to the top 15**: ```bash # Each candidate has at minimum: number, reported, behind, age_days diff --git a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/environment-and-reproducer.md b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/environment-and-reproducer.md index 07299a3ffc9..c05873488e0 100644 --- a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/environment-and-reproducer.md +++ b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/environment-and-reproducer.md @@ -18,7 +18,7 @@ Use after a candidate passes selection. Classify the environment, prompt for pro **CPU vs GPU:** GPU if any of these signals are present, else CPU. -- Labels: `Platform: GB10`, `Platform: DGX Spark`. +- Labels: `platform: gb10`, `platform: dgx-spark`. - Body keywords (whole-word, case-insensitive): `nvidia-smi`, `cuda`, `H100`, `A100`, `L40S`, `L4`, `T4`, `GB10`, `DGX`, `vllm`, `tensorrt`. Match as whole words — `inference` and `model serving` are too noisy (e.g. `models.providers.inference.baseUrl` is a config path on CPU bugs, not a GPU need) and intentionally excluded. CPU default keeps cost low. Only escalate to GPU when the reproducer needs one. @@ -38,10 +38,10 @@ Most bugs are `functional`. The other three classes need verification harnesses | Detection signal | Provider | |---|---| -| `Provider: NVIDIA` label, body mentions `NVIDIA NIM`, `build.nvidia.com`, `nvapi-...`, `NVIDIA_API_KEY`, or `NEMOCLAW_PROVIDER=build` | `nim` | -| `Provider: Gemini` label, body mentions `Gemini`, `gemini-flash`, `gemini-pro`, `GEMINI_API_KEY` | `gemini` | -| `Provider: Anthropic` / `Provider: AWS` (Bedrock) labels or matching keywords | `anthropic`/`bedrock` | -| `Provider: Ollama`, body mentions `ollama` or `NEMOCLAW_PROVIDER=ollama`, or no provider mentioned at all | `ollama` (default) | +| `provider: nvidia` label, body mentions `NVIDIA NIM`, `build.nvidia.com`, `nvapi-...`, `NVIDIA_API_KEY`, or `NEMOCLAW_PROVIDER=build` | `nim` | +| Body mentions `Gemini`, `gemini-flash`, `gemini-pro`, or `GEMINI_API_KEY` | `gemini` | +| `provider: anthropic` label or body keywords for Anthropic or AWS Bedrock | `anthropic`/`bedrock` | +| `provider: ollama`, body mentions `ollama` or `NEMOCLAW_PROVIDER=ollama`, or no provider is mentioned | `ollama` (default) | Set `BUG_PROVIDER=<provider>`. @@ -74,8 +74,9 @@ to verify faithfully. Three options: verdict will be capped because we're not exercising the real provider's code path. - 3. Skip this issue. Mark `verify-inconclusive` with the reason "requires <provider> - API key — not provided in this run." + 3. Skip this issue. Select the `verify-inconclusive` verdict with the reason + "requires <provider> API key — not provided in this run," then request + approval for the proposed comment and durable verdict marker. Choose 1, 2, or 3: ``` @@ -177,8 +178,11 @@ if [ -z "$GH_IDENTITY" ]; then fi echo "gh identity: @$GH_IDENTITY — comments posted by this run will appear under this handle" -# gh 'project' scope — Step 10 moves fixed-on-latest issues to "Needs Review" on Project 199. Warn if missing. -gh auth status 2>&1 | grep -q "'project'" || echo "[verify-stale] WARN gh missing 'project' scope — Step 10 tracker move will skip. Fix: run 'gh auth refresh -h github.com -s project' in a real terminal." +# gh 'project' scope — candidate selection reads Project fields and Step 10 may update them. +gh auth status 2>&1 | grep -q "'project'" || { + echo "ERROR: gh is missing 'project' scope. Run 'gh auth refresh -h github.com -s project' in a real terminal, then re-run this skill." + exit 1 +} # Brev auth — short-circuit only after the auth check, not before. # When auth fails, give the user a directive recipe (the browser-flow path is @@ -205,19 +209,6 @@ MSG exit 1 } -# Repo labels exist — Step 8.5 / Step 10 can't apply a label that doesn't exist. Check -# canonical label names against the live repo so a mismatch fails fast (issue #2168 hit this: -# spec called the label `wontfix`, but the actual repo label is `status: wont-fix`). -EXPECTED_LABELS=("fixed-on-latest" "verify-inconclusive" "status: wont-fix") -LIVE_LABELS=$(gh label list --repo NVIDIA/NemoClaw --limit 200 --json name --jq '.[].name') -for label in "${EXPECTED_LABELS[@]}"; do - printf '%s\n' "$LIVE_LABELS" | grep -Fxq "$label" || { - echo "ERROR: expected label not on repo: '$label'" - echo " create it with: gh label create '$label' --repo NVIDIA/NemoClaw" - exit 1 - } -done - # Install URL reachable — fails fast instead of mid-Brev-run if the host is down or the URL changed. # The default is the public Akamai-hosted entry (301-redirects to the actual installer). The # `nemoclaw.nvidia.com` host that earlier drafts pointed to is NVIDIA-internal and does not @@ -241,7 +232,7 @@ For pure-CLI reproducers (no sandbox state, no GPU, no integration tokens), try **Predicate** — local-first applies if **all** of these hold: - Reproducer is a sequence of `nemoclaw <args>` invocations only. No `docker`, `kubectl`, `curl`, `npm`, networking setup, or filesystem fixtures. -- Issue has no `Sandbox`-only or `Docker` label and no GPU signal from Step 5. +- Issue has no `area: sandbox` or `platform: container` label and no GPU signal from Step 5. - `which nemoclaw` resolves on the maintainer's machine and `nemoclaw --version` reports a build at or past `$LATEST` (a build between `$LATEST` and `$LATEST+main` is fine — these only differ by unmerged WIP). - Maintainer is on Linux or macOS. Windows local repros are out of scope (per Step 3 platform skip rules). diff --git a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/reproduction-rubrics.md b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/reproduction-rubrics.md index cf64ff6f0e0..f15cb70f670 100644 --- a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/reproduction-rubrics.md +++ b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/reproduction-rubrics.md @@ -89,7 +89,7 @@ brev exec "$INSTANCE_NAME" 'export PATH="$HOME/.local/bin:$PATH" && bash ~/repro ``` - **Match:** validated (with −30 baked in). Proceed to 8d. -- **Still no match:** mark `verify-inconclusive`. Post a comment that includes both reproducer attempts and both baseline transcripts with the message "couldn't establish a working reproducer for this bug on `$REPORTED_VERSION`." **Skip 8d** — there's nothing to verify on latest. +- **Still no match:** select the `verify-inconclusive` verdict. Prepare a comment that includes both reproducer attempts and both baseline transcripts with the message "couldn't establish a working reproducer for this bug on `$REPORTED_VERSION`." **Skip 8d** — there's nothing to verify on latest. ### Step 8d: Install latest, run validated reproducer @@ -146,7 +146,7 @@ brev copy ./reproducer.sh "$INSTANCE_NAME":~/reproducer.sh brev exec "$INSTANCE_NAME" 'export PATH="$HOME/.local/bin:$PATH" && bash ~/reproducer.sh' 2>&1 | tee ./latest-transcript.log ``` -If the install of **latest** fails (e.g. installer regression — see #3058 for a current example), this is an infra failure — see Step 11. Do not score or label the issue. +If the install of **latest** fails (e.g. installer regression — see #3058 for a current example), this is an infra failure — see Step 11. Do not score the issue or mutate its labels or Project fields. If install succeeds, `latest-transcript.log` is the input to Step 9 scoring. @@ -237,7 +237,7 @@ Performance bugs (#2598 "10s P50", #2600 "hangs ~2 min", #2733 Ollama tool-call - Latest's p50 within `$SLA_P50_MS` AND baseline's p50 also within → reproducer doesn't actually exercise the bug; route to Step 8c synth-repro. - **p90 backstop**: if `$SLA_P90_MS` was parsed from the issue, latest's p90 outside `$SLA_P90_MS` flips a within-SLA-p50 verdict to `still-reproduces` — tail-latency regressions matter for the issues that name them. -**Hardware-substitution caveat.** Performance numbers are silicon-dependent. When the issue is `Platform: DGX Spark` or `Platform: GB10` and we're measuring on a Brev x86 GPU SKU, the comment must say so explicitly: a Brev p50 of 1.5s on a `H100` does not prove the DGX Spark p50 is fixed. Cap the score at 60 unless the bug is clearly silicon-independent (e.g. an algorithmic regression in user-space JS that would manifest the same on any silicon). +**Hardware-substitution caveat.** Performance numbers are silicon-dependent. When the issue is `platform: dgx-spark` or `platform: gb10` and we're measuring on a Brev x86 GPU SKU, the comment must say so explicitly: a Brev p50 of 1.5s on an `H100` does not prove the DGX Spark p50 is fixed. Cap the score at 60 unless the bug is clearly silicon-independent (e.g. an algorithmic regression in user-space JS that would manifest the same on any silicon). --- diff --git a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/scoring-comments-and-logging.md b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/scoring-comments-and-logging.md index d4f1fe37f0c..3a91e731a4d 100644 --- a/.agents/skills/nemoclaw-maintainer-verify-stale/reference/scoring-comments-and-logging.md +++ b/.agents/skills/nemoclaw-maintainer-verify-stale/reference/scoring-comments-and-logging.md @@ -1,9 +1,9 @@ <!-- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. --> <!-- SPDX-License-Identifier: Apache-2.0 --> -# verify-stale — Scoring, Comments, Labels, and Logging Reference +# verify-stale — Scoring, Comments, Project Fields, and Logging Reference -Use after a latest result exists or after a by-design/inconclusive branch is selected. Covers confidence scoring, redaction, concise comments, labels, project movement, infra failures, and activity logging. +Use after a latest result exists or after a by-design/inconclusive branch is selected. Covers confidence scoring, redaction, concise comments, authorized Project updates, infra failures, and activity logging. ## Contents @@ -40,13 +40,14 @@ The skill needs to know *which* path to `git log v<reported>..$LATEST -- <path>` - `/usr/local/bin/nemoclaw*` → `bin/` - `~/.nemoclaw/<rel>` → most often runtime state, drop unless the bug is config-related → `src/lib/config/` - In-repo paths (e.g., `bin/lib/policies.js` mentioned literally) → use as-is -2. **Component-label-to-directory map.** Pick the first match. Paths verified against the current repo layout — drop any path that doesn't exist on the tag at `$LATEST` rather than passing it to `git log`. - - `NemoClaw CLI` → `bin/`, `src/lib/`, `nemoclaw/src/commands/` - - `Sandbox` → `nemoclaw/src/blueprint/`, `nemoclaw-blueprint/` - - `OpenShell` → cross-repo (lives at `github.com/NVIDIA/OpenShell`, not in this repo). Skip the +25 signal for OpenShell-only issues; cross-repo `git log` is out of v1 scope. - - `Docker` → `Dockerfile`, `Dockerfile.base`, `scripts/install-openshell.sh`, `scripts/install.sh` - - `Getting Started` → `docs/`, `scripts/install.sh` - - `Integration: <X>` — no `src/lib/integrations/` exists in this repo. Skip the +25 signal for integration-component issues unless source 1 (file paths in body) yielded a path. +2. **Canonical routing-label-to-directory map.** Pick the first match. Drop paths that do not exist at `$LATEST`. + - `area: cli` → `bin/`, `src/commands/`, `src/lib/cli/` + - `area: sandbox` → `src/lib/sandbox/`, `nemoclaw/src/blueprint/`, `nemoclaw-blueprint/` + - `platform: container` or `area: packaging` → `Dockerfile`, `Dockerfile.base`, `scripts/install-openshell.sh`, `scripts/install.sh` + - `area: install` or `area: onboarding` → `scripts/install.sh`, `src/lib/onboard/` + - `area: policy` → `nemoclaw-blueprint/policies/`, `nemoclaw/src/blueprint/` + - `area: messaging` → `src/lib/messaging/` + - `integration: *` with no body path → skip the +25 signal; no generic integration directory owns every integration. 3. **Title keywords.** "policy" → `nemoclaw-blueprint/policies/`, `nemoclaw/src/blueprint/`. "inference" → `docs/inference/` is docs-only; skip the +25 signal unless source 1 surfaces actual code paths. If none of the above produces a path, **skip the +25 signal entirely** rather than guessing. Floating the +25 on every issue would inflate scores meaninglessly. @@ -77,22 +78,24 @@ If neither query returns anything, **skip the +25 signal**. **Action (when latest run was clean — bug not reproduced):** -| Score | Label | Comment | -|---|---|---| -| ≥85 | `fixed-on-latest` | Evidence-rich, no @-mention. | -| 60–84 | `fixed-on-latest` | Evidence-rich, **@-mention the original reporter** to confirm. | -| <60 | `verify-inconclusive` | Short, honest "couldn't verify" explanation. | +| Score | Verdict | Proposed Project action | Comment | +|---|---|---|---| +| ≥85 | `fixed-on-latest` | `Needs Review` | Evidence-rich; ask the reporter to confirm. | +| 60–84 | `fixed-on-latest` | `Needs Review` | Evidence-rich; ask the reporter to confirm and state the confidence cap. | +| <60 | `verify-inconclusive` | No field change | Short, honest "couldn't verify" explanation. | + +Verdict names are comment and log vocabulary, not GitHub labels. Prepare the exact comment, Project update, assignment, and durable verdict marker as a dry run with `human_review_required: true`; apply only the accepted write set. **Special case: latest output matches the issue symptom (bug still reproduces on latest).** This is not a flake — the skill positively confirmed the bug is still live. Don't apply the +50 weight (the bug isn't fixed) and skip the score table entirely. -- Post a "still reproduces on latest" comment with both transcripts. -- Apply **no label**. -- Include the marker `<!-- nemoclaw-verify-stale v1 YYYY-MM-DD -->` with today's date so the candidate filter applies the 7-day TTL (Step 3 idempotency). +- Post a 30–80 word "still reproduces on latest" comment without transcripts. Keep the redacted baseline/latest transcripts in the local activity log as evidence. +- Make no Project field or label change. +- Include the marker `<!-- nemoclaw-verify-stale v1 verdict=still-reproduces YYYY-MM-DD -->` with today's date so the candidate filter applies the 7-day TTL (Step 3 idempotency). - Next weekly run picks the issue back up after the TTL — if the bug gets fixed in the meantime, that run catches it. -The skill **never closes issues** in any branch. A maintainer pulls that trigger after reviewing the label and comment. +The skill **never closes issues** in any branch. Project fields, assignments, and public comments require explicit approval of the proposed write set. --- @@ -157,7 +160,7 @@ Patterns live in a fenced block (not a Markdown table) because patterns 8 and 9 **File paths under the reporter's home directory** (`/Users/<name>/`, `/home/<name>/`) → replace with `~/`. Run last; catches incidental username PII. -**Comment authoring principle.** Every section in a rendered comment must either change a reader's mind about the verdict, or be cut. Word counts follow from that — **300 is a hard ceiling** for the main verdicts (fixed-on-latest, wontfix). Simple cases (clear PR ref, deterministic check) land under 200. The principle generalizes: comments posted by this skill compete for a maintainer's attention against every other in-flight thread, and "AI-slop" prose — architectural sidebars, file:line citations the maintainer can find via the PR ref, bare-output reproductions when the load-bearing evidence is elsewhere, "if this verification is wrong, please reopen…" boilerplate — actively reduces the comment's signal-to-noise ratio. +**Comment authoring principle.** Every section in a rendered comment must either change a reader's mind about the verdict, or be cut. Word counts follow from that — **300 is a hard ceiling** for the main verdicts (`fixed-on-latest`, `by-design`). Simple cases (clear PR ref, deterministic check) land under 200. The principle generalizes: comments posted by this skill compete for a maintainer's attention against every other in-flight thread, and "AI-slop" prose — architectural sidebars, file:line citations the maintainer can find via the PR ref, bare-output reproductions when the load-bearing evidence is elsewhere, "if this verification is wrong, please reopen…" boilerplate — actively reduces the comment's signal-to-noise ratio. **For each section in a draft, ask: would the maintainer reach a different conclusion *without* this section? If no, delete.** Lessons accumulated from real runs: @@ -169,9 +172,9 @@ Patterns live in a fenced block (not a Markdown table) because patterns 8 and 9 | Verdict | Target | Rationale | |---|---|---| | `fixed-on-latest` | **200–300 words** | Header + evidence + verdict + @-mention. Add hardware-substitution caveat or related-failure-mode section only if they shift the maintainer's read. If you're past 300, you're padding. | -| `wontfix` (by-design) | **200–300 words** | Structurally-fixed + vestigial + what's-not-the-same-bug, each one to two sentences max. The PR ref carries the detail; the comment carries the verdict. | +| `by-design` | **200–300 words** | Structurally-fixed + vestigial + what's-not-the-same-bug, each one to two sentences max. The PR ref carries the detail; the comment carries the verdict. | | `verify-inconclusive` | 100–200 words | One paragraph naming what the skill couldn't establish. No transcripts beyond a single quoted line. | -| **Still-reproduces (no label)** | **30–80 words** | The reporter already has the symptom; the maintainer can see the issue is open. The skill is just confirming + setting the TTL marker. **No transcripts** (the issue body has them), **no closing reporter @-mention** (the reporter knows their bug is real), **no architectural prose**. One sentence stating "skill ran reproducer on `<latest>`, symptom still present" + one sentence on any partial-fix PR if relevant + marker. That's it. The unanswered-question lead paragraph (rule below) is the one allowed exception when `UNANSWERED_MAINT_LOGIN` is set — it adds one maintainer @-mention as a lead, never a closing pair. | +| **Still-reproduces** | **30–80 words** | The reporter already has the symptom; the maintainer can see the issue is open. The skill is just confirming + setting the TTL marker. **No transcripts** (the issue body has them), **no closing reporter @-mention** (the reporter knows their bug is real), **no architectural prose**. One sentence stating "skill ran reproducer on `<latest>`, symptom still present" + one sentence on any partial-fix PR if relevant + marker. That's it. The unanswered-question lead paragraph (rule below) is the one allowed exception when `UNANSWERED_MAINT_LOGIN` is set — it adds one maintainer @-mention as a lead, never a closing pair. | **Cut, by default:** @@ -184,7 +187,7 @@ Patterns live in a fenced block (not a Markdown table) because patterns 8 and 9 **Mandatory cap caveat.** When the score is capped (Step 9 baseline-validation gating, or any Step 11 degraded-mode path), the rendered Verdict section must include a one-line caveat naming the cap and the reason. Example: `Capped at 84 because Step 9's baseline-validation gate did not run (sandbox-build rot on v0.0.18: Dockerfile symlink layer removed by #2227).` Don't make readers reverse-engineer why the score didn't go higher — name it. -**Mandatory hardware-substitution caveat.** When the issue carries `Platform: DGX Spark` or `Platform: GB10` and Step 7 provisioned a Brev SKU that is not the same silicon (Brev's stoppable GPU catalog is x86 + discrete H100/A100/L40S/T4 — not Grace Hopper / GB10 unified-memory ARM64), the rendered comment must include a one-line "Hardware substitution" note. Example: `Hardware substitution: verified on Brev n1-standard-4:nvidia-tesla-t4 (x86_64 + T4) as a substitute for the reporter's DGX Spark (ARM64 + GB10). For silicon-shape bugs (perf, memory architecture, drivers) this is not a faithful repro — please confirm on actual DGX Spark.` This goes in the metadata block right after `Verification mode:` so it's visible at the top, not buried in the analysis. +**Mandatory hardware-substitution caveat.** When the issue carries `platform: dgx-spark` or `platform: gb10` and Step 7 provisioned a Brev SKU that is not the same silicon, the rendered comment must include a one-line "Hardware substitution" note. Name both the actual Brev SKU and the reported hardware, and state that performance, memory-architecture, and driver results require confirmation on the real target. **Mandatory `Verification mode` header line.** All three templates below include a `**Verification mode:**` line in the metadata block, naming what we did and didn't actually run (e.g., "runtime reproduction on Brev <SKU>; baseline + latest both installed and run" for the standard template; "static analysis at the verified-on tag — no runtime reproduction" for the by-design template; "runtime reproduction on Brev <SKU>; bug confirmed live on latest" for still-reproduces). Reader should never have to guess whether the verdict came from real install logs or from static analysis. @@ -253,7 +256,7 @@ The skill becomes the *unsticking voice* on a thread that has gone quiet — nev ### Verdict -**Confidence:** 88 / 100. Labelling `fixed-on-latest`. +**Confidence:** 88 / 100. Verdict: `fixed-on-latest`; proposing Project Status `Needs Review`. <details><summary>Relevant changes since v0.0.31</summary> @@ -264,9 +267,15 @@ The skill becomes the *unsticking voice* on a thread that has gone quiet — nev @<reporter> — please confirm the symptom is gone on a recent build (≥ v0.0.<Z>) and reopen with a fresh reproducer if you observe otherwise. -<!-- nemoclaw-verify-stale v1 YYYY-MM-DD --> +<!-- nemoclaw-verify-stale v1 verdict=fixed-on-latest YYYY-MM-DD --> ```` +For a score below 60, use the same evidence structure only as far as needed for the shorter inconclusive comment, state `Verdict: verify-inconclusive; no Project field change proposed`, and end with: + +```text +<!-- nemoclaw-verify-stale v1 verdict=verify-inconclusive YYYY-MM-DD --> +``` + **Comment template (still reproduces — Step 9 special case).** Keep this minimal — per L174 it caps at 30–80 words, drops transcripts (issue body has them), and omits the closing reporter @-mention (the reporter knows their own bug is real). Only the unanswered-question lead paragraph (when fired) adds an @-mention; no closing dual @-mention even then. ````markdown @@ -276,89 +285,38 @@ The skill becomes the *unsticking voice* on a thread that has gone quiet — nev **Verified on:** v0.0.34 **Verification mode:** runtime reproduction on Brev `<instance-class>` — bug confirmed live on latest. -Skill ran the reported reproducer on v0.0.34 and observed the same symptom. No label applied; will re-verify on the next weekly pass. +Skill ran the reported reproducer on v0.0.34 and observed the same symptom. No Project field or label change proposed; will re-verify on the next weekly pass. -<!-- nemoclaw-verify-stale v1 YYYY-MM-DD --> +<!-- nemoclaw-verify-stale v1 verdict=still-reproduces YYYY-MM-DD --> ```` If a partial-fix PR is in flight that targets the same surface, add one sentence naming it between the verification line and the marker: `Partial fix tracked in #NNNN (not yet released).` Keep the total under 80 words. -The trailing HTML comment is the **idempotency marker** Step 3 looks for. Always include today's date in `YYYY-MM-DD` format so the candidate filter can apply the 7-day TTL. +The trailing HTML comment is the **idempotency marker** Step 3 looks for. Always include today's date in `YYYY-MM-DD` format. Final verdict markers are durable; only `still-reproduces` uses the seven-day TTL. + +**Authorization boundary.** Before any write, present a dry run containing: + +- the verdict and confidence; +- the exact redacted public comment, including its durable marker; +- the proposed Project Status change (`Needs Review` only for `fixed-on-latest`; none for inconclusive or still-reproduces); +- the proposed self-assignment, if any; +- `human_review_required: true`. + +Wait for explicit approval of that exact write set. Comment approval does not authorize a Project change, and Project approval does not authorize modified comment text. -**Pre-post state-check.** A long-running verification can race with the maintainer closing the issue independently — happened on #2513 and #2519 (mid-batch closes by @jyaunches with their own verification). Re-check `state == OPEN` right before posting. If closed, apply the label tag-only (skipping the comment, since the maintainer's own close-comment is now the authoritative record) and skip the Project 199 move. +**Pre-post state-check.** A long-running verification can race with a maintainer closing the issue independently. Re-check `state == OPEN` immediately before applying an accepted write set. If closed, skip every write and report that the maintainer's close action is now authoritative. ```bash STATE=$(gh issue view "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --json state --jq .state) if [ "$STATE" != "OPEN" ]; then - echo "[verify-stale] #$ISSUE_NUMBER closed since verification started — applying label tag-only, skipping comment + tracker move" - gh issue edit "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --add-label "$LABEL" + echo "[verify-stale] #$ISSUE_NUMBER closed since verification started — skipping Project, assignment, and comment writes" exit 0 fi ``` -**Post the comment and apply the label:** - -```bash -gh issue comment "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --body-file comment.md -gh issue edit "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --add-label "fixed-on-latest" -# or for <60: -# gh issue edit "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --add-label "verify-inconclusive" -``` - -**Move the issue to "Needs Review" on the NemoClaw Development Tracker AND self-assign (only on `fixed-on-latest`).** The tracker is GitHub Project [NVIDIA/199](https://github.com/orgs/NVIDIA/projects/199) ("NemoClaw Development Tracker"). When the skill's verdict is `fixed-on-latest`, the issue moves to **Needs Review** AND the issue is assigned to the maintainer who ran the skill (`$GH_IDENTITY` from Step 6.5) — assignment puts the issue in their personal review queue so they don't lose track of what they've staked their name on. After the reporter confirms and the maintainer closes, existing Project automation (or a manual move) advances it to Done. **No move and no assign on `wontfix` / `verify-inconclusive` / no-label-still-reproduces** — those have separate close paths. - -This step requires the `project` scope on the maintainer's gh CLI (`gh auth refresh -h github.com -s project` in a real terminal once; OAuth device-code flow). If the scope is missing, the lookup query returns an auth error — fall through with a one-line warning rather than failing the whole run. - -```bash -# Project 199 constants (re-run gh project field-list 199 --owner NVIDIA --format json -# if the project gets renamed/restructured and these IDs drift): -PROJECT_ID="PVT_kwDOABpemM4BSCP5" -STATUS_FIELD_ID="PVTSSF_lADOABpemM4BSCP5zg_r9p8" -NEEDS_REVIEW_OPTION_ID="5c5922a9" - -# Only fire on fixed-on-latest. Skip silently otherwise. -if [ "$VERDICT" = "fixed-on-latest" ]; then - # Find the issue's existing project item, if any. - ITEM_ID=$(gh api graphql -f query=' - query($num: Int!) { - repository(owner: "NVIDIA", name: "NemoClaw") { - issue(number: $num) { - projectItems(first: 10) { - nodes { id project { number } } - } - } - } - }' -F num="$ISSUE_NUMBER" \ - --jq '.data.repository.issue.projectItems.nodes[] | select(.project.number == 199) | .id' \ - 2>/dev/null | head -1) - - # If the issue isn't on the project yet, add it. (NV QA bots usually add new - # issues automatically, but cover the gap.) - if [ -z "$ITEM_ID" ]; then - ITEM_ID=$(gh project item-add 199 --owner NVIDIA \ - --url "https://github.com/NVIDIA/NemoClaw/issues/$ISSUE_NUMBER" \ - --format json --jq .id 2>/dev/null) - fi - - if [ -n "$ITEM_ID" ]; then - gh project item-edit \ - --id "$ITEM_ID" \ - --project-id "$PROJECT_ID" \ - --field-id "$STATUS_FIELD_ID" \ - --single-select-option-id "$NEEDS_REVIEW_OPTION_ID" \ - >/dev/null && echo "[verify-stale] moved #$ISSUE_NUMBER to 'Needs Review' on Project 199" - else - echo "[verify-stale] WARN could not resolve project item for #$ISSUE_NUMBER on Project 199 — label applied but tracker not moved" - fi - - # Self-assign the issue to the maintainer who ran the skill — puts it in their - # personal review queue alongside the Needs Review state. - gh issue edit "$ISSUE_NUMBER" --repo NVIDIA/NemoClaw --add-assignee "$GH_IDENTITY" \ - >/dev/null && echo "[verify-stale] assigned #$ISSUE_NUMBER to @$GH_IDENTITY" -fi -``` +**Apply the accepted write set in canonical order.** Resolve Project 199, Status-field, option, and item IDs from live GitHub data immediately before writing; do not use hardcoded IDs. For an accepted `fixed-on-latest` plan, set Project Status `Needs Review`, then self-assign only if that assignment was accepted. Treat the Project update and accepted assignment as one fail-fast write set: if either write fails, stop before posting the comment. For inconclusive and still-reproduces verdicts, do not change Project fields or assignment. Post the exact accepted comment last. -The Step 12 activity log line should record the project move (or the warn-and-skip case) so a maintainer scanning the log can spot tracker drift. Add a `Tracker:` row to the per-issue entry: `Tracker: moved to Needs Review` | `not moved (verdict: <X>)` | `not moved (project lookup failed)`. +If Project resolution, update, or accepted assignment fails, stop without posting the comment so the accepted write set is not partially represented. Record the Project update, assignment, and comment outcome in the activity log. --- @@ -369,7 +327,7 @@ Two different failure types, two different responses. **Latest-install failure** (Step 8d) or reuse-check / provisioning / harness errors: hard infra failure. - Print the error. -- Apply **no label** — infra failures must not pollute the verification record. +- Apply no Project field or label change — infra failures must not pollute the verification record. - Post a short comment **only if explicitly requested by the invoking user**. Default is silent move-on. - Continue to the next candidate in batch mode. @@ -418,8 +376,8 @@ After each issue (verified, inconclusive, by-design, or infra-failed), append to **Latest install:** succeeded | failed (infra error) **Latest result:** not-reproduced (clean) | still-reproduces | partial / flake | n/a (skipped 8d) **Confidence:** 88 / 100 | n/a (still-reproduces) -**Label applied:** fixed-on-latest | verify-inconclusive | status: wont-fix | none (still-reproduces) | none (infra) -**Tracker:** moved to Needs Review on Project 199 | not moved (verdict: <X>) | not moved (project lookup failed) +**Verdict marker:** fixed-on-latest | verify-inconclusive | by-design | still-reproduces | none (infra) +**Project Status:** moved to Needs Review | moved to Won't Fix | unchanged | update failed **Assignee:** @<GH_IDENTITY> | not assigned (verdict: <X>) **Brev wall time (approx):** N min @@ -443,8 +401,8 @@ At end of a batch session, prepend a session summary: ## YYYY-MM-DD — Verify Session **Issues considered:** N **Verified `fixed-on-latest`:** N -**Marked `status: wont-fix` (by-design path):** N -**Marked `verify-inconclusive`:** N +**Approved `Won't Fix` Project updates (by-design path):** N +**Recorded `verify-inconclusive` verdicts:** N **Local-first short-circuits (no Brev cost):** N **Skipped (Windows / macOS / integration / no version):** N **Infra failures:** N @@ -466,14 +424,14 @@ Never stage or commit the log to the NemoClaw repo. ## Out of Scope (v1) -- Auto-closing issues. Always tag-only; a human pulls the trigger. +- Auto-closing issues. The skill may make only the explicitly approved Project, assignment, and comment writes described above; a human separately decides whether to close. - macOS verification *via the Brev path*. Brev offers no macOS instances. The Step 6.7 local-first short-circuit *does* run on a maintainer's macOS laptop — so manual single-issue runs against pure-CLI bugs work on macOS. The weekly batch cron is Linux-only because that path always uses Brev. - Issues requiring third-party integration credentials (Slack, Discord, Telegram, Hermes, OpenClaw, WeChat). - Service-account bot identity. v1 runs under each maintainer's own GitHub credentials. -- Versioned labels. `fixed-on-latest` and `verify-inconclusive` are persistent maintainer-review labels, not per-release labels. +- Verdict labels. `fixed-on-latest`, `verify-inconclusive`, and `status: wont-fix` are not canonical labels; durable comment markers and Project fields carry the workflow state. --- ## Companion Behavior -`nemoclaw-maintainer-cut-release-tag` does not sweep issue labels during release. A `fixed-on-latest` or `verify-inconclusive` label stays until a maintainer removes it or explicitly re-runs verification for that issue. The by-design path uses the existing repo `status: wont-fix` label; that label is also persistent because it is applied for non-skill reasons such as scope or priority decisions. +`nemoclaw-maintainer-cut-release-tag` does not alter verify-stale verdict markers or Project Status. Those remain durable until a maintainer explicitly re-runs verification or changes the Project field. diff --git a/.agents/skills/nemoclaw-skills-guide/SKILL.md b/.agents/skills/nemoclaw-skills-guide/SKILL.md index 901d26ea2fb..369bc87e6ff 100644 --- a/.agents/skills/nemoclaw-skills-guide/SKILL.md +++ b/.agents/skills/nemoclaw-skills-guide/SKILL.md @@ -51,14 +51,14 @@ Covers creating pull requests that follow the project template, monitoring CI an | `nemoclaw-maintainer-policies` | Read-only maintainer workflow policy reference for Issue Type, labels, Project fields, daily release labels, triage, duplicates, blocked items, and workflow decisions. | | `nemoclaw-maintainer-cross-issue-sweep` | Scan open issues for adjacent fixes or contradiction risks when reviewing a PR. | | `nemoclaw-maintainer-day` | Daytime loop: pick the highest-value version-targeted item and execute the right workflow (merge gate, salvage, security sweep, test gaps, hotspot cooling, or sequencing). Designed for `/loop`. | -| `nemoclaw-maintainer-evening` | End-of-day handoff: check version progress, bump stragglers to the next patch, generate a QA handoff summary, cut the release tag, and hand off release notes. | -| `nemoclaw-maintainer-cut-release-tag` | Cut an annotated semver tag on a maintainer-confirmed `origin/main` commit; the GitHub workflow moves `latest`, and `lkg` stays manual. | +| `nemoclaw-maintainer-evening` | End-of-day handoff: check version progress, identify stragglers, generate a QA handoff summary, cut the release tag, bump stragglers to the next patch, and hand off release notes. | +| `nemoclaw-maintainer-cut-release-tag` | Cut an annotated semver tag on a maintainer-confirmed `origin/main` commit, wait for workflow-managed `latest`, and bump remaining open items to the next patch; `lkg` stays manual. | | `nemoclaw-maintainer-release-notes` | Draft release notes from live tag/compare data, with the three-paragraph narrative, categorized change list, and external-only contributor thanks. | -| `nemoclaw-maintainer-find-review-pr` | Find open PRs labeled security + priority-high, link each to its issue, detect duplicates, and present a review summary. | +| `nemoclaw-maintainer-find-review-pr` | Find open security PRs with Urgent or High Project Priority, link each to its issue, detect duplicates, and present a review summary. | | `nemoclaw-maintainer-pr-comparator` | Compare competing PRs for the same issue and recommend which one to merge. | | `nemoclaw-maintainer-normalize-title-tags` | Preview and remove bracketed `NemoClaw` title tags from issues and PRs case-insensitively, even when the tag appears later in the title. | | `nemoclaw-maintainer-security-code-review` | Perform a 9-category security review of a PR or issue, producing per-category PASS/WARNING/FAIL verdicts. | -| `nemoclaw-maintainer-verify-stale` | Verify whether old bug reports still reproduce on latest. Reuses or provisions a Brev box (CPU or GPU), runs the extracted reproducer, scores confidence, and posts an evidence-backed comment with `fixed-on-latest` or `verify-inconclusive`. Tag-only — never auto-closes. | +| `nemoclaw-maintainer-verify-stale` | Verify whether old issues with native Issue Type `Bug` still reproduce on latest. Reuses or provisions a Brev box, scores confidence, and proposes evidence-backed Project/comment writes for approval; never auto-closes. | ### Contributor Skills diff --git a/test/bump-stragglers.test.ts b/test/bump-stragglers.test.ts index 62dce392bcf..ebdbbaf1e5f 100644 --- a/test/bump-stragglers.test.ts +++ b/test/bump-stragglers.test.ts @@ -37,6 +37,31 @@ function runBumpStragglers(fakeGh: string) { } describe("bump-stragglers release housekeeping", () => { + it("creates the next label and moves open PRs and issues", () => { + const result = runBumpStragglers(`#!/usr/bin/env bash +set -euo pipefail +case "$*" in + "label list"*) printf '[]' ;; + "label create v1.2.4"*) ;; + "pr list"*) printf '[{"number":42,"title":"needs more work"}]' ;; + "pr edit 42"*) ;; + "issue list"*) printf '[{"number":84,"title":"still open"}]' ;; + "issue edit 84"*) ;; + *) echo "unexpected gh args: $*" >&2; exit 9 ;; +esac +`); + + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + from: "v1.2.3", + to: "v1.2.4", + bumped: [ + { number: 42, title: "needs more work", type: "pr" }, + { number: 84, title: "still open", type: "issue" }, + ], + }); + }); + it("fails visibly when gh label lookup fails", () => { const result = runBumpStragglers(`#!/usr/bin/env bash set -euo pipefail diff --git a/test/maintainer-skills-policy.test.ts b/test/maintainer-skills-policy.test.ts new file mode 100644 index 00000000000..eebfab8dc28 --- /dev/null +++ b/test/maintainer-skills-policy.test.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const root = process.cwd(); + +function read(relativePath: string): string { + return fs.readFileSync(path.join(root, relativePath), "utf-8"); +} + +function readMarkdownTree(relativeDir: string): string { + const absoluteDir = path.join(root, relativeDir); + return fs + .readdirSync(absoluteDir, { recursive: true }) + .filter((entry): entry is string => typeof entry === "string" && entry.endsWith(".md")) + .map((entry) => fs.readFileSync(path.join(absoluteDir, entry), "utf-8")) + .join("\n"); +} + +describe("maintainer skills follow canonical workflow policy", () => { + it("routes triage through the canonical policy package", () => { + const skill = read(".agents/skills/nemoclaw-maintainer-triage/SKILL.md"); + + expect(skill).toContain("../nemoclaw-maintainer-policies/references/triage-instructions.md"); + expect(skill).toContain("native Issue Type"); + expect(skill).toContain("Project Priority and Status"); + expect(skill).not.toMatch( + /`(?:bug|documentation|question|priority: high|status: needs-info)`/u, + ); + expect( + fs.existsSync( + path.join( + root, + ".agents/skills/nemoclaw-maintainer-triage/references/triage-instructions.md", + ), + ), + ).toBe(false); + }); + + it("reads priority from Project 199 instead of a priority label", () => { + const finder = read(".agents/skills/nemoclaw-maintainer-find-review-pr/SKILL.md"); + const triage = read(".agents/skills/nemoclaw-maintainer-day/scripts/triage.ts"); + + expect(finder).toContain("gh project item-list 199"); + expect(finder).toContain('select(.priority == "Urgent" or .priority == "High")'); + expect(finder).not.toContain("priority: high"); + expect(triage).toContain('select(.field.name == "Priority")'); + expect(triage).toContain('item.projectPriority === "Urgent"'); + expect(triage).toContain('item.projectPriority === "High"'); + expect(triage.indexOf("const projectPriorities")).toBeLessThan( + triage.indexOf("const candidates"), + ); + expect(triage).not.toContain("priority: high"); + }); + + it("describes the current morning-triage data sources", () => { + const morning = read(".agents/skills/nemoclaw-maintainer-morning/SKILL.md"); + + expect(morning).not.toContain("gh-pr-merge-now --json"); + expect(morning).toContain("fetches open PRs through `gh`"); + expect(morning).toContain("reads Project 199 Priority"); + expect(morning).toContain("review, CI, file, and risky-area data"); + }); + + it("moves post-tag stragglers to the next patch version", () => { + const evening = read(".agents/skills/nemoclaw-maintainer-evening/SKILL.md"); + const release = read(".agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md"); + const morning = read(".agents/skills/nemoclaw-maintainer-morning/SKILL.md"); + const priorities = read(".agents/skills/nemoclaw-maintainer-day/PR-REVIEW-PRIORITIES.md"); + const policy = read(".agents/skills/nemoclaw-maintainer-policies/references/release-train.md"); + + expect(evening).toContain("automatically bump stragglers to the next patch"); + expect(release).toContain("scripts/bump-stragglers.ts"); + expect(release).toContain("Do not run it before Step 4"); + expect(morning).toContain("post-tag housekeeping was interrupted"); + expect(priorities).toContain("automatically bump stragglers to the next patch"); + expect(policy).toContain("automatically move every open straggler to the next patch label"); + expect( + fs.existsSync( + path.join(root, ".agents/skills/nemoclaw-maintainer-day/scripts/bump-stragglers.ts"), + ), + ).toBe(true); + }); + + it("keeps cross-issue sweeping separate from comparator scoring", () => { + const sweep = read(".agents/skills/nemoclaw-maintainer-cross-issue-sweep/SKILL.md"); + const comparator = read(".agents/skills/nemoclaw-maintainer-pr-comparator/SKILL.md"); + + expect(sweep).toContain("The comparator does not call it"); + expect(comparator).toContain("Cross-issue regression sweep (separate skill)"); + }); + + it("uses native bug type and approved Project writes for stale verification", () => { + const stale = readMarkdownTree(".agents/skills/nemoclaw-maintainer-verify-stale"); + + expect(stale).toContain('select(.issueType.name == "Bug")'); + expect(stale).toContain("Verdict names are comment and log vocabulary, not GitHub labels"); + expect(stale).toContain("Project Status `Won't Fix`"); + expect(stale).not.toMatch(/gh issue edit[^\n]*--add-label/u); + expect(stale).not.toContain("--label bug"); + }); + + it("makes DCO and GitHub verification explicit approval gates", () => { + const mergeGate = read(".agents/skills/nemoclaw-maintainer-day/MERGE-GATE.md"); + const comparator = read( + ".agents/skills/nemoclaw-maintainer-pr-comparator/scripts/collect-gates.sh", + ); + + expect(mergeGate).toContain("every PR commit appears as `Verified` in GitHub"); + expect(comparator).toContain("gate_contributor_compliance"); + expect(comparator).toContain(".commit.verification.verified"); + }); + + it("gives distinct remediation for PR-body and commit-verification failures", () => { + const verdict = read(".agents/skills/nemoclaw-maintainer-pr-comparator/templates/verdict.md"); + + expect(verdict).toContain("Missing PR-body DCO declaration: update the PR body"); + expect(verdict).toContain( + "Missing GitHub Verified commit history: replace the branch with compliant history", + ); + expect(verdict).not.toContain( + "PR-body DCO declaration or GitHub Verified commit history is missing", + ); + }); +}); diff --git a/test/skills/check-gates-compliance.test.ts b/test/skills/check-gates-compliance.test.ts new file mode 100644 index 00000000000..ae864c6ddc8 --- /dev/null +++ b/test/skills/check-gates-compliance.test.ts @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +interface ComplianceFixture { + body: string; + commitOutput?: string; + verified: boolean; + reason?: string; +} + +interface ComparatorFixture extends ComplianceFixture { + checkNames?: string[]; + checkConclusions?: Record<string, string>; + headRefOid?: string; + state?: string; + mergeable?: string; + mergeStateStatus?: string; + reviewDecision?: string; +} + +function shellSingleQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function runGate(fixture: ComplianceFixture) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "check-gates-compliance-")); + const bin = path.join(tmp, "bin"); + fs.mkdirSync(bin); + const ghPath = path.join(bin, "gh"); + + const pr = { + number: 42, + title: "fix(policy): align maintainer workflow", + url: "https://github.com/NVIDIA/NemoClaw/pull/42", + body: fixture.body, + files: [], + statusCheckRollup: ["checks", "commit-lint", "dco-check"].map((name) => ({ + __typename: "CheckRun", + name, + status: "COMPLETED", + conclusion: "SUCCESS", + })), + mergeStateStatus: "CLEAN", + headRefOid: "abc123", + }; + const commit = { + sha: "abc123", + verified: fixture.verified, + reason: fixture.reason ?? (fixture.verified ? "valid" : "unsigned"), + }; + const commitOutput = fixture.commitOutput ?? JSON.stringify(commit); + + fs.writeFileSync( + ghPath, + `#!/usr/bin/env bash +set -euo pipefail +case "$1 $2" in + "pr view") printf '%s' ${shellSingleQuote(JSON.stringify(pr))} ;; + "api graphql") printf '%s' '{"data":{"repository":{"pullRequest":{"reviewThreads":{"nodes":[]}}}}}' ;; + "api repos/NVIDIA/NemoClaw/issues/42/comments") printf '%s' '{"id":1,"body":"ordinary comment","user":{"login":"reviewer"},"updated_at":"2026-01-01T00:00:00Z"}' ;; + "api repos/NVIDIA/NemoClaw/pulls/42/commits") printf '%s' ${shellSingleQuote(commitOutput)} ;; + *) echo "unexpected gh args: $*" >&2; exit 9 ;; +esac +`, + ); + fs.chmodSync(ghPath, 0o755); + + try { + return spawnSync( + process.execPath, + [ + "--experimental-strip-types", + "--no-warnings", + ".agents/skills/nemoclaw-maintainer-day/scripts/check-gates.ts", + "42", + ], + { + cwd: process.cwd(), + encoding: "utf-8", + env: { ...process.env, PATH: `${bin}${path.delimiter}${process.env.PATH ?? ""}` }, + }, + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +function runComparatorGate(fixture: ComparatorFixture, prNumber = "42") { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "collect-gates-compliance-")); + const bin = path.join(tmp, "bin"); + fs.mkdirSync(bin); + const ghPath = path.join(bin, "gh"); + + const pr = { + number: 42, + state: fixture.state ?? "OPEN", + body: fixture.body, + headRefOid: fixture.headRefOid ?? "abc123", + statusCheckRollup: (fixture.checkNames ?? ["checks", "commit-lint", "dco-check"]).map( + (name) => ({ + name, + status: "COMPLETED", + conclusion: fixture.checkConclusions?.[name] ?? "SUCCESS", + }), + ), + mergeable: fixture.mergeable ?? "MERGEABLE", + mergeStateStatus: fixture.mergeStateStatus ?? "CLEAN", + reviewDecision: fixture.reviewDecision ?? "APPROVED", + }; + const commit = { + sha: "abc123", + verified: fixture.verified, + reason: fixture.reason ?? (fixture.verified ? "valid" : "unsigned"), + }; + const commitOutput = fixture.commitOutput ?? JSON.stringify(commit); + + fs.writeFileSync( + ghPath, + `#!/usr/bin/env bash +set -euo pipefail +case "$1 $2" in + "pr view") printf '%s' ${shellSingleQuote(JSON.stringify(pr))} ;; + "api repos/NVIDIA/NemoClaw/pulls/42/commits") printf '%s' ${shellSingleQuote(commitOutput)} ;; + *) echo "unexpected gh args: $*" >&2; exit 9 ;; +esac +`, + ); + fs.chmodSync(ghPath, 0o755); + + try { + return spawnSync( + "bash", + [ + ".agents/skills/nemoclaw-maintainer-pr-comparator/scripts/collect-gates.sh", + prNumber, + "--repo", + "NVIDIA/NemoClaw", + ], + { + cwd: process.cwd(), + encoding: "utf-8", + env: { ...process.env, PATH: `${bin}${path.delimiter}${process.env.PATH ?? ""}` }, + }, + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +describe("maintainer merge-gate contributor compliance", () => { + it("passes when the PR body has DCO and every commit is GitHub Verified", () => { + const result = runGate({ + body: "## Summary\n\nPolicy alignment.\n\nSigned-off-by: Example User <user@example.com>", + verified: true, + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.contributorCompliance).toMatchObject({ + pass: true, + dcoDeclarationPresent: true, + unverifiedCommits: [], + }); + expect(output.allPass).toBe(true); + }); + + it("fails closed when the PR body lacks the DCO declaration", () => { + const result = runGate({ body: "## Summary\n\nNo declaration.", verified: true }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.contributorCompliance.pass).toBe(false); + expect(output.gates.contributorCompliance.details).toContain("lacks a valid Signed-off-by"); + expect(output.allPass).toBe(false); + }); + + it("fails closed when any PR commit is not GitHub Verified", () => { + const result = runGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: false, + reason: "unsigned", + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.contributorCompliance).toMatchObject({ + pass: false, + dcoDeclarationPresent: true, + unverifiedCommits: [{ sha: "abc123", reason: "unsigned" }], + }); + expect(output.allPass).toBe(false); + }); + + it("fails closed for type-skewed commit verification data", () => { + const result = runGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + commitOutput: JSON.stringify({ + sha: "abc123", + verified: "false", + reason: "unsigned", + }), + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.contributorCompliance).toMatchObject({ + pass: false, + unverifiedCommits: [{ sha: "abc123", reason: "malformed_commit_verification_data" }], + }); + expect(output.allPass).toBe(false); + }); +}); + +describe("maintainer PR comparator contributor compliance", () => { + it("passes when DCO and every commit are verified", () => { + const result = runComparatorGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.ci_green_latest_sha).toBe(true); + expect(output.gates.contributor_compliance).toBe(true); + expect(output.details).toMatchObject({ + dco_declaration_present: true, + commit_count: 1, + unverified_commits: [], + }); + }); + + it("fails when a commit is not verified", () => { + const result = runComparatorGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: false, + reason: "unsigned", + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.contributor_compliance).toBe(false); + expect(output.details.unverified_commits).toEqual([{ sha: "abc123", reason: "unsigned" }]); + expect(output.failures).toContain("ineligible:contributor_compliance"); + }); + + it("emits fail-closed JSON when commit API output is malformed", () => { + const result = runComparatorGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + commitOutput: "not-json", + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.contributor_compliance).toBe(false); + expect(output.details).toMatchObject({ + commit_count: 0, + unverified_commits: [], + commit_fetch_failed: false, + commit_parse_failed: true, + }); + expect(output.failures).toContain("ineligible:contributor_compliance"); + }); + + it("fails when the PR body lacks the DCO declaration", () => { + const result = runComparatorGate({ + body: "## Summary\n\nNo declaration.", + verified: true, + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.contributor_compliance).toBe(false); + expect(output.details.dco_declaration_present).toBe(false); + expect(output.failures).toContain("ineligible:contributor_compliance"); + }); + + it("rejects a non-numeric PR argument without emitting malformed JSON", () => { + const result = runComparatorGate( + { + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + }, + '42,"injected":true', + ); + + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + pr: '42,"injected":true', + error: "invalid_pr_number", + }); + expect(result.stderr).toBe(""); + }); + + it("serializes unusual GitHub string values as valid JSON", () => { + const result = runComparatorGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + headRefOid: 'abc"123\\nnext', + state: 'OPEN"unexpected', + mergeable: 'MERGEABLE"unexpected', + mergeStateStatus: 'CLEAN"unexpected', + reviewDecision: 'APPROVED"unexpected', + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.head_sha).toBe('abc"123\\nnext'); + expect(output.details).toMatchObject({ + state: 'OPEN"unexpected', + mergeable: 'MERGEABLE"unexpected', + merge_state_status: 'CLEAN"unexpected', + review_decision: 'APPROVED"unexpected', + }); + }); + + it("fails closed when the status check rollup is empty", () => { + const result = runComparatorGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + checkNames: [], + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.ci_green_latest_sha).toBe(false); + expect(output.details.ci_missing_required_checks).toEqual([ + "checks", + "commit-lint", + "dco-check", + ]); + expect(output.failures).toContain( + "substantive:ci_failures=0,pending=0,missing=checks,commit-lint,dco-check", + ); + }); + + describe("contributor-compliance DCO parity", () => { + it("requires the canonical Signed-off-by trailer casing in both gates", () => { + const fixture = { + body: "signed-off-by: Example User <user@example.com>", + verified: true, + }; + const mergeGate = runGate(fixture); + const comparator = runComparatorGate(fixture); + + expect(mergeGate.status).toBe(0); + expect(comparator.status).toBe(0); + expect(JSON.parse(mergeGate.stdout).gates.contributorCompliance.pass).toBe(false); + expect(JSON.parse(comparator.stdout).gates.contributor_compliance).toBe(false); + }); + }); + + it("names a missing required check and fails the CI gate", () => { + const result = runComparatorGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + checkNames: ["checks", "commit-lint"], + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.ci_green_latest_sha).toBe(false); + expect(output.details.ci_missing_required_checks).toEqual(["dco-check"]); + expect(output.failures).toContain("substantive:ci_failures=0,pending=0,missing=dco-check"); + }); + + it.each([ + "ACTION_REQUIRED", + "STARTUP_FAILURE", + "STALE", + ])("fails closed for a completed required check with conclusion %s", (conclusion) => { + const result = runComparatorGate({ + body: "Signed-off-by: Example User <user@example.com>", + verified: true, + checkConclusions: { checks: conclusion }, + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.gates.ci_green_latest_sha).toBe(false); + expect(output.details.ci_failing_checks).toEqual([`checks: ${conclusion}`]); + expect(output.failures).toContain("substantive:ci_failures=1,pending=0,missing="); + }); +}); diff --git a/test/skills/triage-runtime.test.ts b/test/skills/triage-runtime.test.ts new file mode 100644 index 00000000000..f00eac89488 --- /dev/null +++ b/test/skills/triage-runtime.test.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +interface TriageFixture { + projectOutput: string; + reviewDecisions?: Record<number, string>; + approvedOnly?: boolean; +} + +const requiredChecks = ["checks", "commit-lint", "dco-check"].map((name) => ({ + name, + status: "COMPLETED", + conclusion: "SUCCESS", +})); + +const pullRequests = [ + { + number: 101, + title: "Urgent Project item", + url: "https://github.com/NVIDIA/NemoClaw/pull/101", + author: { login: "one" }, + additions: 0, + deletions: 0, + changedFiles: 0, + isDraft: false, + createdAt: "2099-01-01T00:00:00Z", + updatedAt: "2099-01-01T00:00:00Z", + mergeStateStatus: "CLEAN", + reviewDecision: "", + labels: [], + statusCheckRollup: [], + }, + { + number: 102, + title: "High Project item", + url: "https://github.com/NVIDIA/NemoClaw/pull/102", + author: { login: "two" }, + additions: 0, + deletions: 0, + changedFiles: 0, + isDraft: false, + createdAt: "2099-01-01T00:00:00Z", + updatedAt: "2099-01-01T00:00:00Z", + mergeStateStatus: "CLEAN", + reviewDecision: "", + labels: [], + statusCheckRollup: [], + }, + { + number: 103, + title: "Legacy label only", + url: "https://github.com/NVIDIA/NemoClaw/pull/103", + author: { login: "three" }, + additions: 0, + deletions: 0, + changedFiles: 0, + isDraft: false, + createdAt: "2099-01-01T00:00:00Z", + updatedAt: "2099-01-01T00:00:00Z", + mergeStateStatus: "CLEAN", + reviewDecision: "", + labels: [{ name: "priority: high" }], + statusCheckRollup: [], + }, +]; + +function runTriage(fixture: TriageFixture) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "maintainer-triage-runtime-")); + const bin = path.join(tmp, "bin"); + fs.mkdirSync(bin); + const ghPath = path.join(bin, "gh"); + const config = { + pullRequests, + projectOutput: fixture.projectOutput, + reviewDecisions: fixture.reviewDecisions ?? { + 101: "APPROVED", + 102: "APPROVED", + 103: "APPROVED", + }, + requiredChecks, + }; + + fs.writeFileSync( + ghPath, + `#!/usr/bin/env node +const config = ${JSON.stringify(config)}; +const args = process.argv.slice(2); +if (args[0] === "api" && args[1] === "--paginate" && args[2]?.startsWith("repos/NVIDIA/NemoClaw/pulls?")) { + process.stdout.write(config.pullRequests.map(JSON.stringify).join("\\n")); +} else if (args[0] === "api" && args[1] === "graphql") { + process.stdout.write(config.projectOutput); +} else if (args[0] === "pr" && args[1] === "view") { + const number = Number(args[2]); + process.stdout.write(JSON.stringify({ + reviewDecision: config.reviewDecisions[number] ?? "", + statusCheckRollup: config.requiredChecks, + additions: 1, + deletions: 1, + changedFiles: 1, + })); +} else if (args[0] === "api" && args[1]?.startsWith("repos/NVIDIA/NemoClaw/pulls/") && args[1]?.includes("/files?")) { + process.stdout.write("[]"); +} else { + process.stderr.write(\`unexpected gh args: \${args.join(" ")}\\n\`); + process.exit(9); +} +`, + ); + fs.chmodSync(ghPath, 0o755); + + const args = [ + "--experimental-strip-types", + "--no-warnings", + ".agents/skills/nemoclaw-maintainer-day/scripts/triage.ts", + "--limit", + "10", + ...(fixture.approvedOnly ? ["--approved-only"] : []), + ]; + + try { + return spawnSync(process.execPath, args, { + cwd: process.cwd(), + encoding: "utf-8", + env: { ...process.env, PATH: `${bin}${path.delimiter}${process.env.PATH ?? ""}` }, + }); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +describe("maintainer triage runtime behavior", () => { + it("maps live Project Priority into scoring and ignores legacy priority labels", () => { + const result = runTriage({ + projectOutput: [ + { number: 101, repository: "NVIDIA/NemoClaw", priority: "Urgent" }, + { number: 102, repository: "NVIDIA/NemoClaw", priority: "High" }, + { number: 103, repository: "another/repository", priority: "High" }, + { number: null, repository: "NVIDIA/NemoClaw", priority: "Urgent" }, + ] + .map((item) => JSON.stringify(item)) + .join("\n"), + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.queue.map((item: { number: number }) => item.number)).toEqual([101, 102, 103]); + expect(output.queue).toEqual( + expect.arrayContaining([ + expect.objectContaining({ number: 101, projectPriority: "Urgent", score: 55 }), + expect.objectContaining({ number: 102, projectPriority: "High", score: 50 }), + expect.objectContaining({ number: 103, projectPriority: null, score: 40 }), + ]), + ); + }); + + it("applies --approved-only after review decisions are enriched", () => { + const result = runTriage({ + projectOutput: "", + reviewDecisions: { 101: "APPROVED", 102: "REVIEW_REQUIRED", 103: "" }, + approvedOnly: true, + }); + + expect(result.status).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.scanned).toBe(3); + expect(output.queue.map((item: { number: number }) => item.number)).toEqual([101]); + expect(output.nearMisses).toEqual([]); + }); + + it("reports malformed Project data and continues without priority boosts", () => { + const result = runTriage({ projectOutput: "not-json" }); + + expect(result.status).toBe(0); + expect(result.stderr).toContain( + "Could not parse Project 199 item data; continuing without priority boosts.", + ); + const output = JSON.parse(result.stdout); + expect(output.queue).toHaveLength(3); + expect( + output.queue.every( + (item: { projectPriority: string | null }) => item.projectPriority === null, + ), + ).toBe(true); + }); +});