From cdc072f1ceb77f9037da9e9f88ef94d8dfb3ee72 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Mon, 27 Apr 2026 02:32:13 -0500 Subject: [PATCH 1/2] fix: preserve issue-backed PR body sync --- .../__tests__/agents-pr-meta-update-body.test.js | 13 +++++-------- .github/scripts/__tests__/source-context.test.js | 14 ++++++++++++++ .github/scripts/agents_pr_meta_update_body.js | 3 +-- .github/scripts/source_context.js | 2 +- .../.github/scripts/agents_pr_meta_update_body.js | 3 +-- .../.github/scripts/source_context.js | 2 +- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/scripts/__tests__/agents-pr-meta-update-body.test.js b/.github/scripts/__tests__/agents-pr-meta-update-body.test.js index 8500fca26..61ee68901 100644 --- a/.github/scripts/__tests__/agents-pr-meta-update-body.test.js +++ b/.github/scripts/__tests__/agents-pr-meta-update-body.test.js @@ -564,7 +564,7 @@ test('extractExplicitIssueSyncNumbers returns only explicit issue references', ( ); }); -test('resolveNonIssueWorkflowSourceContextForBodySync honors explicit non-issue markers over heuristics', () => { +test('resolveNonIssueWorkflowSourceContextForBodySync yields to branch-inferred issue numbers', () => { const branchHeuristicContext = resolveNonIssueWorkflowSourceContextForBodySync( { body: [ @@ -577,8 +577,7 @@ test('resolveNonIssueWorkflowSourceContextForBodySync honors explicit non-issue 123, ); - assert.equal(branchHeuristicContext.sourceType, 'local_request'); - assert.equal(branchHeuristicContext.sourceRef, 'codex-thread-2026-04-26'); + assert.equal(branchHeuristicContext, null); const prReferenceContext = resolveNonIssueWorkflowSourceContextForBodySync( { @@ -591,8 +590,7 @@ test('resolveNonIssueWorkflowSourceContextForBodySync honors explicit non-issue 123, ); - assert.equal(prReferenceContext.sourceType, 'review_followup'); - assert.equal(prReferenceContext.sourceRef, 'PR #123'); + assert.equal(prReferenceContext, null); }); test('resolveNonIssueWorkflowSourceContextForBodySync yields to explicit issue references', () => { @@ -610,7 +608,7 @@ test('resolveNonIssueWorkflowSourceContextForBodySync yields to explicit issue r assert.equal(context, null); }); -test('resolveNonIssueWorkflowSourceContextForBodySync preserves non-issue context when explicit issue differs from heuristic', () => { +test('resolveNonIssueWorkflowSourceContextForBodySync yields to issue numbers even when explicit issue differs', () => { const context = resolveNonIssueWorkflowSourceContextForBodySync( { body: [ @@ -623,8 +621,7 @@ test('resolveNonIssueWorkflowSourceContextForBodySync preserves non-issue contex 99, ); - assert.equal(context.sourceType, 'local_request'); - assert.equal(context.sourceRef, 'codex-thread-2026-04-26'); + assert.equal(context, null); }); test('resolveSourceContextRepairComment updates an existing warning once', async () => { diff --git a/.github/scripts/__tests__/source-context.test.js b/.github/scripts/__tests__/source-context.test.js index 2acc7157b..1dd1ae1d9 100644 --- a/.github/scripts/__tests__/source-context.test.js +++ b/.github/scripts/__tests__/source-context.test.js @@ -242,6 +242,20 @@ automation: no_automation }), true); }); +test('resolvePrSourceContext preserves legacy human-only no-automation wording', () => { + const context = resolvePrSourceContext({ + body: ` +## Workflow Source + +Started from: +- [x] Human-only +`, + }); + + assert.equal(context.sourceType, SOURCE_TYPES.MANUAL_REMOTE); + assert.equal(context.noAutomation, true); +}); + test('sourceTypeFromCheckedTemplate rejects ambiguous checked source choices', () => { const body = ` ## Workflow Source diff --git a/.github/scripts/agents_pr_meta_update_body.js b/.github/scripts/agents_pr_meta_update_body.js index d84f89d47..d3e00b73a 100644 --- a/.github/scripts/agents_pr_meta_update_body.js +++ b/.github/scripts/agents_pr_meta_update_body.js @@ -1055,8 +1055,7 @@ function resolveNonIssueWorkflowSourceContextForBodySync(pr = {}, issueNumber = if (!explicitNonIssueSourceContext) { return null; } - const explicitIssueNumbers = extractExplicitIssueSyncNumbers(pr); - if (issueNumber && explicitIssueNumbers.has(Number(issueNumber))) { + if (issueNumber != null) { return null; } return explicitNonIssueSourceContext; diff --git a/.github/scripts/source_context.js b/.github/scripts/source_context.js index be77e4015..22cba6179 100644 --- a/.github/scripts/source_context.js +++ b/.github/scripts/source_context.js @@ -49,7 +49,7 @@ const CHECKBOX_SOURCE_PATTERNS = Object.freeze([ [SOURCE_TYPES.DEPENDABOT, /\bdependabot\b|\bdependency\s+update\b/i], ]); -const NO_AUTOMATION_CHECKBOX_PATTERN = /\bdo\s+not\s+automate\b/i; +const NO_AUTOMATION_CHECKBOX_PATTERN = /\bdo\s+not\s+automate\b|\bhuman[- ]only\b/i; function cleanString(value) { return String(value || '').trim(); diff --git a/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js b/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js index d84f89d47..d3e00b73a 100644 --- a/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js +++ b/templates/consumer-repo/.github/scripts/agents_pr_meta_update_body.js @@ -1055,8 +1055,7 @@ function resolveNonIssueWorkflowSourceContextForBodySync(pr = {}, issueNumber = if (!explicitNonIssueSourceContext) { return null; } - const explicitIssueNumbers = extractExplicitIssueSyncNumbers(pr); - if (issueNumber && explicitIssueNumbers.has(Number(issueNumber))) { + if (issueNumber != null) { return null; } return explicitNonIssueSourceContext; diff --git a/templates/consumer-repo/.github/scripts/source_context.js b/templates/consumer-repo/.github/scripts/source_context.js index be77e4015..22cba6179 100644 --- a/templates/consumer-repo/.github/scripts/source_context.js +++ b/templates/consumer-repo/.github/scripts/source_context.js @@ -49,7 +49,7 @@ const CHECKBOX_SOURCE_PATTERNS = Object.freeze([ [SOURCE_TYPES.DEPENDABOT, /\bdependabot\b|\bdependency\s+update\b/i], ]); -const NO_AUTOMATION_CHECKBOX_PATTERN = /\bdo\s+not\s+automate\b/i; +const NO_AUTOMATION_CHECKBOX_PATTERN = /\bdo\s+not\s+automate\b|\bhuman[- ]only\b/i; function cleanString(value) { return String(value || '').trim(); From ef3879ccf0773df0d8fe6b4a2ef73913860f36bb Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Mon, 27 Apr 2026 05:48:32 -0500 Subject: [PATCH 2/2] fix: address consumer sync review blockers --- .../agents-pr-meta-update-body.test.js | 17 + docs/LABELS.md | 10 +- scripts/aggregate_agent_metrics.py | 22 +- templates/consumer-repo/docs/LABELS.md | 349 +++++++++++++----- .../scripts/aggregate_agent_metrics.py | 22 +- tests/scripts/test_aggregate_agent_metrics.py | 29 +- 6 files changed, 334 insertions(+), 115 deletions(-) diff --git a/.github/scripts/__tests__/agents-pr-meta-update-body.test.js b/.github/scripts/__tests__/agents-pr-meta-update-body.test.js index 61ee68901..31f563604 100644 --- a/.github/scripts/__tests__/agents-pr-meta-update-body.test.js +++ b/.github/scripts/__tests__/agents-pr-meta-update-body.test.js @@ -593,6 +593,23 @@ test('resolveNonIssueWorkflowSourceContextForBodySync yields to branch-inferred assert.equal(prReferenceContext, null); }); +test('resolveNonIssueWorkflowSourceContextForBodySync preserves explicit non-issue markers without issue number', () => { + const context = resolveNonIssueWorkflowSourceContextForBodySync( + { + body: [ + '', + '', + ].join('\n'), + head: { ref: 'codex/local-request' }, + title: 'Follow-up', + }, + null, + ); + + assert.equal(context.sourceType, 'local_request'); + assert.equal(context.sourceRef, 'codex-thread-2026-04-26'); +}); + test('resolveNonIssueWorkflowSourceContextForBodySync yields to explicit issue references', () => { const context = resolveNonIssueWorkflowSourceContextForBodySync( { diff --git a/docs/LABELS.md b/docs/LABELS.md index e59637470..d7ae3d6e2 100644 --- a/docs/LABELS.md +++ b/docs/LABELS.md @@ -263,7 +263,7 @@ Workflows source classification without forcing a GitHub issue. | `workflow:source-dependabot` | Pull Requests | PR source is Dependabot or dependency automation. | `workflow:source-review-followup` | Pull Requests | PR source is review feedback follow-up. | `workflow:source-direct-pr` | Pull Requests | PR was started directly on GitHub without a source issue. -| `workflow:no-automation` | Pull Requests | Automation should not manage the PR unless checks fail. +| `workflow:no-automation` | Pull Requests | Fully opts the PR out of automation management and automation-triggered follow-up actions. | `workflow:source-needed` | Pull Requests | Source context is missing or ambiguous. The Workflow Source table is validated as a three-column Markdown table so label @@ -416,7 +416,7 @@ These labels are used for categorization but do not trigger workflows. ### `needs-formatting` -**Applies to:** Issues +**Applies to:** Issues **Effect:** Indicates the issue needs formatting to AGENT_ISSUE_TEMPLATE structure. @@ -448,6 +448,12 @@ These labels are used for categorization but do not trigger workflows. | (none) | `agents:optimize` | Analyzes and posts suggestions | `agents:optimize` | `agents:apply-suggestions` | Applies suggestions, adds `agents:formatted` | `agents:formatted` | `agent:codex` | Issue ready for agent processing +| `agents:auto-pilot` | `runner:` | Auto-pilot uses the selected runner when it dispatches work +| (none) | `agents:keepalive` | Enables keepalive monitoring for an agent PR +| `agents:keepalive` | `agents:paused` | Pauses keepalive and agent dispatch until resumed +| `agents:paused` | (removed) | Keepalive can resume on the next eligible event +| Merged PR with verifier report | `verify:create-issue` | Creates a verifier follow-up issue +| Merged PR with verifier report | `verify:create-new-pr` | Creates and bootstraps a verifier follow-up PR --- diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index e1359c5c2..b60849c81 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -21,9 +21,21 @@ _DEFAULT_ARTIFACT_SELECTION_PATH = "artifacts/metric-artifacts-selection.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "" -_NULL_EQUIVALENT_TOKENS = {"", "none", "null", "nil", "n/a", "na", "undefined", "unknown"} +_NULL_EQUIVALENT_TOKENS = { + "", + "0", + "false", + "none", + "null", + "nil", + "n/a", + "na", + "off", + "disabled", + "undefined", + "unknown", +} _KNOWN_VERIFIER_MODES = {"checkbox", "compare", "evaluate"} -_MODEL_METADATA_REQUIRED_MODES = {"checkbox", "compare"} _TERMINAL_ARTIFACT_FAMILIES = ( "review-thread-terminal-disposition", "verifier-terminal-disposition", @@ -472,13 +484,9 @@ def _normalize_verifier_mode(value: Any) -> str: return verifier_mode -def _is_known_verifier_mode(value: Any) -> bool: - return _normalize_verifier_mode(value) in _KNOWN_VERIFIER_MODES - - def _verifier_mode_requires_model_metadata(entry: dict[str, Any]) -> bool: verifier_mode = _normalize_verifier_mode(entry.get("verifier_mode")) - return verifier_mode in _MODEL_METADATA_REQUIRED_MODES + return bool(verifier_mode) and verifier_mode != "evaluate" def _summarise_keepalive(entries: list[dict[str, Any]]) -> dict[str, Any]: diff --git a/templates/consumer-repo/docs/LABELS.md b/templates/consumer-repo/docs/LABELS.md index 498bbaba6..bf77f8276 100644 --- a/templates/consumer-repo/docs/LABELS.md +++ b/templates/consumer-repo/docs/LABELS.md @@ -4,19 +4,28 @@ This document describes all labels that trigger automated workflows or affect CI ## Quick Reference -| Label | Trigger | Effect | -|-------|---------|--------| -| `autofix` | PR labeled | Triggers automated code fixes | -| `autofix:clean` | PR labeled | Triggers clean-mode autofix (more aggressive) | -| `agent:codex` | Issue labeled | Triggers Codex agent assignment | -| `agent:codex-invite` | Issue labeled | Invites Codex agent to participate | -| `agent:needs-attention` | Auto-applied | Indicates agent needs human intervention | -| `agents:format` | Issue labeled | Formats raw issue into AGENT_ISSUE_TEMPLATE | -| `agents:formatted` | Auto-applied | Issue has been formatted by LangChain | -| `status:ready` | Issue labeled | Marks issue as ready for agent processing | -| `verify:checkbox` | PR labeled | Runs verifier checkbox mode after merge | -| `verify:evaluate` | PR labeled | Runs verifier evaluation mode after merge | -| `verify:compare` | PR labeled | Runs verifier comparison mode after merge | +| Label | Trigger | Effect +|-------|---------|-------- +| `autofix` | PR labeled | Triggers automated code fixes +| `autofix:clean` | PR labeled | Triggers clean-mode autofix (more aggressive) +| `agent:codex` | Issue labeled | Triggers Codex agent assignment +| `agent:codex-invite` | Issue labeled | Invites Codex agent to participate +| `agent:needs-attention` | Auto-applied | Indicates agent needs human intervention +| `status:ready` | Issue labeled | Marks issue as ready for agent processing +| `agents:format` | Issue labeled | Direct issue formatting +| `agents:formatted` | Auto-applied | Indicates issue has been formatted +| `agents:optimize` | Issue labeled | Analyzes issue and posts suggestions +| `agents:apply-suggestions` | Issue labeled | Applies optimization suggestions +| `agents:auto-pilot` | Issue labeled | Runs issue-to-PR automation +| `agents:auto-pilot-pause` | Issue labeled | Pauses auto-pilot dispatch +| `agents:paused` | PR labeled | Pauses keepalive loop on PR +| `agents:keepalive` | PR labeled | Enables keepalive loop on PR +| `runner:` | Issue labeled | Selects an auto-pilot runner without triggering issue intake +| `verify:checkbox` | PR labeled | Runs verifier checkbox mode after merge +| `verify:evaluate` | PR labeled | Runs verifier evaluation mode after merge +| `verify:compare` | PR labeled | Runs verifier comparison mode after merge +| `verify:create-issue` | PR labeled | Creates follow-up issue from verification +| `verify:create-new-pr` | PR labeled | Creates follow-up issue and PR from verification --- @@ -106,74 +115,18 @@ This document describes all labels that trigger automated workflows or affect CI --- -### `agents:format` - -**Applies to:** Issues - -**Trigger:** When applied to an issue with unstructured content - -**Effect:** -1. Triggers the LangChain Issue Formatter workflow (`agents-issue-optimizer.yml`) -2. Parses the raw issue body using LLM (with regex fallback) -3. Rewrites the issue into the structured AGENT_ISSUE_TEMPLATE format: - - **Why** - Purpose/motivation - - **Scope** - Boundaries of work - - **Non-Goals** - Explicitly out of scope - - **Tasks** - Actionable checkboxes - - **Acceptance Criteria** - Testable success conditions - - **Implementation Notes** - Technical hints (if any) -4. Preserves the original issue body in a collapsed `
` block -5. Removes `agents:format` and applies `agents:formatted` on success - -**Prerequisites:** -- Issue must have non-empty body text -- Repository must have `OPENAI_API_KEY` secret for LLM mode (optional) - -**Workflow:** `agents-issue-optimizer.yml` (LangChain Issue Optimizer) - -**Example:** - -Before (raw issue): -``` -The TripPlan contract in models.py is confusing - it has different shapes in -different places. Let's pick one canonical form and update all the converters. -``` - -After (formatted): -```markdown -## Why -To establish a single canonical TripPlan contract and eliminate inconsistencies... - -## Scope -- Review all TripPlan definitions in models.py -- Choose canonical form -- Update converters - -## Non-Goals -- Changing the underlying data model -- Modifying external API contracts - -## Tasks -- [ ] Audit all TripPlan shapes in codebase -- [ ] Select canonical form -- [ ] Update converters to use canonical form -... -``` - ---- - -### `agents:formatted` +### `runner:` **Applies to:** Issues -**Trigger:** Automatically applied by `agents-issue-optimizer.yml` +**Trigger:** When applied to an issue that also has `agents:auto-pilot` **Effect:** -1. Indicates the issue has been processed by the LangChain formatter -2. Replaces the `agents:format` label -3. No additional workflow triggers (safe terminal state) +1. Overrides the agent that auto-pilot will use (`runner:claude`, `runner:codex`, etc.) +2. Auto-pilot reads this label during capability/check-pr steps and adds the matching `agent:` label when it dispatches the belt +3. Does **not** trigger the issue intake workflow by itself, so manual `agent:` behavior is unaffected -**Note:** This is a status label, not a trigger label. Do not apply manually. +**Workflow:** `agents-auto-pilot.yml` --- @@ -212,22 +165,109 @@ To establish a single canonical TripPlan contract and eliminate inconsistencies. --- +## Issue Formatting Labels (LangChain Enhancement) + +These labels control the LangChain-powered issue formatting pipeline introduced in #484. + +### `agents:format` + +**Applies to:** Issues + +**Trigger:** When applied to an issue + +**Effect:** +1. Automatically formats the raw issue body into the AGENT_ISSUE_TEMPLATE structure +2. Uses LLM (GitHub Models API) with fallback to regex-based formatting +3. Adds proper sections: Why, Scope, Non-Goals, Tasks, Acceptance Criteria, Implementation Notes +4. Converts task items to checkboxes +5. Replaces the issue body with formatted version +6. Removes `agents:format` label and adds `agents:formatted` + +**Use Case:** Quick, one-step formatting without review. Best for issues that are already well-structured but need template compliance. + +**Workflow:** `agents-issue-optimizer.yml` + +--- + +### `agents:formatted` + +**Applies to:** Issues + +**Trigger:** Automatically applied after formatting completes + +**Effect:** +1. Indicates the issue has been formatted to AGENT_ISSUE_TEMPLATE +2. Signals the issue is ready for agent processing +3. Prevents re-formatting (workflows skip issues with this label) + +**Note:** This is a result label, not a trigger label. Do not apply manually. + +**Workflow:** Applied by `agents-issue-optimizer.yml` + +--- + +### `agents:optimize` + +**Applies to:** Issues + +**Trigger:** When applied to an issue + +**Effect:** +1. Analyzes the issue for agent compatibility and formatting quality +2. Posts a comment with suggestions including: + - Tasks that are too broad (should be split) + - Tasks the agent cannot complete (with reasons) + - Subjective acceptance criteria (with objective alternatives) + - Missing sections or formatting issues +3. Includes embedded JSON with structured suggestions +4. Prompts user to add `agents:apply-suggestions` to apply changes + +**Use Case:** Two-step formatting with human review. Best for issues needing significant restructuring. + +**Workflow:** `agents-issue-optimizer.yml` + +--- + +### `agents:apply-suggestions` + +**Applies to:** Issues + +**Trigger:** When applied to an issue that has received optimization suggestions + +**Prerequisites:** +- Issue must have a comment with optimization suggestions (from `agents:optimize`) +- The suggestions comment must contain valid JSON in `` marker + +**Effect:** +1. Extracts approved suggestions from the analysis comment +2. Applies all suggestions to reformat the issue body +3. Moves blocked tasks to "## Deferred Tasks (Requires Human)" section +4. Removes both `agents:optimize` and `agents:apply-suggestions` labels +5. Adds `agents:formatted` label + +**Workflow:** `agents-issue-optimizer.yml` + +--- + ## Workflow Source Labels These labels let direct GitHub PRs and non-issue-origin PRs integrate with Workflows source classification without forcing a GitHub issue. -| Label | Applies to | Effect | -|-------|------------|--------| -| `workflow:source-issue` | Pull Requests | PR source is a GitHub issue. | -| `workflow:source-local-request` | Pull Requests | PR source is a local Codex/user request. | -| `workflow:source-automation` | Pull Requests | PR source is an automation or workflow run. | -| `workflow:source-sync` | Pull Requests | PR source is a sync or maintenance campaign. | -| `workflow:source-dependabot` | Pull Requests | PR source is Dependabot or dependency automation. | -| `workflow:source-review-followup` | Pull Requests | PR source is review feedback follow-up. | -| `workflow:source-direct-pr` | Pull Requests | PR was started directly on GitHub without a source issue. | -| `workflow:no-automation` | Pull Requests | Automation should not manage the PR unless checks fail. | -| `workflow:source-needed` | Pull Requests | Source context is missing or ambiguous. | +| Label | Applies to | Effect +|-------|------------|-------- +| `workflow:source-issue` | Pull Requests | PR source is a GitHub issue. +| `workflow:source-local-request` | Pull Requests | PR source is a local Codex/user request. +| `workflow:source-automation` | Pull Requests | PR source is an automation or workflow run. +| `workflow:source-sync` | Pull Requests | PR source is a sync or maintenance campaign. +| `workflow:source-dependabot` | Pull Requests | PR source is Dependabot or dependency automation. +| `workflow:source-review-followup` | Pull Requests | PR source is review feedback follow-up. +| `workflow:source-direct-pr` | Pull Requests | PR was started directly on GitHub without a source issue. +| `workflow:no-automation` | Pull Requests | Fully opts the PR out of automation management and automation-triggered follow-up actions. +| `workflow:source-needed` | Pull Requests | Source context is missing or ambiguous. + +The Workflow Source table is validated as a three-column Markdown table so label +rows do not introduce an extra empty column in GitHub rendering. Use these labels as a backup to the PR template's Workflow Source section. If a PR has no linked issue and no valid Workflow Source, the PR metadata automation @@ -276,6 +316,114 @@ These labels trigger the post-merge verifier workflow on a merged PR. --- +### `verify:create-issue` + +**Applies to:** Pull Requests + +**Trigger:** When applied to a merged PR that has verification feedback + +**Prerequisites:** +- PR must be merged +- PR must have a verification comment (from `verify:evaluate` or `verify:compare`) + +**Effect:** +1. Extracts concerns and low scores from verification feedback +2. Creates a new follow-up issue with: + - Link to original PR + - Extracted concerns from verification + - Scores below 7/10 + - Suggested tasks for addressing issues +3. Posts comment on original PR linking to new issue +4. Removes the `verify:create-issue` label after completion +5. Adds `agents:optimize` label to new issue for agent formatting + +**Use Case:** User-triggered creation of follow-up work from verification feedback. Replaces automatic issue creation which was too aggressive. + +**Workflow:** `agents-verify-to-issue-v2.yml` + +--- + +### `verify:create-new-pr` + +**Applies to:** Pull Requests + +**Trigger:** When applied to a merged PR that has verification feedback + +**Prerequisites:** +- PR must be merged +- PR must already have verification context (for example from `verify:evaluate` or `verify:compare`) + +**Effect:** +1. Creates a follow-up issue from verification concerns +2. Creates and bootstraps a follow-up PR for that issue +3. Removes `verify:create-new-pr` label after processing + +**Workflow:** `agents-verify-to-new-pr.yml` + +--- + +## Keepalive Control Labels + +### `agents:paused` + +**Applies to:** Pull Requests + +**Trigger:** When applied to a PR with active keepalive + +**Effect:** +1. Pauses all keepalive activity on the PR +2. Agent will not be dispatched until label is removed +3. Useful for manual intervention or debugging + +**To Resume:** Remove the `agents:paused` label. + +**Workflow:** `agents-keepalive-loop.yml` + +--- + +### `agents:keepalive` + +**Applies to:** Pull Requests + +**Trigger:** When applied to a PR + +**Effect:** +1. Enables the keepalive loop for the PR +2. Agent continues working until all tasks are complete +3. Tracks progress and updates PR status + +**Prerequisites:** +- PR must have an `agent:*` label +- Gate workflow must pass + +**Workflow:** `agents-keepalive-loop.yml` + +--- + +## Informational Labels + +These labels are used for categorization but do not trigger workflows. + +### `follow-up` + +**Applies to:** Issues + +**Effect:** Indicates this issue was created as follow-up to another issue or PR. + +**Applied by:** `agents-verify-to-issue-v2.yml` workflow + +--- + +### `needs-formatting` + +**Applies to:** Issues + +**Effect:** Indicates the issue needs formatting to AGENT_ISSUE_TEMPLATE structure. + +**Applied by:** Issue lint workflow (when enabled) + +--- + ## CI/Build Labels ### `skip-ci` (if configured) @@ -288,17 +436,24 @@ These labels trigger the post-merge verifier workflow on a merged PR. ## Label Interaction Matrix -| Existing Label | New Label Added | Result | -|---------------|-----------------|--------| -| (none) | `autofix` | Triggers autofix | -| `autofix` | `autofix:clean` | May trigger clean mode | -| (none) | `agent:codex` | Triggers agent assignment | -| `agent:codex` | `agent:codex-invite` | Sends agent invitation | -| `agent:codex` | `status:ready` | Agent begins processing | -| `agent:needs-attention` | (removed) | Agent resumes processing | -| (none) | `agents:format` | Triggers LangChain formatter | -| `agents:format` | (auto-removed) | Label replaced by `agents:formatted` | -| `agents:formatted` | `agent:codex` | Ready for agent assignment | +| Existing Label | New Label Added | Result +|---------------|-----------------|-------- +| (none) | `autofix` | Triggers autofix +| `autofix` | `autofix:clean` | May trigger clean mode +| (none) | `agent:codex` | Triggers agent assignment +| `agent:codex` | `agent:codex-invite` | Sends agent invitation +| `agent:codex` | `status:ready` | Agent begins processing +| `agent:needs-attention` | (removed) | Agent resumes processing +| (none) | `agents:format` | Direct formatting +| (none) | `agents:optimize` | Analyzes and posts suggestions +| `agents:optimize` | `agents:apply-suggestions` | Applies suggestions, adds `agents:formatted` +| `agents:formatted` | `agent:codex` | Issue ready for agent processing +| `agents:auto-pilot` | `runner:` | Auto-pilot uses the selected runner when it dispatches work +| (none) | `agents:keepalive` | Enables keepalive monitoring for an agent PR +| `agents:keepalive` | `agents:paused` | Pauses keepalive and agent dispatch until resumed +| `agents:paused` | (removed) | Keepalive can resume on the next eligible event +| Merged PR with verifier report | `verify:create-issue` | Creates a verifier follow-up issue +| Merged PR with verifier report | `verify:create-new-pr` | Creates and bootstraps a verifier follow-up PR --- diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index e1359c5c2..b60849c81 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -21,9 +21,21 @@ _DEFAULT_ARTIFACT_SELECTION_PATH = "artifacts/metric-artifacts-selection.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "" -_NULL_EQUIVALENT_TOKENS = {"", "none", "null", "nil", "n/a", "na", "undefined", "unknown"} +_NULL_EQUIVALENT_TOKENS = { + "", + "0", + "false", + "none", + "null", + "nil", + "n/a", + "na", + "off", + "disabled", + "undefined", + "unknown", +} _KNOWN_VERIFIER_MODES = {"checkbox", "compare", "evaluate"} -_MODEL_METADATA_REQUIRED_MODES = {"checkbox", "compare"} _TERMINAL_ARTIFACT_FAMILIES = ( "review-thread-terminal-disposition", "verifier-terminal-disposition", @@ -472,13 +484,9 @@ def _normalize_verifier_mode(value: Any) -> str: return verifier_mode -def _is_known_verifier_mode(value: Any) -> bool: - return _normalize_verifier_mode(value) in _KNOWN_VERIFIER_MODES - - def _verifier_mode_requires_model_metadata(entry: dict[str, Any]) -> bool: verifier_mode = _normalize_verifier_mode(entry.get("verifier_mode")) - return verifier_mode in _MODEL_METADATA_REQUIRED_MODES + return bool(verifier_mode) and verifier_mode != "evaluate" def _summarise_keepalive(entries: list[dict[str, Any]]) -> dict[str, Any]: diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index 6272a4603..33d2b0918 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -1093,7 +1093,7 @@ def test_verifier_summary_ignores_missing_model_metadata_for_null_equivalent_mod assert verifier["unknown_verifier_modes"] == Counter() -def test_verifier_summary_ignores_missing_model_metadata_for_unrecognized_mode( +def test_verifier_summary_counts_missing_model_metadata_for_unrecognized_mode( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv( @@ -1122,10 +1122,35 @@ def test_verifier_summary_ignores_missing_model_metadata_for_unrecognized_mode( ] ) - assert verifier["missing_verifier_model_metadata"] == Counter() + assert verifier["missing_verifier_model_metadata"]["verifier-error"] == 2 assert verifier["unknown_verifier_modes"]["compare-lite"] == 2 +def test_verifier_summary_treats_false_mode_as_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv( + "TERMINAL_DISPOSITION_VERIFIER_MODEL_METADATA_REQUIRED_AFTER", + "2026-04-26T04:25:00Z", + ) + + verifier = aggregate_agent_metrics._summarise_verifier( + [ + { + "schema": "workflows-terminal-disposition/v1", + "artifact_family": "verifier-terminal-disposition", + "run_id": "24948023784", + "pr_number": 1879, + "disposition": "verifier-error", + "verifier_mode": False, + }, + ] + ) + + assert verifier["missing_verifier_model_metadata"] == Counter() + assert verifier["unknown_verifier_modes"] == Counter() + + def test_verifier_summary_counts_missing_model_metadata_for_non_evaluate_mode( monkeypatch: pytest.MonkeyPatch, ) -> None: