diff --git a/.github/scripts/issue_format.py b/.github/scripts/issue_format.py index 07c73aa4f..84fe6d37d 100644 --- a/.github/scripts/issue_format.py +++ b/.github/scripts/issue_format.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Validate a GitHub issue body against the fleet's AGENT_ISSUE_FORMAT contract. Synced to every consumer repo by `maint-68-sync-consumer-repos.yml`. This is the @@ -23,9 +24,14 @@ Rules mirror docs/AGENT_ISSUE_FORMAT.md rather than inventing a parallel standard: Tasks and Acceptance Criteria are REQUIRED; Why / Scope / -Implementation Notes / Non-Goals are recommended; and at least one acceptance -criterion must name a real test, runnable command, or observable verification -gate. +Implementation Notes / Non-Goals are reported as recommended; and at least one +acceptance criterion must name a real test, runnable command, or observable +verification gate. + +Recommended sections are advisory: their absence is reported to help authors +improve an issue, but does not change the exit code or route an otherwise valid +work order through the optimizer. Keeping that distinction prevents the guard +from flagging well-formed work orders solely for an optional heading. `_headings()` skips fenced code blocks, and that is load-bearing rather than cosmetic. Without it a body whose ONLY "Tasks" and "Acceptance Criteria" lines @@ -61,15 +67,15 @@ r"|\bgh workflow run\b|\bgh run\b" r"|\bcurl\b|\bHTTP [1-5]\d\d\b" r"|\b(?:API|endpoint|request|response)\s+(?:returns?|responds with)\s+[1-5]\d\d(?:\s+status)?\b" - r"|\bsmoke\b|\bverif)", + r"|\bsmoke\b|\bverif\w*)", re.I, ) BANNED_ADJECTIVES = ("clean", "nice", "good", "fast", "better", "intuitive", "polished") -def _headings(body: str) -> list[tuple[str, int]]: +def _headings(body: str) -> list[tuple[str, int, int]]: """Return markdown headings outside fenced code blocks with line indexes.""" - out: list[tuple[str, int]] = [] + out: list[tuple[str, int, int]] = [] fence: tuple[str, int] | None = None for i, line in enumerate(body.splitlines()): fence_match = re.match(r"\s{0,3}(`{3,}|~{3,})", line) @@ -84,20 +90,21 @@ def _headings(body: str) -> list[tuple[str, int]]: continue heading = re.match(r"\s{0,3}(#{1,6})\s+(.+?)\s*#*\s*$", line) if heading: - out.append((heading.group(2).strip().strip(":").lower(), i)) + out.append((heading.group(2).strip().strip(":").lower(), i, len(heading.group(1)))) return out def _find(body: str, aliases: tuple[str, ...]) -> int | None: - for text, idx in _headings(body): - if any(text == alias or text.startswith(alias) for alias in aliases): + for text, idx, _ in _headings(body): + if text in aliases: return idx return None def _section_text(body: str, start: int) -> str: lines = body.splitlines() - following = [idx for _, idx in _headings(body) if idx > start] + start_level = next(level for _, idx, level in _headings(body) if idx == start) + following = [idx for _, idx, level in _headings(body) if idx > start and level <= start_level] end = following[0] if following else len(lines) return "\n".join(lines[start + 1 : end]) diff --git a/.github/workflows/agents-issue-format-guard.yml b/.github/workflows/agents-issue-format-guard.yml index afbdc8aef..055453387 100644 --- a/.github/workflows/agents-issue-format-guard.yml +++ b/.github/workflows/agents-issue-format-guard.yml @@ -21,6 +21,14 @@ concurrency: jobs: check: + if: >- + github.event_name != 'issues' || + github.event.action == 'opened' || github.event.action == 'edited' || + github.event.action == 'reopened' || + ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && + (github.event.label.name == 'agents:auto-pilot-pause' || + github.event.label.name == 'needs-human' || + github.event.label.name == 'tracker:durable' || github.event.label.name == 'wontfix')) runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -68,8 +76,10 @@ jobs: exit "$rc" fi - - name: Invalidate stale format completion while held - if: github.event.action == 'edited' && steps.issue.outputs.held == 'true' + - name: Invalidate stale format completion after an invalid edit + if: >- + steps.issue.outputs.exempt != 'true' && github.event.action == 'edited' && + steps.validate.outputs.rc == '1' env: GH_TOKEN: ${{ github.token }} NUMBER: ${{ steps.issue.outputs.number }} @@ -78,7 +88,7 @@ jobs: if gh issue view "$NUMBER" --repo "$GITHUB_REPOSITORY" --json labels \ --jq '.labels[].name' | grep -qx 'agents:formatted'; then gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "agents:formatted" - echo "Held issue was edited — cleared agents:formatted so resume revalidates it." + echo "Invalid issue edit — cleared agents:formatted before rerouting." fi - name: Route non-conforming issue to the optimizer @@ -104,6 +114,11 @@ jobs: echo "" } | gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body-file - fi + if gh issue view "$NUMBER" --repo "$GITHUB_REPOSITORY" --json labels \ + --jq '.labels[].name' | grep -qx 'agents:formatted'; then + gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "agents:formatted" \ + || echo "::warning::could not remove agents:formatted" + fi gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "agents:format" \ || echo "::warning::could not apply agents:format (label missing in this repo?)" # GITHUB_TOKEN label edits do not start issues:labeled workflows; dispatch is explicit. @@ -113,9 +128,8 @@ jobs: - name: Restore format completion after an unheld revalidation if: >- steps.issue.outputs.exempt != 'true' && steps.validate.outputs.rc == '0' && - (steps.issue.outputs.held == 'true' || - (github.event.action == 'unlabeled' && - (github.event.label.name == 'agents:auto-pilot-pause' || github.event.label.name == 'needs-human'))) + github.event.action == 'unlabeled' && + (github.event.label.name == 'agents:auto-pilot-pause' || github.event.label.name == 'needs-human') env: GH_TOKEN: ${{ github.token }} NUMBER: ${{ steps.issue.outputs.number }} @@ -126,8 +140,9 @@ jobs: echo "Issue remains held — leaving agents:formatted cleared." exit 0 fi - gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "agents:formatted" - echo "Held edit revalidated after resume — restored agents:formatted." + gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "agents:formatted" \ + || echo "::warning::could not apply agents:formatted (label missing in this repo?)" + echo "Unheld issue revalidated — restored agents:formatted." - name: Clear the stale format trigger if: steps.issue.outputs.exempt != 'true' && steps.issue.outputs.held != 'true' && steps.validate.outputs.rc == '0' diff --git a/docs/ci/WORKFLOWS.md b/docs/ci/WORKFLOWS.md index 813d44f4a..61a9ec6d3 100644 --- a/docs/ci/WORKFLOWS.md +++ b/docs/ci/WORKFLOWS.md @@ -154,7 +154,7 @@ Consumer default note: `agents-pr-meta-v4.yml` is a Workflows-repo service workf * [`agents-63-issue-intake.yml`](../../.github/workflows/agents-63-issue-intake.yml) is the canonical front door. It now listens for `agent:codex` labels directly and routes both label triggers and ChatGPT sync requests through the shared normalization pipeline. * [`agents-64-verify-agent-assignment.yml`](../../.github/workflows/agents-64-verify-agent-assignment.yml) validates that labelled issues retain an approved agent assignee and publishes the verification outputs. * [`agents-issue-optimizer.yml`](../../.github/workflows/agents-issue-optimizer.yml) runs issue optimization passes when `agents:optimize` or `agents:apply-suggestions` labels are applied. -* [`agents-issue-format-guard.yml`](../../.github/workflows/agents-issue-format-guard.yml) validates opened, edited, reopened, and hold-label changes against `AGENT_ISSUE_FORMAT`; durable/wontfix and bot issues remain exempt. `agents:auto-pilot-pause` and `needs-human` hold optimizer dispatch, clear a stale `agents:formatted` marker when an issue is edited while held, and revalidate when the hold is removed. Non-conforming unheld work is explicitly dispatched to the optimizer's `format` phase. +* [`agents-issue-format-guard.yml`](../../.github/workflows/agents-issue-format-guard.yml) validates opened, edited, reopened, and hold/exemption-label changes against `AGENT_ISSUE_FORMAT`; durable/wontfix and bot issues remain exempt. The validator requires the `Tasks` and `Acceptance Criteria` structure plus concrete task/verification evidence, preserves nested Markdown subsections, and reports `Why`, `Scope`, `Implementation Notes`, and `Non-Goals` as advisory. `agents:auto-pilot-pause` and `needs-human` hold optimizer dispatch, invalid edits clear a stale `agents:formatted` marker, and hold removal revalidates before restoring it. Non-conforming unheld work is explicitly dispatched to the optimizer's `format` phase. * [`agents-moderate-connector.yml`](../../.github/workflows/agents-moderate-connector.yml) moderates connector-authored PR comments, enforcing repository allow/deny lists and applying the debugging label when deletions occur. * [`agents-guard.yml`](../../.github/workflows/agents-guard.yml) applies repository-level guardrails before agent workflows run. * [`pr-46-dependency-repair-contract.yml`](../../.github/workflows/pr-46-dependency-repair-contract.yml) keeps clean Renovate/Dependabot branches bot-owned and validates the bot-delta first commit on agent-owned dependency repair promotions. See [`docs/ops/DEPENDENCY_REPAIR_PROMOTION.md`](../ops/DEPENDENCY_REPAIR_PROMOTION.md) for the lane-selection contract and marker. diff --git a/templates/consumer-repo/.github/scripts/issue_format.py b/templates/consumer-repo/.github/scripts/issue_format.py index 07c73aa4f..84fe6d37d 100644 --- a/templates/consumer-repo/.github/scripts/issue_format.py +++ b/templates/consumer-repo/.github/scripts/issue_format.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Validate a GitHub issue body against the fleet's AGENT_ISSUE_FORMAT contract. Synced to every consumer repo by `maint-68-sync-consumer-repos.yml`. This is the @@ -23,9 +24,14 @@ Rules mirror docs/AGENT_ISSUE_FORMAT.md rather than inventing a parallel standard: Tasks and Acceptance Criteria are REQUIRED; Why / Scope / -Implementation Notes / Non-Goals are recommended; and at least one acceptance -criterion must name a real test, runnable command, or observable verification -gate. +Implementation Notes / Non-Goals are reported as recommended; and at least one +acceptance criterion must name a real test, runnable command, or observable +verification gate. + +Recommended sections are advisory: their absence is reported to help authors +improve an issue, but does not change the exit code or route an otherwise valid +work order through the optimizer. Keeping that distinction prevents the guard +from flagging well-formed work orders solely for an optional heading. `_headings()` skips fenced code blocks, and that is load-bearing rather than cosmetic. Without it a body whose ONLY "Tasks" and "Acceptance Criteria" lines @@ -61,15 +67,15 @@ r"|\bgh workflow run\b|\bgh run\b" r"|\bcurl\b|\bHTTP [1-5]\d\d\b" r"|\b(?:API|endpoint|request|response)\s+(?:returns?|responds with)\s+[1-5]\d\d(?:\s+status)?\b" - r"|\bsmoke\b|\bverif)", + r"|\bsmoke\b|\bverif\w*)", re.I, ) BANNED_ADJECTIVES = ("clean", "nice", "good", "fast", "better", "intuitive", "polished") -def _headings(body: str) -> list[tuple[str, int]]: +def _headings(body: str) -> list[tuple[str, int, int]]: """Return markdown headings outside fenced code blocks with line indexes.""" - out: list[tuple[str, int]] = [] + out: list[tuple[str, int, int]] = [] fence: tuple[str, int] | None = None for i, line in enumerate(body.splitlines()): fence_match = re.match(r"\s{0,3}(`{3,}|~{3,})", line) @@ -84,20 +90,21 @@ def _headings(body: str) -> list[tuple[str, int]]: continue heading = re.match(r"\s{0,3}(#{1,6})\s+(.+?)\s*#*\s*$", line) if heading: - out.append((heading.group(2).strip().strip(":").lower(), i)) + out.append((heading.group(2).strip().strip(":").lower(), i, len(heading.group(1)))) return out def _find(body: str, aliases: tuple[str, ...]) -> int | None: - for text, idx in _headings(body): - if any(text == alias or text.startswith(alias) for alias in aliases): + for text, idx, _ in _headings(body): + if text in aliases: return idx return None def _section_text(body: str, start: int) -> str: lines = body.splitlines() - following = [idx for _, idx in _headings(body) if idx > start] + start_level = next(level for _, idx, level in _headings(body) if idx == start) + following = [idx for _, idx, level in _headings(body) if idx > start and level <= start_level] end = following[0] if following else len(lines) return "\n".join(lines[start + 1 : end]) diff --git a/templates/consumer-repo/.github/workflows/agents-issue-format-guard.yml b/templates/consumer-repo/.github/workflows/agents-issue-format-guard.yml index afbdc8aef..055453387 100644 --- a/templates/consumer-repo/.github/workflows/agents-issue-format-guard.yml +++ b/templates/consumer-repo/.github/workflows/agents-issue-format-guard.yml @@ -21,6 +21,14 @@ concurrency: jobs: check: + if: >- + github.event_name != 'issues' || + github.event.action == 'opened' || github.event.action == 'edited' || + github.event.action == 'reopened' || + ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && + (github.event.label.name == 'agents:auto-pilot-pause' || + github.event.label.name == 'needs-human' || + github.event.label.name == 'tracker:durable' || github.event.label.name == 'wontfix')) runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -68,8 +76,10 @@ jobs: exit "$rc" fi - - name: Invalidate stale format completion while held - if: github.event.action == 'edited' && steps.issue.outputs.held == 'true' + - name: Invalidate stale format completion after an invalid edit + if: >- + steps.issue.outputs.exempt != 'true' && github.event.action == 'edited' && + steps.validate.outputs.rc == '1' env: GH_TOKEN: ${{ github.token }} NUMBER: ${{ steps.issue.outputs.number }} @@ -78,7 +88,7 @@ jobs: if gh issue view "$NUMBER" --repo "$GITHUB_REPOSITORY" --json labels \ --jq '.labels[].name' | grep -qx 'agents:formatted'; then gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "agents:formatted" - echo "Held issue was edited — cleared agents:formatted so resume revalidates it." + echo "Invalid issue edit — cleared agents:formatted before rerouting." fi - name: Route non-conforming issue to the optimizer @@ -104,6 +114,11 @@ jobs: echo "" } | gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body-file - fi + if gh issue view "$NUMBER" --repo "$GITHUB_REPOSITORY" --json labels \ + --jq '.labels[].name' | grep -qx 'agents:formatted'; then + gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --remove-label "agents:formatted" \ + || echo "::warning::could not remove agents:formatted" + fi gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "agents:format" \ || echo "::warning::could not apply agents:format (label missing in this repo?)" # GITHUB_TOKEN label edits do not start issues:labeled workflows; dispatch is explicit. @@ -113,9 +128,8 @@ jobs: - name: Restore format completion after an unheld revalidation if: >- steps.issue.outputs.exempt != 'true' && steps.validate.outputs.rc == '0' && - (steps.issue.outputs.held == 'true' || - (github.event.action == 'unlabeled' && - (github.event.label.name == 'agents:auto-pilot-pause' || github.event.label.name == 'needs-human'))) + github.event.action == 'unlabeled' && + (github.event.label.name == 'agents:auto-pilot-pause' || github.event.label.name == 'needs-human') env: GH_TOKEN: ${{ github.token }} NUMBER: ${{ steps.issue.outputs.number }} @@ -126,8 +140,9 @@ jobs: echo "Issue remains held — leaving agents:formatted cleared." exit 0 fi - gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "agents:formatted" - echo "Held edit revalidated after resume — restored agents:formatted." + gh issue edit "$NUMBER" --repo "$GITHUB_REPOSITORY" --add-label "agents:formatted" \ + || echo "::warning::could not apply agents:formatted (label missing in this repo?)" + echo "Unheld issue revalidated — restored agents:formatted." - name: Clear the stale format trigger if: steps.issue.outputs.exempt != 'true' && steps.issue.outputs.held != 'true' && steps.validate.outputs.rc == '0' diff --git a/tests/scripts/test_issue_format.py b/tests/scripts/test_issue_format.py index 1751f1dc1..c652e0391 100644 --- a/tests/scripts/test_issue_format.py +++ b/tests/scripts/test_issue_format.py @@ -6,6 +6,11 @@ import pytest +VALID_CONTEXT = ( + "## Why\nCurrent evidence\n\n## Scope\nBounded scope\n\n" + "## Implementation Notes\nDetails\n\n## Non-Goals\nNo expansion\n\n" +) + def _validator(): path = Path(".github/scripts/issue_format.py") @@ -38,7 +43,8 @@ def test_checkbox_and_subjective_errors_are_non_conforming() -> None: def test_runner_and_curl_are_accepted_gates() -> None: validator = _validator() report = validator.validate( - "## Tasks\n- [ ] Implement it\n\n## Acceptance Criteria\n- gh run watch succeeds\n- curl endpoint returns 200\n" + VALID_CONTEXT + + "## Tasks\n- [ ] Implement it\n\n## Acceptance Criteria\n- gh run watch succeeds\n- curl endpoint returns 200\n" ) assert report.ok @@ -56,7 +62,7 @@ def test_runner_and_curl_are_accepted_gates() -> None: def test_api_status_sentence_is_an_accepted_gate(criterion: str) -> None: validator = _validator() report = validator.validate( - f"## Tasks\n- [ ] Implement it\n\n## Acceptance Criteria\n- {criterion}\n" + VALID_CONTEXT + f"## Tasks\n- [ ] Implement it\n\n## Acceptance Criteria\n- {criterion}\n" ) assert report.ok @@ -74,3 +80,35 @@ def test_implementation_notes_is_a_recommended_section() -> None: "## Tasks\n- [ ] Implement it\n\n## Acceptance Criteria\n- pytest tests/test_x.py passes\n" ) assert "Implementation Notes" in report.missing_recommended + assert report.ok + + +def test_implementation_notes_does_not_satisfy_tasks() -> None: + validator = _validator() + report = validator.validate( + "## Implementation Notes\n- [ ] Not a task\n\n" + "## Acceptance Criteria\n- pytest tests/test_x.py passes\n" + ) + assert "Tasks" in report.missing_required + + +def test_nested_headings_remain_inside_their_parent_section() -> None: + validator = _validator() + report = validator.validate( + "## Why\nCurrent evidence\n\n## Scope\nBounded scope\n\n" + "## Tasks\n### Backend\n- [ ] Implement it\n\n" + "## Acceptance Criteria\n### Verification\n- pytest tests/test_x.py passes\n\n" + "## Implementation Notes\nDetails\n\n## Non-Goals\nNo expansion\n" + ) + assert report.ok + + +def test_verify_is_an_accepted_gate() -> None: + validator = _validator() + report = validator.validate( + "## Why\nCurrent evidence\n\n## Scope\nBounded scope\n\n" + "## Tasks\n- [ ] Implement it\n\n" + "## Acceptance Criteria\n- Verify the endpoint response\n\n" + "## Implementation Notes\nDetails\n\n## Non-Goals\nNo expansion\n" + ) + assert report.ok