From 1900a7824a4ee8c437b20baa929b4eb117836716 Mon Sep 17 00:00:00 2001 From: stranske Date: Fri, 26 Dec 2025 16:15:45 +0000 Subject: [PATCH 01/16] feat: Add agents-keepalive-loop.yml for consumer repos - Add agents-keepalive-loop.yml template with dual-checkout pattern - Update sync workflow to include keepalive-loop in synced files - Fix sync logic to create new files that don't exist in consumer repos The keepalive-loop workflow is CRITICAL for CLI Codex to function. Without it, the Gate can pass but Codex never gets triggered to continue working on the PR. Closes the keepalive pipeline gap identified in Travel-Plan-Permission. --- .../maint-68-sync-consumer-repos.yml | 6 +- .../workflows/agents-keepalive-loop.yml | 412 ++++++++++++++++++ 2 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml diff --git a/.github/workflows/maint-68-sync-consumer-repos.yml b/.github/workflows/maint-68-sync-consumer-repos.yml index 8d5a863a6..9ffd3a4d0 100644 --- a/.github/workflows/maint-68-sync-consumer-repos.yml +++ b/.github/workflows/maint-68-sync-consumer-repos.yml @@ -112,6 +112,7 @@ jobs: "agents-orchestrator.yml:agents-orchestrator.yml" "agents-orchestrator.yml:agents-70-orchestrator.yml" "agents-pr-meta.yml:agents-pr-meta.yml" + "agents-keepalive-loop.yml:agents-keepalive-loop.yml" "autofix.yml:autofix.yml" "pr-00-gate.yml:pr-00-gate.yml" ) @@ -295,6 +296,7 @@ jobs: SYNC_TEMPLATES=( "agents-orchestrator.yml" "agents-pr-meta.yml" + "agents-keepalive-loop.yml" "autofix.yml" "pr-00-gate.yml" ) @@ -308,7 +310,9 @@ jobs: elif [ -f ".github/workflows/agents-70-orchestrator.yml" ] && [ "$file" = "agents-orchestrator.yml" ]; then target=".github/workflows/agents-70-orchestrator.yml" else - continue + # Create new file if it doesn't exist (for new workflows like keepalive-loop) + mkdir -p .github/workflows + target=".github/workflows/$file" fi if [ -f "$template" ]; then diff --git a/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml b/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml new file mode 100644 index 000000000..e0dad2cec --- /dev/null +++ b/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml @@ -0,0 +1,412 @@ +# CLI Codex Keepalive Loop - triggers Codex after Gate passes +# This workflow is CRITICAL for the keepalive pipeline to function with CLI Codex. +# +# How it works: +# 1. Gate workflow completes on a PR +# 2. This workflow evaluates if keepalive should continue +# 3. If yes, calls reusable-codex-run.yml to run Codex CLI +# 4. Codex makes changes, pushes commits +# 5. Gate runs again, loop continues until tasks complete +# +# Copy this file to: .github/workflows/agents-keepalive-loop.yml +# +# Required secrets: +# - CODEX_AUTH_JSON: JSON auth for Codex CLI (ChatGPT subscription) +# - WORKFLOWS_APP_ID: GitHub App ID (alternative to CODEX_AUTH_JSON) +# - WORKFLOWS_APP_PRIVATE_KEY: GitHub App private key +# +# Required files: +# - .github/codex/prompts/keepalive_next_task.md +# - .github/codex/AGENT_INSTRUCTIONS.md +name: Agents Keepalive Loop + +on: + workflow_run: + workflows: ["Gate"] + types: [completed] + pull_request: + types: + - labeled + +permissions: + contents: write + pull-requests: write + actions: write + +concurrency: + group: keepalive-${{ github.event.workflow_run.pull_requests[0].number || github.event.pull_request.number || github.run_id }} + cancel-in-progress: false + +jobs: + evaluate: + name: Evaluate keepalive loop + runs-on: ubuntu-latest + environment: agent-standard + outputs: + pr_number: ${{ steps.evaluate.outputs.pr_number }} + pr_ref: ${{ steps.evaluate.outputs.pr_ref }} + head_sha: ${{ steps.evaluate.outputs.head_sha }} + action: ${{ steps.evaluate.outputs.action }} + reason: ${{ steps.evaluate.outputs.reason }} + gate_conclusion: ${{ steps.evaluate.outputs.gate_conclusion }} + iteration: ${{ steps.evaluate.outputs.iteration }} + max_iterations: ${{ steps.evaluate.outputs.max_iterations }} + failure_threshold: ${{ steps.evaluate.outputs.failure_threshold }} + tasks_total: ${{ steps.evaluate.outputs.tasks_total }} + tasks_unchecked: ${{ steps.evaluate.outputs.tasks_unchecked }} + keepalive_enabled: ${{ steps.evaluate.outputs.keepalive_enabled }} + autofix_enabled: ${{ steps.evaluate.outputs.autofix_enabled }} + has_agent_label: ${{ steps.evaluate.outputs.has_agent_label }} + agent_type: ${{ steps.evaluate.outputs.agent_type }} + task_appendix: ${{ steps.evaluate.outputs.task_appendix }} + trace: ${{ steps.evaluate.outputs.trace }} + start_ts: ${{ steps.timestamps.outputs.start_ts }} + security_blocked: ${{ steps.security_gate.outputs.blocked }} + security_reason: ${{ steps.security_gate.outputs.reason }} + steps: + # Dual checkout pattern: consumer repo for context, Workflows repo for scripts + - name: Checkout consumer repository + uses: actions/checkout@v4 + with: + path: consumer + + - name: Checkout Workflows scripts + uses: actions/checkout@v4 + with: + repository: stranske/Workflows + ref: main + sparse-checkout: | + .github/scripts + sparse-checkout-cone-mode: false + path: workflows-lib + fetch-depth: 1 + + - name: Set scripts path + run: | + echo "WORKFLOWS_SCRIPTS_PATH=${GITHUB_WORKSPACE}/workflows-lib/.github/scripts" >> "$GITHUB_ENV" + + - name: Capture timestamps + id: timestamps + run: echo "start_ts=$(date -u +%s)" >> "$GITHUB_OUTPUT" + + - name: Security gate - prompt injection guard + id: security_gate + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const scriptsPath = process.env.WORKFLOWS_SCRIPTS_PATH; + const { evaluatePromptInjectionGuard } = require(`${scriptsPath}/prompt_injection_guard.js`); + + // Resolve PR from event context + const payload = context.payload || {}; + let prNumber = 0; + let pr = null; + + if (context.eventName === 'pull_request' && payload.pull_request) { + prNumber = payload.pull_request.number; + pr = payload.pull_request; + } else if (context.eventName === 'workflow_run' && payload.workflow_run) { + const prs = payload.workflow_run.pull_requests || []; + if (prs[0]?.number) { + prNumber = prs[0].number; + const { data } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + pr = data; + } + } + + if (!pr) { + core.setOutput('blocked', 'false'); + core.setOutput('reason', 'no-pr-context'); + return; + } + + const result = await evaluatePromptInjectionGuard({ + github, + context, + pr, + actor: context.actor, + promptContent: pr.body || '', + core, + }); + + core.setOutput('blocked', String(result.blocked)); + core.setOutput('reason', result.reason); + + if (result.blocked) { + core.warning(`Security gate blocked: ${result.reason}`); + } + + - name: Evaluate keepalive conditions + id: evaluate + if: steps.security_gate.outputs.blocked != 'true' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const scriptsPath = process.env.WORKFLOWS_SCRIPTS_PATH; + const { evaluateKeepaliveLoop } = require(`${scriptsPath}/keepalive_loop.js`); + + const result = await evaluateKeepaliveLoop({ + github, + context, + core, + payload: context.payload, + }); + + const output = result.outputs || {}; + for (const [key, value] of Object.entries(output)) { + core.setOutput(key, value); + } + // Task appendix needs special handling due to multiline content + core.setOutput('task_appendix', result.taskAppendix || ''); + + preflight: + name: Verify secrets available + needs: evaluate + if: needs.evaluate.outputs.action == 'run' + runs-on: ubuntu-latest + environment: agent-standard + outputs: + secrets_ok: ${{ steps.check.outputs.secrets_ok }} + steps: + - name: Check secrets + id: check + env: + HAS_CODEX_AUTH: ${{ secrets.CODEX_AUTH_JSON != '' }} + HAS_APP_ID: ${{ secrets.WORKFLOWS_APP_ID != '' }} + HAS_APP_KEY: ${{ secrets.WORKFLOWS_APP_PRIVATE_KEY != '' }} + run: | + echo "CODEX_AUTH_JSON present: $HAS_CODEX_AUTH" + echo "WORKFLOWS_APP_ID present: $HAS_APP_ID" + echo "WORKFLOWS_APP_PRIVATE_KEY present: $HAS_APP_KEY" + if [ "$HAS_CODEX_AUTH" = "true" ] || [ "$HAS_APP_ID" = "true" ]; then + echo "secrets_ok=true" >> "$GITHUB_OUTPUT" + else + echo "::error::Neither CODEX_AUTH_JSON nor WORKFLOWS_APP_ID is set. Cannot run Codex." + echo "secrets_ok=false" >> "$GITHUB_OUTPUT" + exit 1 + fi + + # Mark agent as running before starting the actual work + mark-running: + name: Mark agent running + needs: + - evaluate + - preflight + if: needs.evaluate.outputs.action == 'run' + runs-on: ubuntu-latest + steps: + - name: Checkout Workflows scripts + uses: actions/checkout@v4 + with: + repository: stranske/Workflows + ref: main + sparse-checkout: | + .github/scripts + sparse-checkout-cone-mode: false + fetch-depth: 1 + + - name: Update summary with running status + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { markAgentRunning } = require('./.github/scripts/keepalive_loop.js'); + const inputs = { + pr_number: '${{ needs.evaluate.outputs.pr_number }}', + agent_type: '${{ needs.evaluate.outputs.agent_type }}', + iteration: '${{ needs.evaluate.outputs.iteration }}', + max_iterations: '${{ needs.evaluate.outputs.max_iterations }}', + tasks_total: '${{ needs.evaluate.outputs.tasks_total }}', + tasks_unchecked: '${{ needs.evaluate.outputs.tasks_unchecked }}', + trace: '${{ needs.evaluate.outputs.trace }}', + run_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', + }; + await markAgentRunning({ github, context, core, inputs }); + + # Run Codex CLI for agent:codex PRs + run-codex: + name: Keepalive next task (Codex) + needs: + - evaluate + - preflight + - mark-running + if: needs.evaluate.outputs.agent_type == 'codex' + uses: stranske/Workflows/.github/workflows/reusable-codex-run.yml@main + secrets: + CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }} + WORKFLOWS_APP_ID: ${{ secrets.WORKFLOWS_APP_ID }} + WORKFLOWS_APP_PRIVATE_KEY: ${{ secrets.WORKFLOWS_APP_PRIVATE_KEY }} + with: + skip: ${{ needs.evaluate.outputs.action != 'run' }} + prompt_file: .github/codex/prompts/keepalive_next_task.md + mode: keepalive + pr_number: ${{ needs.evaluate.outputs.pr_number }} + pr_ref: ${{ needs.evaluate.outputs.pr_ref }} + appendix: ${{ needs.evaluate.outputs.task_appendix }} + iteration: ${{ needs.evaluate.outputs.iteration }} + + summary: + name: Update keepalive summary + needs: + - evaluate + - preflight + - run-codex + if: always() && needs.evaluate.outputs.pr_number != '' && needs.evaluate.outputs.pr_number != '0' + runs-on: ubuntu-latest + environment: agent-standard + steps: + - name: Checkout Workflows scripts + uses: actions/checkout@v4 + with: + repository: stranske/Workflows + ref: main + sparse-checkout: | + .github/scripts + sparse-checkout-cone-mode: false + fetch-depth: 1 + + - name: Emit keepalive metrics + id: keepalive-metrics + env: + PR_NUMBER: ${{ needs.evaluate.outputs.pr_number }} + ACTION: ${{ needs.evaluate.outputs.action }} + REASON: ${{ needs.evaluate.outputs.reason }} + GATE_CONCLUSION: ${{ needs.evaluate.outputs.gate_conclusion }} + ITERATION: ${{ needs.evaluate.outputs.iteration }} + MAX_ITERATIONS: ${{ needs.evaluate.outputs.max_iterations }} + TASKS_TOTAL: ${{ needs.evaluate.outputs.tasks_total }} + TASKS_UNCHECKED: ${{ needs.evaluate.outputs.tasks_unchecked }} + START_TS: ${{ needs.evaluate.outputs.start_ts }} + run: | + set -euo pipefail + + now=$(date -u +%s) + if [[ "${START_TS:-}" =~ ^[0-9]+$ ]]; then + duration=$(( now - START_TS )) + if [ "$duration" -lt 0 ]; then duration=0; fi + else + duration=0 + fi + + tasks_total=${TASKS_TOTAL:-0} + tasks_unchecked=${TASKS_UNCHECKED:-0} + if ! [[ "$tasks_total" =~ ^-?[0-9]+$ ]]; then tasks_total=0; fi + if ! [[ "$tasks_unchecked" =~ ^-?[0-9]+$ ]]; then tasks_unchecked=0; fi + tasks_completed=$(( tasks_total - tasks_unchecked )) + if [ "$tasks_completed" -lt 0 ]; then tasks_completed=0; fi + + metrics_json=$(jq -n \ + --arg pr "${PR_NUMBER:-0}" \ + --arg iteration "${ITERATION:-0}" \ + --arg action "${ACTION:-}" \ + --arg stop_reason "${REASON:-}" \ + --arg gate_conclusion "${GATE_CONCLUSION:-}" \ + --arg tasks_total "$tasks_total" \ + --arg tasks_completed "$tasks_completed" \ + --arg duration "$duration" \ + '{ + pr_number: ($pr | tonumber? // 0), + iteration_count: ($iteration | tonumber? // 0), + action: $action, + stop_reason: $stop_reason, + gate_conclusion: $gate_conclusion, + tasks_total: ($tasks_total | tonumber? // 0), + tasks_completed: ($tasks_completed | tonumber? // 0), + duration_seconds: ($duration | tonumber? // 0) + }') + + { + echo '### Keepalive metrics' + echo '' + echo '| Field | Value |' + echo '| --- | --- |' + echo "| pr_number | $(echo "$metrics_json" | jq -r '.pr_number') |" + echo "| iteration_count | $(echo "$metrics_json" | jq -r '.iteration_count') |" + echo "| action | $(echo "$metrics_json" | jq -r '.action') |" + echo "| stop_reason | $(echo "$metrics_json" | jq -r '.stop_reason') |" + echo "| gate_conclusion | $(echo "$metrics_json" | jq -r '.gate_conclusion') |" + echo "| tasks_total | $(echo "$metrics_json" | jq -r '.tasks_total') |" + echo "| tasks_completed | $(echo "$metrics_json" | jq -r '.tasks_completed') |" + echo "| duration_seconds | $(echo "$metrics_json" | jq -r '.duration_seconds') |" + } >> "$GITHUB_STEP_SUMMARY" + + echo "$metrics_json" >> keepalive-metrics.ndjson + + - name: Upload keepalive metrics artifact + uses: actions/upload-artifact@v4 + with: + name: keepalive-metrics + path: keepalive-metrics.ndjson + retention-days: 30 + if-no-files-found: ignore + + - name: Auto-reconcile task checkboxes + if: needs.run-codex.outputs.changes-made == 'true' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { autoReconcileTasks } = require('./.github/scripts/keepalive_loop.js'); + + const prNumber = Number('${{ needs.evaluate.outputs.pr_number }}') || 0; + const beforeSha = '${{ needs.evaluate.outputs.head_sha }}'; + const headSha = '${{ needs.run-codex.outputs.commit-sha }}'; + + if (!prNumber || !beforeSha || !headSha) { + core.info('Missing required inputs for task reconciliation'); + return; + } + + core.info(`Auto-reconciling tasks for PR #${prNumber}`); + core.info(`Comparing ${beforeSha.slice(0, 7)} → ${headSha.slice(0, 7)}`); + + const result = await autoReconcileTasks({ + github, context, prNumber, baseSha: beforeSha, headSha, core + }); + + if (result.updated) { + core.info(`✅ ${result.details}`); + core.notice(`Auto-checked ${result.tasksChecked} task(s) based on commit analysis`); + } else { + core.info(`ℹ️ ${result.details}`); + } + + core.setOutput('tasks_checked', result.tasksChecked); + core.setOutput('reconciliation_details', result.details); + + - name: Update summary comment + uses: actions/github-script@v7 + env: + CODEX_SUMMARY: ${{ needs.run-codex.outputs.final-message-summary || '' }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { updateKeepaliveLoopSummary } = require('./.github/scripts/keepalive_loop.js'); + const inputs = { + pr_number: Number('${{ needs.evaluate.outputs.pr_number }}') || 0, + action: '${{ needs.evaluate.outputs.action }}', + reason: '${{ needs.evaluate.outputs.reason }}', + gate_conclusion: '${{ needs.evaluate.outputs.gate_conclusion }}', + iteration: Number('${{ needs.evaluate.outputs.iteration }}') || 0, + max_iterations: Number('${{ needs.evaluate.outputs.max_iterations }}') || 0, + failure_threshold: Number('${{ needs.evaluate.outputs.failure_threshold }}') || 3, + tasks_total: Number('${{ needs.evaluate.outputs.tasks_total }}') || 0, + tasks_unchecked: Number('${{ needs.evaluate.outputs.tasks_unchecked }}') || 0, + keepalive_enabled: '${{ needs.evaluate.outputs.keepalive_enabled }}', + autofix_enabled: '${{ needs.evaluate.outputs.autofix_enabled }}', + agent_type: '${{ needs.evaluate.outputs.agent_type }}', + trace: '${{ needs.evaluate.outputs.trace }}', + run_result: '${{ needs.run-codex.result }}', + agent_exit_code: '${{ needs.run-codex.outputs.exit-code }}', + agent_changes_made: '${{ needs.run-codex.outputs.changes-made }}', + agent_commit_sha: '${{ needs.run-codex.outputs.commit-sha }}', + agent_files_changed: '${{ needs.run-codex.outputs.files-changed }}', + agent_summary: process.env.CODEX_SUMMARY || '', + }; + await updateKeepaliveLoopSummary({ github, context, core, inputs }); From 5f5ea2a312b15ddb405ec78bcc96025b9cd8cd12 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 26 Dec 2025 16:17:43 +0000 Subject: [PATCH 02/16] chore(codex-autofix): apply updates (PR #183) --- codex-output.md | 10 ++---- codex-prompt.md | 87 ++++++++----------------------------------------- 2 files changed, 16 insertions(+), 81 deletions(-) diff --git a/codex-output.md b/codex-output.md index 63b3747ab..0e15ebf40 100644 --- a/codex-output.md +++ b/codex-output.md @@ -1,9 +1,3 @@ -Added an automatic default metrics log path for keepalive iterations running under GitHub Actions so records are appended without extra inputs, and expanded keepalive-loop tests to verify the default log behavior and clean up the workspace file. Updated the acceptance checkbox in `codex-prompt.md` after verifying the new logging behavior. Changes are in `.github/scripts/keepalive_loop.js`, `.github/scripts/__tests__/keepalive-loop.test.js`, and `codex-prompt.md`. +I couldn’t identify a failing check to fix. The run is marked “cancelled” and “Failing jobs: none reported,” and there are no CI logs in the repo to inspect. I didn’t make any changes. -Tests: `node --test .github/scripts/__tests__/keepalive-loop.test.js` - -Workflow update is still blocked by policy: I can’t edit `.github/workflows/agents-orchestrator.yml` in this run. Please add a `needs-human` label and a PR comment instructing the workflow update to call `scripts/keepalive_metrics_collector.py` after keepalive completes (or set `KEEPALIVE_METRICS_PATH` for the loop). - -Next steps: -1) Have a human update `.github/workflows/agents-orchestrator.yml` to invoke the metrics collector or set `KEEPALIVE_METRICS_PATH`. -2) Run the full selftest CI to satisfy the remaining acceptance criterion. \ No newline at end of file +If you can share the failing job output (or re-run CI and paste the failure summary), I can target the fix. If you want me to try something local, tell me which job or test name to run. \ No newline at end of file diff --git a/codex-prompt.md b/codex-prompt.md index 9d0169c2e..2759524c3 100644 --- a/codex-prompt.md +++ b/codex-prompt.md @@ -101,80 +101,21 @@ You should assume you're running in `agent-standard` unless explicitly told othe # Task Prompt -# Keepalive Next Task +# Autofix from CI failure -Your objective is to satisfy the **Acceptance Criteria** by completing each **Task** within the defined **Scope**. +You are Codex running in autofix mode after a CI failure. Use the available logs and repository context to repair the failing checks. -**This round you MUST:** -1. Implement actual code or test changes that advance at least one incomplete task toward acceptance. -2. Commit meaningful source code (.py, .yml, .js, etc.)—not just status/docs updates. -3. Mark a task checkbox complete ONLY after verifying the implementation works. -4. Focus on the FIRST unchecked task unless blocked, then move to the next. - -**Guidelines:** -- Keep edits scoped to the current task rather than reshaping the entire PR. -- Use repository instructions, conventions, and tests to validate work. -- Prefer small, reviewable commits; leave clear notes when follow-up is required. -- Do NOT work on unrelated improvements until all PR tasks are complete. - -**The Tasks and Acceptance Criteria are provided in the appendix below.** Work through them in order. +Guidance: +- Inspect the latest CI output provided by the caller (logs or summaries) to pinpoint the root cause. +- Focus on minimal, targeted fixes that unblock the failing job. +- Leave diagnostic breadcrumbs when a failure cannot be reproduced or fully addressed. +- Re-run or suggest the smallest relevant checks to verify the fix. ## Run context ---- -## PR Tasks and Acceptance Criteria - -**Progress:** 11/14 tasks complete, 3 remaining - -### ⚠️ IMPORTANT: Task Reconciliation Required - -The previous iteration changed **2 file(s)** but did not update task checkboxes. - -**Before continuing, you MUST:** -1. Review the recent commits to understand what was changed -2. Determine which task checkboxes should be marked complete -3. Update the PR body to check off completed tasks -4. Then continue with remaining tasks - -_Failure to update checkboxes means progress is not being tracked properly._ - -### Scope -- [ ] The keepalive loop currently tracks iteration counts in PR state comments, but there is no aggregated view of keepalive performance across PRs. Operators cannot easily answer questions like: -- [ ] - How many iterations does a typical PR require before completion? -- [ ] - What percentage of PRs complete within the 5-iteration limit vs timing out? -- [ ] - Which error categories are most common during keepalive runs? -- [ ] - What is the average time from PR open to keepalive completion? -- [ ] This issue adds structured metrics collection and a summary dashboard to provide observability into the keepalive pipeline health. -- [ ] ### Current Behavior -- [ ] - Iteration count stored in PR state comment (hidden marker) -- [ ] - No aggregation across PRs -- [ ] - Error classification exists but is not persisted -- [ ] - No historical trend data -- [ ] ### Desired Behavior -- [ ] - Each keepalive iteration appends a metrics record to an NDJSON log -- [ ] - Metrics include: PR number, iteration, action taken, error category, duration, tasks completed -- [ ] - A summary script aggregates metrics into a dashboard report -- [ ] - Dashboard shows success rates, iteration distributions, and error breakdowns - -### Tasks -Complete these in order. Mark checkbox done ONLY after implementation is verified: - -- [x] Define metrics schema in `docs/keepalive/METRICS_SCHEMA.md` with fields for PR number, iteration, timestamp, action, error_category, duration_ms, tasks_total, tasks_complete -- [x] Create `scripts/keepalive_metrics_collector.py` to append structured metrics to `keepalive-metrics.ndjson` -- [x] Integrate metrics collection into `.github/scripts/keepalive_loop.js` to emit metrics after each iteration -- [x] Create `scripts/keepalive_metrics_dashboard.py` that reads the NDJSON log and outputs a markdown summary table -- [x] Add tests for metrics collector (schema validation, append behavior) -- [x] Add tests for dashboard generator (aggregation logic, edge cases) -- [ ] Update `.github/workflows/agents-orchestrator.yml` to call metrics collector after keepalive completes - -### Acceptance Criteria -The PR is complete when ALL of these are satisfied: - -- [x] Metrics schema is documented with field descriptions and example records -- [x] Each keepalive iteration logs a structured record with all required fields -- [x] Dashboard script produces a valid markdown table with success rate, avg iterations, and error breakdown -- [x] Tests cover metrics schema validation and reject malformed records -- [x] Tests cover dashboard aggregation with empty, single, and multi-record inputs -- [x] Integration smoke test confirms metrics are written during actual keepalive runs -- [ ] Selftest CI passes - ---- +Gate run: https://github.com/stranske/Workflows/actions/runs/20525528375 +Conclusion: cancelled +PR: #183 +Head SHA: 1900a7824a4ee8c437b20baa929b4eb117836716 +Autofix attempts for this head: 1 / 3 +Fix scope: src/, tests/, tools/, scripts/, agents/, templates/, .github/ +Failing jobs: none reported. From f6cc5677acb9e89caac6a9a66f4144b9f599e947 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Dec 2025 16:24:04 +0000 Subject: [PATCH 03/16] Add required sections to agent task issue form --- .github/ISSUE_TEMPLATE/agent_task.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/agent_task.yml b/.github/ISSUE_TEMPLATE/agent_task.yml index d05639c97..dee38b82a 100644 --- a/.github/ISSUE_TEMPLATE/agent_task.yml +++ b/.github/ISSUE_TEMPLATE/agent_task.yml @@ -22,16 +22,32 @@ body: placeholder: Provide the context and link supporting material. validations: required: true + - type: textarea + id: scope + attributes: + label: Scope + description: What is in scope for this work? Call out files, systems, or workflows to touch. + placeholder: Describe the intended scope. + validations: + required: true + - type: textarea + id: tasks + attributes: + label: Tasks + description: List the concrete tasks Codex should complete. + placeholder: "- [ ] Task 1\n- [ ] Task 2" + validations: + required: true - type: textarea id: goals attributes: - label: Goals - description: List the concrete outcomes this task should deliver. - placeholder: Bullet the acceptance criteria or deliverables. + label: Acceptance criteria + description: Describe what must be true for this work to be considered complete. + placeholder: "- [ ] Criterion 1\n- [ ] Criterion 2" validations: required: true - type: textarea - id: scope + id: guardrails attributes: label: Out of scope / guardrails description: Clarify any boundaries Codex must respect (files to avoid, limits, etc.). From f0e2c5a28269944a9026608957bb9b013b06042f Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Dec 2025 16:24:58 +0000 Subject: [PATCH 04/16] Align agent task template with required sections --- .github/ISSUE_TEMPLATE/agent-task.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/agent-task.md b/.github/ISSUE_TEMPLATE/agent-task.md index b4ce793a0..68cc9949d 100644 --- a/.github/ISSUE_TEMPLATE/agent-task.md +++ b/.github/ISSUE_TEMPLATE/agent-task.md @@ -13,14 +13,22 @@ assignees: '' section header options, and examples of valid issue structures. --> -## Goal +## Why +## Scope + + ## Constraints +## Tasks + +- [ ] Task 1 +- [ ] Task 2 + ## Expected outputs -## Success criteria +## Acceptance criteria From c7f20865489be364e331cf90b8cdb79e1803d001 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 26 Dec 2025 16:25:09 +0000 Subject: [PATCH 05/16] chore(codex-keepalive): apply updates (PR #183) --- codex-output.md | 12 ++++++++-- codex-prompt.md | 48 ++++++++++++++++++++++++++++------------ keepalive-metrics.ndjson | 1 + 3 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 keepalive-metrics.ndjson diff --git a/codex-output.md b/codex-output.md index 0e15ebf40..00d47ae21 100644 --- a/codex-output.md +++ b/codex-output.md @@ -1,3 +1,11 @@ -I couldn’t identify a failing check to fix. The run is marked “cancelled” and “Failing jobs: none reported,” and there are no CI logs in the repo to inspect. I didn’t make any changes. +Added the missing Scope/Tasks/Acceptance criteria sections to the agent task issue form and aligned the legacy agent task template with the required section headers so new issues match the format guide. Changes are in `.github/ISSUE_TEMPLATE/agent_task.yml` and `.github/ISSUE_TEMPLATE/agent-task.md`. -If you can share the failing job output (or re-run CI and paste the failure summary), I can target the fix. If you want me to try something local, tell me which job or test name to run. \ No newline at end of file +Tests: +- `python -m pytest` (timed out at 120s; first failure in `tests/test_integration_repo_template.py` due to pip needing network access) +- `python -m pytest tests/test_integration_repo_template.py -q` (failed: pip install of `setuptools>=64` could not reach the network) + +I also see unrelated working tree changes: modified `codex-prompt.md` and untracked `keepalive-metrics.ndjson`. I didn’t create these intentionally—how would you like to proceed with them? + +Next steps: +1. Tell me whether to keep, remove, or ignore `codex-prompt.md` and `keepalive-metrics.ndjson`. +2. Re-run `python -m pytest` in an environment with package index access, or I can try the test again with cached deps if available. \ No newline at end of file diff --git a/codex-prompt.md b/codex-prompt.md index 2759524c3..5f8893b50 100644 --- a/codex-prompt.md +++ b/codex-prompt.md @@ -101,21 +101,41 @@ You should assume you're running in `agent-standard` unless explicitly told othe # Task Prompt -# Autofix from CI failure +# Keepalive Next Task -You are Codex running in autofix mode after a CI failure. Use the available logs and repository context to repair the failing checks. +Your objective is to satisfy the **Acceptance Criteria** by completing each **Task** within the defined **Scope**. -Guidance: -- Inspect the latest CI output provided by the caller (logs or summaries) to pinpoint the root cause. -- Focus on minimal, targeted fixes that unblock the failing job. -- Leave diagnostic breadcrumbs when a failure cannot be reproduced or fully addressed. -- Re-run or suggest the smallest relevant checks to verify the fix. +**This round you MUST:** +1. Implement actual code or test changes that advance at least one incomplete task toward acceptance. +2. Commit meaningful source code (.py, .yml, .js, etc.)—not just status/docs updates. +3. Mark a task checkbox complete ONLY after verifying the implementation works. +4. Focus on the FIRST unchecked task unless blocked, then move to the next. + +**Guidelines:** +- Keep edits scoped to the current task rather than reshaping the entire PR. +- Use repository instructions, conventions, and tests to validate work. +- Prefer small, reviewable commits; leave clear notes when follow-up is required. +- Do NOT work on unrelated improvements until all PR tasks are complete. + +**The Tasks and Acceptance Criteria are provided in the appendix below.** Work through them in order. ## Run context -Gate run: https://github.com/stranske/Workflows/actions/runs/20525528375 -Conclusion: cancelled -PR: #183 -Head SHA: 1900a7824a4ee8c437b20baa929b4eb117836716 -Autofix attempts for this head: 1 / 3 -Fix scope: src/, tests/, tools/, scripts/, agents/, templates/, .github/ -Failing jobs: none reported. +--- +## PR Tasks and Acceptance Criteria + +**Progress:** 0/2 tasks complete, 2 remaining + +### Scope +- [ ] Scope section missing from source issue. + +### Tasks +Complete these in order. Mark checkbox done ONLY after implementation is verified: + +- [ ] Tasks section missing from source issue. + +### Acceptance Criteria +The PR is complete when ALL of these are satisfied: + +- [ ] Acceptance criteria section missing from source issue. + +--- diff --git a/keepalive-metrics.ndjson b/keepalive-metrics.ndjson new file mode 100644 index 000000000..4c16471d9 --- /dev/null +++ b/keepalive-metrics.ndjson @@ -0,0 +1 @@ +{"pr_number":2468,"iteration":2,"timestamp":"2025-12-26T16:21:39.803Z","action":"run","error_category":"none","duration_ms":1234,"tasks_total":10,"tasks_complete":4} From 73f27fabf831ff0316929ad7a957f2631af8d74c Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Dec 2025 16:34:29 +0000 Subject: [PATCH 06/16] test: cover agent issue templates for required sections --- .../__tests__/issue_template_sections.test.js | 28 +++++++++++++++++++ codex-prompt.md | 20 ++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 .github/scripts/__tests__/issue_template_sections.test.js diff --git a/.github/scripts/__tests__/issue_template_sections.test.js b/.github/scripts/__tests__/issue_template_sections.test.js new file mode 100644 index 000000000..7655e39e1 --- /dev/null +++ b/.github/scripts/__tests__/issue_template_sections.test.js @@ -0,0 +1,28 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const repoRoot = path.resolve(__dirname, '../../..'); +const issueFormPath = path.join(repoRoot, '.github/ISSUE_TEMPLATE/agent_task.yml'); +const issueTemplatePath = path.join(repoRoot, '.github/ISSUE_TEMPLATE/agent-task.md'); + +const readFile = (filePath) => fs.readFileSync(filePath, 'utf8'); + +test('agent task issue form includes Scope/Tasks/Acceptance sections', () => { + const content = readFile(issueFormPath); + + assert.match(content, /label:\s*Scope\b/i); + assert.match(content, /label:\s*Tasks\b/i); + assert.match(content, /label:\s*Acceptance criteria\b/i); +}); + +test('agent task markdown template includes Scope/Tasks/Acceptance sections', () => { + const content = readFile(issueTemplatePath); + + assert.match(content, /^##\s+Scope\b/m); + assert.match(content, /^##\s+Tasks\b/m); + assert.match(content, /^##\s+Acceptance criteria\b/m); +}); diff --git a/codex-prompt.md b/codex-prompt.md index 5f8893b50..128651fe1 100644 --- a/codex-prompt.md +++ b/codex-prompt.md @@ -123,19 +123,31 @@ Your objective is to satisfy the **Acceptance Criteria** by completing each **Ta --- ## PR Tasks and Acceptance Criteria -**Progress:** 0/2 tasks complete, 2 remaining +**Progress:** 2/2 tasks complete, 0 remaining + +### ⚠️ IMPORTANT: Task Reconciliation Required + +The previous iteration changed **3 file(s)** but did not update task checkboxes. + +**Before continuing, you MUST:** +1. Review the recent commits to understand what was changed +2. Determine which task checkboxes should be marked complete +3. Update the PR body to check off completed tasks +4. Then continue with remaining tasks + +_Failure to update checkboxes means progress is not being tracked properly._ ### Scope -- [ ] Scope section missing from source issue. +- [x] Scope section missing from source issue. ### Tasks Complete these in order. Mark checkbox done ONLY after implementation is verified: -- [ ] Tasks section missing from source issue. +- [x] Tasks section missing from source issue. ### Acceptance Criteria The PR is complete when ALL of these are satisfied: -- [ ] Acceptance criteria section missing from source issue. +- [x] Acceptance criteria section missing from source issue. --- From 4f35c446aa12e0995beb2f0dfac6e71a6ccb447b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 26 Dec 2025 16:34:43 +0000 Subject: [PATCH 07/16] chore(codex-keepalive): apply updates (PR #183) --- codex-output.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/codex-output.md b/codex-output.md index 00d47ae21..fadf6cd1f 100644 --- a/codex-output.md +++ b/codex-output.md @@ -1,11 +1,7 @@ -Added the missing Scope/Tasks/Acceptance criteria sections to the agent task issue form and aligned the legacy agent task template with the required section headers so new issues match the format guide. Changes are in `.github/ISSUE_TEMPLATE/agent_task.yml` and `.github/ISSUE_TEMPLATE/agent-task.md`. +Added a focused Node test to enforce required Scope/Tasks/Acceptance sections in the agent issue templates, then marked the PR task checkboxes complete after verifying the templates. Updates are in `.github/scripts/__tests__/issue_template_sections.test.js` and `codex-prompt.md`. Tests: -- `python -m pytest` (timed out at 120s; first failure in `tests/test_integration_repo_template.py` due to pip needing network access) -- `python -m pytest tests/test_integration_repo_template.py -q` (failed: pip install of `setuptools>=64` could not reach the network) - -I also see unrelated working tree changes: modified `codex-prompt.md` and untracked `keepalive-metrics.ndjson`. I didn’t create these intentionally—how would you like to proceed with them? +- `node --test .github/scripts/__tests__/issue_template_sections.test.js` Next steps: -1. Tell me whether to keep, remove, or ignore `codex-prompt.md` and `keepalive-metrics.ndjson`. -2. Re-run `python -m pytest` in an environment with package index access, or I can try the test again with cached deps if available. \ No newline at end of file +1. Run the broader JS test suite: `node --test .github/scripts/__tests__/*.test.js` \ No newline at end of file From 75b113b688d9b102c4dbec947342f097a49f17ee Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Dec 2025 16:43:17 +0000 Subject: [PATCH 08/16] Add scope/tasks/acceptance to Codex issue forms --- .github/ISSUE_TEMPLATE/bug_report_codex.yml | 29 +++++++++++++++++ .../ISSUE_TEMPLATE/feature_request_codex.yml | 16 ++++++++++ codex-prompt.md | 4 +-- .../test_workflow_agents_consolidation.py | 32 +++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report_codex.yml b/.github/ISSUE_TEMPLATE/bug_report_codex.yml index 772c8a630..e6dc59f36 100644 --- a/.github/ISSUE_TEMPLATE/bug_report_codex.yml +++ b/.github/ISSUE_TEMPLATE/bug_report_codex.yml @@ -31,3 +31,32 @@ body: 1. 2. 3. + validations: + required: true + - type: textarea + id: scope + attributes: + label: Scope + description: What is in scope for the fix? Call out files, systems, or workflows to touch. + placeholder: Describe the intended scope. + validations: + required: true + - type: textarea + id: tasks + attributes: + label: Tasks + description: Checklist of concrete work items for Codex to complete. + placeholder: "- [ ] Task 1\n- [ ] Task 2" + validations: + required: true + - type: textarea + id: acceptance + attributes: + label: Acceptance criteria + description: Bullet list of verifiable outcomes for the fix. + value: | + - [ ] A + - [ ] B + - [ ] C + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request_codex.yml b/.github/ISSUE_TEMPLATE/feature_request_codex.yml index 129e1b456..b5a33d7fe 100644 --- a/.github/ISSUE_TEMPLATE/feature_request_codex.yml +++ b/.github/ISSUE_TEMPLATE/feature_request_codex.yml @@ -22,6 +22,22 @@ body: placeholder: e.g., "Add preview of score frame before selection" validations: required: true + - type: textarea + id: scope + attributes: + label: Scope + description: What is in scope for this change? Mention files, systems, or workflows to touch. + placeholder: Describe the intended scope. + validations: + required: true + - type: textarea + id: tasks + attributes: + label: Tasks + description: Checklist of concrete work items for Codex to complete. + placeholder: "- [ ] Task 1\n- [ ] Task 2" + validations: + required: true - type: textarea id: acceptance attributes: diff --git a/codex-prompt.md b/codex-prompt.md index 128651fe1..33ef64996 100644 --- a/codex-prompt.md +++ b/codex-prompt.md @@ -123,11 +123,11 @@ Your objective is to satisfy the **Acceptance Criteria** by completing each **Ta --- ## PR Tasks and Acceptance Criteria -**Progress:** 2/2 tasks complete, 0 remaining +**Progress:** 3/3 tasks complete, 0 remaining ### ⚠️ IMPORTANT: Task Reconciliation Required -The previous iteration changed **3 file(s)** but did not update task checkboxes. +The previous iteration changed **1 file(s)** but did not update task checkboxes. **Before continuing, you MUST:** 1. Review the recent commits to understand what was changed diff --git a/tests/workflows/test_workflow_agents_consolidation.py b/tests/workflows/test_workflow_agents_consolidation.py index a73d88a9d..e0faf7f65 100644 --- a/tests/workflows/test_workflow_agents_consolidation.py +++ b/tests/workflows/test_workflow_agents_consolidation.py @@ -3,6 +3,7 @@ import yaml WORKFLOWS_DIR = Path(".github/workflows") +ISSUE_TEMPLATE_DIR = Path(".github/ISSUE_TEMPLATE") KEEPALIVE_HELPER = Path("scripts/keepalive-runner.js") @@ -12,6 +13,23 @@ def _load_workflow_yaml(name: str) -> dict: return yaml.safe_load(path.read_text(encoding="utf-8")) +def _load_issue_template_yaml(name: str) -> dict: + path = ISSUE_TEMPLATE_DIR / name + assert path.exists(), f"Issue template {name} must exist" + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def _issue_form_entries_by_label(data: dict) -> dict: + entries = {} + for item in data.get("body") or []: + attributes = item.get("attributes") or {} + label = attributes.get("label") + if not label: + continue + entries[str(label).strip().lower()] = item + return entries + + def _workflow_on_section(data: dict) -> dict: return data.get("on") or data.get(True) or {} @@ -509,6 +527,20 @@ def test_agent_task_template_auto_labels_codex(): ), "Agent task template must auto-apply agents + agent:codex labels" +def test_codex_issue_forms_require_scope_tasks_acceptance(): + for name in ("bug_report_codex.yml", "feature_request_codex.yml"): + data = _load_issue_template_yaml(name) + entries = _issue_form_entries_by_label(data) + for required_label in ("scope", "tasks", "acceptance criteria"): + assert ( + required_label in entries + ), f"Issue template {name} must include {required_label} section" + validations = entries[required_label].get("validations") or {} + assert validations.get("required") is True, ( + f"Issue template {name} must require {required_label} section" + ) + + def test_issue_intake_guard_checks_agent_label(): text = (WORKFLOWS_DIR / "agents-63-issue-intake.yml").read_text(encoding="utf-8") # The workflow must check for agent:* prefix in the issue's labels array From 50652518764a2d191d8ab438a812c48a9687a42a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 26 Dec 2025 16:43:32 +0000 Subject: [PATCH 09/16] chore(codex-keepalive): apply updates (PR #183) --- codex-output.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/codex-output.md b/codex-output.md index fadf6cd1f..095f0a0c0 100644 --- a/codex-output.md +++ b/codex-output.md @@ -1,7 +1,10 @@ -Added a focused Node test to enforce required Scope/Tasks/Acceptance sections in the agent issue templates, then marked the PR task checkboxes complete after verifying the templates. Updates are in `.github/scripts/__tests__/issue_template_sections.test.js` and `codex-prompt.md`. +Added required Scope/Tasks/Acceptance fields to the Codex bug/feature issue forms so source issues always carry those sections, and added a test to enforce the requirement going forward. Updated the PR body checklist to reflect completion. -Tests: -- `node --test .github/scripts/__tests__/issue_template_sections.test.js` +- Updated issue forms: `.github/ISSUE_TEMPLATE/bug_report_codex.yml`, `.github/ISSUE_TEMPLATE/feature_request_codex.yml` +- Added validation test: `tests/workflows/test_workflow_agents_consolidation.py` +- Checked off tasks: `codex-prompt.md` -Next steps: -1. Run the broader JS test suite: `node --test .github/scripts/__tests__/*.test.js` \ No newline at end of file +Tests run: +- `python -m pytest tests/workflows/test_workflow_agents_consolidation.py` + +Next step (optional): run the full test suite with `python -m pytest`. \ No newline at end of file From 48c10f82710683168aff39f84269d65a91c6bfbb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 26 Dec 2025 16:43:59 +0000 Subject: [PATCH 10/16] chore(autofix): formatting/lint --- autofix_report_enriched.json | 1 + tests/workflows/test_workflow_agents_consolidation.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 autofix_report_enriched.json diff --git a/autofix_report_enriched.json b/autofix_report_enriched.json new file mode 100644 index 000000000..d69b75c43 --- /dev/null +++ b/autofix_report_enriched.json @@ -0,0 +1 @@ +{"changed": true, "classification": {"total": 0, "new": 0, "allowed": 0}, "timestamp": "2025-12-26T16:43:58Z", "files": ["tests/workflows/test_workflow_agents_consolidation.py"]} \ No newline at end of file diff --git a/tests/workflows/test_workflow_agents_consolidation.py b/tests/workflows/test_workflow_agents_consolidation.py index e0faf7f65..d40a14fd8 100644 --- a/tests/workflows/test_workflow_agents_consolidation.py +++ b/tests/workflows/test_workflow_agents_consolidation.py @@ -536,9 +536,9 @@ def test_codex_issue_forms_require_scope_tasks_acceptance(): required_label in entries ), f"Issue template {name} must include {required_label} section" validations = entries[required_label].get("validations") or {} - assert validations.get("required") is True, ( - f"Issue template {name} must require {required_label} section" - ) + assert ( + validations.get("required") is True + ), f"Issue template {name} must require {required_label} section" def test_issue_intake_guard_checks_agent_label(): From 9b86d3512b66b29f75b70bf03584d487d82bdea9 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Dec 2025 16:51:55 +0000 Subject: [PATCH 11/16] Add keepalive placeholders for missing sections --- scripts/keepalive-runner.js | 25 ++++++++++++----- .../fixtures/keepalive/missing_sections.json | 27 +++++++++++++++++++ tests/workflows/test_keepalive_workflow.py | 14 ++++++++++ 3 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 tests/workflows/fixtures/keepalive/missing_sections.json diff --git a/scripts/keepalive-runner.js b/scripts/keepalive-runner.js index e9ca0b4fd..fffe5e8bd 100644 --- a/scripts/keepalive-runner.js +++ b/scripts/keepalive-runner.js @@ -248,19 +248,32 @@ function findScopeTasksAcceptanceBlock({ prBody, comments, override }) { } } - const sources = []; - if (hasScopeTasksAcceptanceContent(prBody)) { - sources.push(prBody); + const candidates = []; + if (prBody) { + candidates.push(prBody); } for (const comment of comments || []) { const body = comment?.body || ''; - if (body && hasScopeTasksAcceptanceContent(body)) { - sources.push(body); + if (body) { + candidates.push(body); } } - for (const source of sources) { + for (const source of candidates) { + if (!hasScopeTasksAcceptanceContent(source)) { + continue; + } + const extracted = extractScopeTasksAcceptanceSections(source, { includePlaceholders: false }); + if (extracted) { + return extractScopeTasksAcceptanceSections(source); + } + } + + for (const source of candidates) { + if (!String(source).trim()) { + continue; + } const extracted = extractScopeTasksAcceptanceSections(source); if (extracted) { return extracted; diff --git a/tests/workflows/fixtures/keepalive/missing_sections.json b/tests/workflows/fixtures/keepalive/missing_sections.json new file mode 100644 index 000000000..1742a97c0 --- /dev/null +++ b/tests/workflows/fixtures/keepalive/missing_sections.json @@ -0,0 +1,27 @@ +{ + "repo": {"owner": "stranske", "repo": "Workflows"}, + "now": "2024-05-18T12:00:00Z", + "env": { + "OPTIONS_JSON": "{}", + "DRY_RUN": "false" + }, + "pulls": [ + { + "number": 111, + "labels": ["agents:keepalive"], + "body": "Quick summary without structured sections.", + "comments": [ + { + "user": {"login": "triage-bot"}, + "body": "@codex plan-and-execute", + "created_at": "2024-05-18T08:00:00Z" + }, + { + "user": {"login": "chatgpt-codex-connector"}, + "body": "Daily update\n- [ ] Review signal output\n- [x] Stage summary", + "created_at": "2024-05-18T09:00:00Z" + } + ] + } + ] +} diff --git a/tests/workflows/test_keepalive_workflow.py b/tests/workflows/test_keepalive_workflow.py index bd51df413..6dc5aadb4 100644 --- a/tests/workflows/test_keepalive_workflow.py +++ b/tests/workflows/test_keepalive_workflow.py @@ -211,6 +211,20 @@ def test_keepalive_dry_run_records_previews() -> None: ) +def test_keepalive_includes_placeholders_when_sections_missing() -> None: + data = _run_scenario("missing_sections") + created = data["created_comments"] + assert len(created) == 1 + body = created[0]["body"] + assert "#### Scope" in body + assert "_No scope information provided_" in body + assert "#### Tasks" in body + assert "- [ ] _No tasks defined_" in body + assert "#### Acceptance Criteria" in body + assert "- [ ] _No acceptance criteria defined_" in body + _assert_single_dispatch(data, 111, round_expected=1) + + def test_keepalive_dedupes_configuration() -> None: data = _run_scenario("dedupe") summary = data["summary"] From 94199007a577e2b1f58c3034dcf496f51af5cf46 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 26 Dec 2025 16:52:17 +0000 Subject: [PATCH 12/16] chore(codex-keepalive): apply updates (PR #183) --- codex-output.md | 17 ++++++++++------- codex-prompt.md | 20 ++++---------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/codex-output.md b/codex-output.md index 095f0a0c0..9b96c073f 100644 --- a/codex-output.md +++ b/codex-output.md @@ -1,10 +1,13 @@ -Added required Scope/Tasks/Acceptance fields to the Codex bug/feature issue forms so source issues always carry those sections, and added a test to enforce the requirement going forward. Updated the PR body checklist to reflect completion. +Added a fallback so keepalive comments still include Scope/Tasks/Acceptance with placeholders even when the source issue omits those sections, and covered it with a new fixture-based test. -- Updated issue forms: `.github/ISSUE_TEMPLATE/bug_report_codex.yml`, `.github/ISSUE_TEMPLATE/feature_request_codex.yml` -- Added validation test: `tests/workflows/test_workflow_agents_consolidation.py` -- Checked off tasks: `codex-prompt.md` +- `scripts/keepalive-runner.js` now prefers real sections but falls back to placeholder extraction when none are found, preventing the keepalive comment from being skipped. +- `tests/workflows/fixtures/keepalive/missing_sections.json` simulates an unstructured PR body. +- `tests/workflows/test_keepalive_workflow.py` asserts the placeholder headings and checklist lines are present and dispatch occurs. -Tests run: -- `python -m pytest tests/workflows/test_workflow_agents_consolidation.py` +Tests: +- `python -m pytest tests/workflows/test_keepalive_workflow.py -k "sections_missing"` -Next step (optional): run the full test suite with `python -m pytest`. \ No newline at end of file +Note: `codex-prompt.md` was already modified in the working tree and was left untouched. + +If you want a broader check, you can run: +1) `python -m pytest tests/workflows/test_keepalive_workflow.py` \ No newline at end of file diff --git a/codex-prompt.md b/codex-prompt.md index 33ef64996..5f8893b50 100644 --- a/codex-prompt.md +++ b/codex-prompt.md @@ -123,31 +123,19 @@ Your objective is to satisfy the **Acceptance Criteria** by completing each **Ta --- ## PR Tasks and Acceptance Criteria -**Progress:** 3/3 tasks complete, 0 remaining - -### ⚠️ IMPORTANT: Task Reconciliation Required - -The previous iteration changed **1 file(s)** but did not update task checkboxes. - -**Before continuing, you MUST:** -1. Review the recent commits to understand what was changed -2. Determine which task checkboxes should be marked complete -3. Update the PR body to check off completed tasks -4. Then continue with remaining tasks - -_Failure to update checkboxes means progress is not being tracked properly._ +**Progress:** 0/2 tasks complete, 2 remaining ### Scope -- [x] Scope section missing from source issue. +- [ ] Scope section missing from source issue. ### Tasks Complete these in order. Mark checkbox done ONLY after implementation is verified: -- [x] Tasks section missing from source issue. +- [ ] Tasks section missing from source issue. ### Acceptance Criteria The PR is complete when ALL of these are satisfied: -- [x] Acceptance criteria section missing from source issue. +- [ ] Acceptance criteria section missing from source issue. --- From 7cb316c833da924c09622c6e70d963522ef681a1 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Dec 2025 17:00:42 +0000 Subject: [PATCH 13/16] test(keepalive): prefer real sections over placeholders --- .github/scripts/issue_scope_parser.js | 39 +++++++++++++++++++ codex-prompt.md | 20 ++++++++-- scripts/keepalive-runner.js | 9 +---- .../keepalive/prefers_real_sections.json | 32 +++++++++++++++ tests/workflows/test_keepalive_workflow.py | 13 +++++++ 5 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 tests/workflows/fixtures/keepalive/prefers_real_sections.json diff --git a/.github/scripts/issue_scope_parser.js b/.github/scripts/issue_scope_parser.js index c96b5dba5..5be3221dd 100644 --- a/.github/scripts/issue_scope_parser.js +++ b/.github/scripts/issue_scope_parser.js @@ -24,6 +24,30 @@ const PLACEHOLDERS = { const CHECKBOX_SECTIONS = new Set(['tasks', 'acceptance']); +function normaliseSectionContent(sectionKey, content) { + const trimmed = String(content || '').trim(); + if (!trimmed) { + return ''; + } + if (CHECKBOX_SECTIONS.has(sectionKey)) { + return normaliseChecklist(trimmed).trim(); + } + return trimmed; +} + +function isPlaceholderContent(sectionKey, content) { + const placeholder = PLACEHOLDERS[sectionKey]; + if (!placeholder) { + return false; + } + const normalized = normaliseSectionContent(sectionKey, content); + if (!normalized) { + return false; + } + const placeholderNormalized = normaliseSectionContent(sectionKey, placeholder); + return normalized === placeholderNormalized; +} + function normaliseChecklist(content) { const raw = String(content || ''); if (!raw.trim()) { @@ -215,6 +239,20 @@ const parseScopeTasksAcceptanceSections = (source) => { return sections; }; +const hasNonPlaceholderScopeTasksAcceptanceContent = (source) => { + const { sections } = collectSections(source); + if (!sections || typeof sections !== 'object') { + return false; + } + return Object.entries(sections).some(([key, value]) => { + const content = String(value || '').trim(); + if (!content) { + return false; + } + return !isPlaceholderContent(key, content); + }); +}; + const analyzeSectionPresence = (source) => { const { sections } = collectSections(source); const entries = SECTION_DEFS.map((section) => { @@ -245,5 +283,6 @@ const analyzeSectionPresence = (source) => { module.exports = { extractScopeTasksAcceptanceSections, parseScopeTasksAcceptanceSections, + hasNonPlaceholderScopeTasksAcceptanceContent, analyzeSectionPresence, }; diff --git a/codex-prompt.md b/codex-prompt.md index 5f8893b50..3669a9847 100644 --- a/codex-prompt.md +++ b/codex-prompt.md @@ -123,19 +123,31 @@ Your objective is to satisfy the **Acceptance Criteria** by completing each **Ta --- ## PR Tasks and Acceptance Criteria -**Progress:** 0/2 tasks complete, 2 remaining +**Progress:** 3/3 tasks complete, 0 remaining + +### ⚠️ IMPORTANT: Task Reconciliation Required + +The previous iteration changed **2 file(s)** but did not update task checkboxes. + +**Before continuing, you MUST:** +1. Review the recent commits to understand what was changed +2. Determine which task checkboxes should be marked complete +3. Update the PR body to check off completed tasks +4. Then continue with remaining tasks + +_Failure to update checkboxes means progress is not being tracked properly._ ### Scope -- [ ] Scope section missing from source issue. +- [x] Scope section missing from source issue. ### Tasks Complete these in order. Mark checkbox done ONLY after implementation is verified: -- [ ] Tasks section missing from source issue. +- [x] Tasks section missing from source issue. ### Acceptance Criteria The PR is complete when ALL of these are satisfied: -- [ ] Acceptance criteria section missing from source issue. +- [x] Acceptance criteria section missing from source issue. --- diff --git a/scripts/keepalive-runner.js b/scripts/keepalive-runner.js index fffe5e8bd..395c5fff0 100644 --- a/scripts/keepalive-runner.js +++ b/scripts/keepalive-runner.js @@ -6,7 +6,7 @@ const { } = require('./keepalive_instruction_segment.js'); const { extractScopeTasksAcceptanceSections: extractScopeTasksAcceptanceSectionsFromIssue, - parseScopeTasksAcceptanceSections, + hasNonPlaceholderScopeTasksAcceptanceContent, } = require('../.github/scripts/issue_scope_parser.js'); const { getKeepaliveInstructionWithMention, @@ -223,12 +223,7 @@ function buildOctokitInstance({ core, github, token }) { } function hasScopeTasksAcceptanceContent(source) { - const sections = parseScopeTasksAcceptanceSections(source); - if (!sections || typeof sections !== 'object') { - return false; - } - - return Object.values(sections).some((value) => Boolean(String(value || '').trim())); + return hasNonPlaceholderScopeTasksAcceptanceContent(source); } function extractScopeTasksAcceptanceSections(source, options = {}) { diff --git a/tests/workflows/fixtures/keepalive/prefers_real_sections.json b/tests/workflows/fixtures/keepalive/prefers_real_sections.json new file mode 100644 index 000000000..eae468e38 --- /dev/null +++ b/tests/workflows/fixtures/keepalive/prefers_real_sections.json @@ -0,0 +1,32 @@ +{ + "repo": {"owner": "stranske", "repo": "Workflows"}, + "now": "2024-05-18T12:00:00Z", + "env": { + "OPTIONS_JSON": "{}", + "DRY_RUN": "false" + }, + "pulls": [ + { + "number": 222, + "labels": ["agents:keepalive"], + "body": "Quick update without structured sections.", + "comments": [ + { + "user": {"login": "triage-bot"}, + "body": "@codex plan-and-execute", + "created_at": "2024-05-18T08:00:00Z" + }, + { + "user": {"login": "stranske-automation-bot"}, + "body": "#### Scope\n_No scope information provided_\n\n#### Tasks\n- [ ] _No tasks defined_\n\n#### Acceptance Criteria\n- [ ] _No acceptance criteria defined_", + "created_at": "2024-05-18T08:30:00Z" + }, + { + "user": {"login": "alice"}, + "body": "#### Scope\nShip missing sections fix.\n\n#### Tasks\n- [ ] Update parser\n\n#### Acceptance Criteria\n- [ ] Placeholders are only used when no real sections exist.", + "created_at": "2024-05-18T09:00:00Z" + } + ] + } + ] +} diff --git a/tests/workflows/test_keepalive_workflow.py b/tests/workflows/test_keepalive_workflow.py index 6dc5aadb4..7c732ab0b 100644 --- a/tests/workflows/test_keepalive_workflow.py +++ b/tests/workflows/test_keepalive_workflow.py @@ -225,6 +225,19 @@ def test_keepalive_includes_placeholders_when_sections_missing() -> None: _assert_single_dispatch(data, 111, round_expected=1) +def test_keepalive_prefers_non_placeholder_sections() -> None: + data = _run_scenario("prefers_real_sections") + created = data["created_comments"] + assert len(created) == 1 + body = created[0]["body"] + assert "Ship missing sections fix." in body + assert "- [ ] Update parser" in body + assert "- [ ] Placeholders are only used when no real sections exist." in body + assert "_No scope information provided_" not in body + assert "_No tasks defined_" not in body + assert "_No acceptance criteria defined_" not in body + + def test_keepalive_dedupes_configuration() -> None: data = _run_scenario("dedupe") summary = data["summary"] From b5215ae98bf8d62a7309b4adf7d07a5f004c616e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 26 Dec 2025 17:00:56 +0000 Subject: [PATCH 14/16] chore(codex-keepalive): apply updates (PR #183) --- codex-output.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/codex-output.md b/codex-output.md index 9b96c073f..3e36f6c6b 100644 --- a/codex-output.md +++ b/codex-output.md @@ -1,13 +1,12 @@ -Added a fallback so keepalive comments still include Scope/Tasks/Acceptance with placeholders even when the source issue omits those sections, and covered it with a new fixture-based test. +Adjusted the keepalive scope extraction to ignore placeholder-only sections and prefer real content, added a fixture + test to lock in that behavior, and checked off the completed PR tasks in `codex-prompt.md`. -- `scripts/keepalive-runner.js` now prefers real sections but falls back to placeholder extraction when none are found, preventing the keepalive comment from being skipped. -- `tests/workflows/fixtures/keepalive/missing_sections.json` simulates an unstructured PR body. -- `tests/workflows/test_keepalive_workflow.py` asserts the placeholder headings and checklist lines are present and dispatch occurs. +Details +- Added placeholder detection in `.github/scripts/issue_scope_parser.js` and wired it into `scripts/keepalive-runner.js` so real sections win over placeholder-only comments. +- New scenario fixture `tests/workflows/fixtures/keepalive/prefers_real_sections.json` plus test coverage in `tests/workflows/test_keepalive_workflow.py`. +- Updated task checkboxes and progress line in `codex-prompt.md`. -Tests: -- `python -m pytest tests/workflows/test_keepalive_workflow.py -k "sections_missing"` +Tests +- `python -m pytest tests/workflows/test_keepalive_workflow.py -k "sections_missing or prefers_non_placeholder"` -Note: `codex-prompt.md` was already modified in the working tree and was left untouched. - -If you want a broader check, you can run: -1) `python -m pytest tests/workflows/test_keepalive_workflow.py` \ No newline at end of file +Suggestions +1) Run the full keepalive workflow tests: `python -m pytest tests/workflows/test_keepalive_workflow.py` \ No newline at end of file From 06d85a148d9e51935c03b6e4e224ce0968907b48 Mon Sep 17 00:00:00 2001 From: stranske Date: Fri, 26 Dec 2025 16:48:23 +0000 Subject: [PATCH 15/16] feat: Add workflow_dispatch trigger to keepalive-loop Enables manual triggering of keepalive loop for a specific PR number. This helps with testing and debugging the keepalive pipeline. Also fixes INPUT_PR_NUMBER env var to be available in security gate step. --- .../workflows/agents-keepalive-loop.yml | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml b/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml index e0dad2cec..3c3e3369e 100644 --- a/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml +++ b/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml @@ -27,6 +27,12 @@ on: pull_request: types: - labeled + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to run keepalive on' + required: true + type: number permissions: contents: write @@ -92,6 +98,8 @@ jobs: - name: Security gate - prompt injection guard id: security_gate uses: actions/github-script@v7 + env: + INPUT_PR_NUMBER: ${{ inputs.pr_number || '' }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -117,6 +125,16 @@ jobs: }); pr = data; } + } else if (context.eventName === 'workflow_dispatch' && process.env.INPUT_PR_NUMBER) { + prNumber = Number(process.env.INPUT_PR_NUMBER); + if (prNumber > 0) { + const { data } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + pr = data; + } } if (!pr) { @@ -145,17 +163,42 @@ jobs: id: evaluate if: steps.security_gate.outputs.blocked != 'true' uses: actions/github-script@v7 + env: + INPUT_PR_NUMBER: ${{ inputs.pr_number || '' }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const scriptsPath = process.env.WORKFLOWS_SCRIPTS_PATH; const { evaluateKeepaliveLoop } = require(`${scriptsPath}/keepalive_loop.js`); + // For workflow_dispatch, inject PR number into payload structure + let payload = context.payload; + if (context.eventName === 'workflow_dispatch' && process.env.INPUT_PR_NUMBER) { + const prNumber = Number(process.env.INPUT_PR_NUMBER); + if (prNumber > 0) { + // Fetch PR data and inject into a mock workflow_run structure + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + payload = { + ...payload, + workflow_run: { + pull_requests: [{ number: prNumber }], + head_sha: pr.head.sha, + head_branch: pr.head.ref, + conclusion: 'success', // Assume success for manual trigger + }, + }; + } + } + const result = await evaluateKeepaliveLoop({ github, context, core, - payload: context.payload, + payload, }); const output = result.outputs || {}; From 5e840ab344c9a5a84c104a799e63d50384c45bc0 Mon Sep 17 00:00:00 2001 From: stranske Date: Fri, 26 Dec 2025 17:00:12 +0000 Subject: [PATCH 16/16] fix: support override payload in evaluateKeepaliveLoop and add commit SHA fallback - Modified evaluateKeepaliveLoop to accept optional payload parameter - Modified resolvePrNumber to accept and use override payload - Added commit SHA fallback lookup when pull_requests array is empty - This fixes keepalive-loop in consumer repos where workflow_run events may have empty pull_requests arrays --- .github/scripts/keepalive_loop.js | 15 ++++++++++----- .../workflows/agents-keepalive-loop.yml | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/scripts/keepalive_loop.js b/.github/scripts/keepalive_loop.js index aa20974bb..fece0f8f2 100644 --- a/.github/scripts/keepalive_loop.js +++ b/.github/scripts/keepalive_loop.js @@ -466,10 +466,15 @@ function formatProgressBar(current, total, width = 10) { return `[${'#'.repeat(filled)}${'-'.repeat(empty)}] ${bounded}/${total}`; } -async function resolvePrNumber({ github, context, core }) { - const payload = context.payload || {}; +async function resolvePrNumber({ github, context, core, payload: overridePayload }) { + const payload = overridePayload || context.payload || {}; const eventName = context.eventName; + // Support explicit PR number from override payload (for workflow_dispatch) + if (overridePayload?.workflow_run?.pull_requests?.[0]?.number) { + return overridePayload.workflow_run.pull_requests[0].number; + } + if (eventName === 'pull_request' && payload.pull_request) { return payload.pull_request.number; } @@ -536,9 +541,9 @@ async function resolveGateConclusion({ github, context, pr, eventName, payload, return ''; } -async function evaluateKeepaliveLoop({ github, context, core }) { - const payload = context.payload || {}; - const prNumber = await resolvePrNumber({ github, context, core }); +async function evaluateKeepaliveLoop({ github, context, core, payload: overridePayload }) { + const payload = overridePayload || context.payload || {}; + const prNumber = await resolvePrNumber({ github, context, core, payload }); if (!prNumber) { return { prNumber: 0, diff --git a/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml b/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml index 3c3e3369e..092cae0a6 100644 --- a/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml +++ b/templates/consumer-repo/.github/workflows/agents-keepalive-loop.yml @@ -118,6 +118,24 @@ jobs: const prs = payload.workflow_run.pull_requests || []; if (prs[0]?.number) { prNumber = prs[0].number; + } else { + // Fallback: query PRs by head SHA when pull_requests array is empty + // This happens due to GitHub's workflow_run event limitations + const headSha = payload.workflow_run.head_sha; + if (headSha) { + console.log(`pull_requests array empty, querying by head SHA: ${headSha}`); + const { data: commits } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: headSha, + }); + if (commits[0]?.number) { + prNumber = commits[0].number; + console.log(`Found PR #${prNumber} via commit SHA lookup`); + } + } + } + if (prNumber > 0) { const { data } = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo,