Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions .github/scripts/keepalive_loop.js
Original file line number Diff line number Diff line change
Expand Up @@ -1134,9 +1134,26 @@ async function evaluateKeepaliveLoop({ github, context, core, payload: overrideP
action = 'run';
reason = 'force-retry-cancelled';
if (core) core.info(`Force retry enabled: bypassing cancelled gate (rate_limit=${gateRateLimit})`);
} else if (gateRateLimit) {
// Rate limit situation: check if we should retry or keep deferring
const now = Date.now();
const lastDeferTime = previousState?.last_defer_timestamp || 0;
const timeSinceLastDefer = now - lastDeferTime;
const RATE_LIMIT_RETRY_INTERVAL = 5 * 60 * 1000; // 5 minutes

if (timeSinceLastDefer >= RATE_LIMIT_RETRY_INTERVAL) {
// Enough time has passed, retry with reduced API usage
action = 'run';
reason = 'retry-after-rate-limit';
if (core) core.info(`Rate limit defer timeout reached (${Math.floor(timeSinceLastDefer / 60000)}min since last defer), retrying with reduced API calls`);
} else {
action = 'defer';
reason = 'gate-cancelled-rate-limit-transient';
if (core) core.info(`Rate limit detected, deferring. Will retry in ${Math.floor((RATE_LIMIT_RETRY_INTERVAL - timeSinceLastDefer) / 60000)}min`);
Comment on lines +1140 to +1152

Copilot AI Jan 9, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Logic issue: The rate limit retry logic will always execute on the first defer because last_defer_timestamp will be 0 (falsy) for the first occurrence. This means timeSinceLastDefer will equal the current timestamp (potentially millions of milliseconds), which will always exceed the 5-minute threshold (300,000ms).

This effectively bypasses the deferral mechanism on the first rate limit encounter. The code should initialize the defer timestamp on the first defer, not check against it. Consider either:

  1. Checking if lastDeferTime is 0 and treating that as the first defer (set the timestamp but don't retry yet)
  2. Setting the defer timestamp immediately and only retry on subsequent checks
Suggested change
const lastDeferTime = previousState?.last_defer_timestamp || 0;
const timeSinceLastDefer = now - lastDeferTime;
const RATE_LIMIT_RETRY_INTERVAL = 5 * 60 * 1000; // 5 minutes
if (timeSinceLastDefer >= RATE_LIMIT_RETRY_INTERVAL) {
// Enough time has passed, retry with reduced API usage
action = 'run';
reason = 'retry-after-rate-limit';
if (core) core.info(`Rate limit defer timeout reached (${Math.floor(timeSinceLastDefer / 60000)}min since last defer), retrying with reduced API calls`);
} else {
action = 'defer';
reason = 'gate-cancelled-rate-limit-transient';
if (core) core.info(`Rate limit detected, deferring. Will retry in ${Math.floor((RATE_LIMIT_RETRY_INTERVAL - timeSinceLastDefer) / 60000)}min`);
const lastDeferTime = previousState?.last_defer_timestamp ?? 0;
const RATE_LIMIT_RETRY_INTERVAL = 5 * 60 * 1000; // 5 minutes
if (!lastDeferTime) {
// First time we've seen a rate limit in this state: defer and initialize timestamp
action = 'defer';
reason = 'gate-cancelled-rate-limit-transient';
if (core) core.info('Rate limit detected for the first time in this state, deferring and initializing defer timestamp');
} else {
const timeSinceLastDefer = now - lastDeferTime;
if (timeSinceLastDefer >= RATE_LIMIT_RETRY_INTERVAL) {
// Enough time has passed, retry with reduced API usage
action = 'run';
reason = 'retry-after-rate-limit';
if (core) core.info(`Rate limit defer timeout reached (${Math.floor(timeSinceLastDefer / 60000)}min since last defer), retrying with reduced API calls`);
} else {
action = 'defer';
reason = 'gate-cancelled-rate-limit-transient';
if (core) core.info(`Rate limit detected, deferring. Will retry in ${Math.floor((RATE_LIMIT_RETRY_INTERVAL - timeSinceLastDefer) / 60000)}min`);
}

Copilot uses AI. Check for mistakes.
}
} else {
action = gateRateLimit ? 'defer' : 'wait';
reason = gateRateLimit ? 'gate-cancelled-rate-limit' : 'gate-cancelled';
action = 'wait';
reason = 'gate-cancelled';
}
} else {
// Gate failed - check if we should route to fix mode or wait
Expand Down Expand Up @@ -1715,6 +1732,8 @@ async function updateKeepaliveLoopSummary({ github, context, core, inputs }) {
attempted_tasks: attemptedTasks,
last_focus: focusTask || '',
verification,
// Rate limit defer tracking
last_defer_timestamp: action === 'defer' && reason.includes('rate-limit') ? Date.now() : (previousState?.last_defer_timestamp || 0),
};
const attemptEntry = buildAttemptEntry({
iteration: metricsIteration,
Expand Down
14 changes: 4 additions & 10 deletions .github/workflows/agents-auto-label.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,16 @@ jobs:
!contains(github.event.issue.labels.*.name, 'automated')

steps:
- name: Checkout Workflows repo
uses: actions/checkout@v6
with:
# Use the repository containing the label_matcher.py script
# For consumer repos, this fetches from the central Workflows repo
repository: ${{ github.repository == 'stranske/Workflows' && github.repository || 'stranske/Workflows' }}
path: workflows-repo
- name: Checkout repository
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
python-version: "3.11"

- name: Install dependencies
run: |
cd workflows-repo
pip install -e ".[langchain]" --quiet

- name: Get repo labels
Expand Down Expand Up @@ -76,8 +70,8 @@ jobs:
LABELS_JSON: ${{ steps.get-labels.outputs.labels_json }}
ISSUE_TITLE: ${{ github.event.issue.title }}
ISSUE_BODY: ${{ github.event.issue.body }}
PYTHONPATH: ${{ github.workspace }}
run: |
cd workflows-repo
python3 << 'PYTHON_SCRIPT'
import json
import os
Expand Down
210 changes: 210 additions & 0 deletions .github/workflows/agents-capability-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
name: Capability Check

# Pre-flight check before agent assignment to identify blockers
# Uses capability_check.py to detect issues agents cannot complete

on:
issues:
types: [labeled]

permissions:
contents: read
issues: write
models: read

jobs:
capability-check:
runs-on: ubuntu-latest
# Trigger when agent:codex is added (pre-agent gate)
if: github.event.label.name == 'agent:codex'

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"

- name: Install dependencies
run: |
pip install -e ".[langchain]" --quiet

- name: Extract issue content
id: extract
uses: actions/github-script@v8
with:
script: |
const issue = context.payload.issue;
const body = issue.body || '';

// Extract Tasks section
const tasksMatch = body.match(/## Tasks\s*\n([\s\S]*?)(?=##|$)/i);
const tasks = tasksMatch ? tasksMatch[1].trim() : '';

// Extract Acceptance Criteria section
const acceptanceMatch = body.match(/## Acceptance [Cc]riteria\s*\n([\s\S]*?)(?=##|$)/i);
const acceptance = acceptanceMatch ? acceptanceMatch[1].trim() : '';

// Write to files for Python script
const fs = require('fs');
fs.writeFileSync('tasks.md', tasks || 'No tasks defined');
fs.writeFileSync('acceptance.md', acceptance || 'No acceptance criteria defined');

core.setOutput('has_tasks', tasks ? 'true' : 'false');
core.setOutput('has_acceptance', acceptance ? 'true' : 'false');

- name: Run capability check
id: check
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PYTHONPATH: ${{ github.workspace }}
run: |
python -c "
import json
import os
import sys
sys.path.insert(0, '.')

from scripts.langchain.capability_check import check_capability

# Read extracted content
tasks = open('tasks.md').read()
acceptance = open('acceptance.md').read()

# Run capability check
result = check_capability(tasks, acceptance)

if result is None:
print('::warning::Could not run capability check (LLM unavailable)')
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
f.write('check_failed=true\n')
sys.exit(0)

# Output results
result_dict = result.to_dict()
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
f.write('check_failed=false\n')
f.write(f'recommendation={result.recommendation}\n')
f.write(f'blocked_count={len(result.blocked_tasks)}\n')
f.write(f'partial_count={len(result.partial_tasks)}\n')
f.write(f'result_json={json.dumps(result_dict)}\n')

print(f'Recommendation: {result.recommendation}')
print(f'Blocked tasks: {len(result.blocked_tasks)}')
print(f'Partial tasks: {len(result.partial_tasks)}')
print(f'Actionable tasks: {len(result.actionable_tasks)}')
"

- name: Add needs-human label if blocked
if: steps.check.outputs.recommendation == 'BLOCKED'
uses: actions/github-script@v8
with:
script: |
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: ['needs-human']
});

// Remove agent:codex since agent can't complete this
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
name: 'agent:codex'
});
} catch (e) {
core.warning('Could not remove agent:codex label');
}

- name: Post capability report
if: steps.check.outputs.check_failed != 'true'
uses: actions/github-script@v8
env:
RESULT_JSON: ${{ steps.check.outputs.result_json }}
RECOMMENDATION: ${{ steps.check.outputs.recommendation }}
with:
script: |
const result = JSON.parse(process.env.RESULT_JSON || '{}');
const recommendation = process.env.RECOMMENDATION || 'UNKNOWN';

let emoji = '✅';
let status = 'Agent can proceed';
if (recommendation === 'BLOCKED') {
emoji = '🚫';
status = 'Agent cannot complete this issue';
} else if (recommendation === 'REVIEW_NEEDED') {
emoji = '⚠️';
status = 'Some tasks may need human assistance';
}

let body = `### ${emoji} Capability Check: ${status}\n\n`;
body += `**Recommendation:** ${recommendation}\n\n`;

if (result.actionable_tasks && result.actionable_tasks.length > 0) {
body += `**✅ Actionable Tasks (${result.actionable_tasks.length}):**\n`;
result.actionable_tasks.forEach(t => { body += `- ${t}\n`; });
body += '\n';
}

if (result.partial_tasks && result.partial_tasks.length > 0) {
body += `**⚠️ Partial Tasks (${result.partial_tasks.length}):**\n`;
result.partial_tasks.forEach(t => {
body += `- ${t.task}\n - *Limitation:* ${t.limitation}\n`;
});
body += '\n';
}

if (result.blocked_tasks && result.blocked_tasks.length > 0) {
body += `**🚫 Blocked Tasks (${result.blocked_tasks.length}):**\n`;
result.blocked_tasks.forEach(t => {
body += `- ${t.task}\n - *Reason:* ${t.reason}\n`;
if (t.suggested_action) {
body += ` - *Suggested Action:* ${t.suggested_action}\n`;
}
});
body += '\n';
}

if (result.human_actions_needed && result.human_actions_needed.length > 0) {
body += `**👤 Human Actions Needed:**\n`;
result.human_actions_needed.forEach(a => { body += `- ${a}\n`; });
body += '\n';
}

body += `---\n*Auto-generated by capability check*`;

// Check for existing comment
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
per_page: 50
});

const existingComment = comments.find(c =>
c.body.includes('### ✅ Capability Check') ||
c.body.includes('### ⚠️ Capability Check') ||
c.body.includes('### 🚫 Capability Check')
);

if (existingComment) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existingComment.id,
body: body
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: body
});
}
Loading
Loading