Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions .github/scripts/__tests__/agents-verifier-context.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,102 @@ test('buildVerifierContext skips forked pull requests', async () => {
assert.ok(core.outputs.skip_reason.includes('fork'));
});

test('buildVerifierContext skips when no acceptance criteria found', async () => {
const core = buildCore();
// PR body with no acceptance criteria section
const prBodyNoAcceptance = `## Summary
This PR adds a new feature.

## Tasks
- [x] Implement the feature
- [x] Add documentation
`;
const prDetails = {
number: 88,
title: 'Feature without acceptance',
body: prBodyNoAcceptance,
html_url: 'https://example.com/pr/88',
merge_commit_sha: 'merge-sha-88',
base: { ref: 'main' },
head: { sha: 'head-sha-88' },
};
const context = {
eventName: 'pull_request',
repo: { owner: 'octo', repo: 'workflows' },
payload: {
repository: { default_branch: 'main' },
pull_request: {
merged: true,
number: 88,
base: { ref: 'main' },
html_url: 'https://example.com/pr/88',
},
},
sha: 'sha-88',
};
// No linked issues, no acceptance criteria in PR
const result = await buildVerifierContext({
github: buildGithubStub({ prDetails, closingIssues: [] }),
context,
core,
});
assert.equal(result.shouldRun, false);
assert.equal(core.outputs.should_run, 'false');
assert.equal(core.outputs.pr_number, '88');
assert.ok(core.outputs.skip_reason.includes('No acceptance criteria'));
assert.equal(core.outputs.acceptance_count, '0');
});

test('buildVerifierContext runs when acceptance criteria exists in linked issue', async () => {
const core = buildCore();
// PR body with no acceptance criteria
const prBodyNoAcceptance = `## Summary
Simple change.
`;
const prDetails = {
number: 89,
title: 'PR with issue acceptance',
body: prBodyNoAcceptance,
html_url: 'https://example.com/pr/89',
merge_commit_sha: 'merge-sha-89',
base: { ref: 'main' },
head: { sha: 'head-sha-89' },
};
const context = {
eventName: 'pull_request',
repo: { owner: 'octo', repo: 'workflows' },
payload: {
repository: { default_branch: 'main' },
pull_request: {
merged: true,
number: 89,
base: { ref: 'main' },
html_url: 'https://example.com/pr/89',
},
},
sha: 'sha-89',
};
// Linked issue HAS acceptance criteria
const issueWithAcceptance = {
number: 100,
title: 'Issue with acceptance',
body: `## Acceptance Criteria
- [ ] Feature works correctly
- [ ] Tests pass
`,
state: 'OPEN',
url: 'https://example.com/issues/100',
};
const result = await buildVerifierContext({
github: buildGithubStub({ prDetails, closingIssues: [issueWithAcceptance] }),
context,
core,
});
assert.equal(result.shouldRun, true);
assert.equal(core.outputs.should_run, 'true');
assert.equal(core.outputs.pr_number, '89');
});

test('buildVerifierContext writes verifier context with linked issues', async () => {
const core = buildCore();
const prDetails = {
Expand Down
16 changes: 1 addition & 15 deletions .github/workflows/reusable-agents-verifier.yml
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs');
const { formatFollowUpIssue, formatSimpleFollowUpIssue, formatErrorIssue } = require('./.workflows-lib/.github/scripts/verifier_issue_formatter.js');
const { formatFollowUpIssue, formatSimpleFollowUpIssue } = require('./.workflows-lib/.github/scripts/verifier_issue_formatter.js');

const rawPrNumber = Number('${{ steps.context.outputs.pr_number }}');
const prNumber = !Number.isNaN(rawPrNumber) && rawPrNumber > 0 ? rawPrNumber : null;
Expand All @@ -245,20 +245,6 @@ jobs:

// Handle Codex crash/error case
if (verdict === 'error') {
// Check if formatErrorIssue exists, otherwise use fallback
if (typeof formatErrorIssue === 'function') {
const result = formatErrorIssue({ prNumber, prUrl, runUrl, issueNumbers });
const { data: issue } = await github.rest.issues.create({
...context.repo,
title: result.title,
body: result.body,
labels: ['agent:codex', 'bug'],
});
core.setOutput('issue_number', issue?.number ? String(issue.number) : '');
core.info(`Created error follow-up issue #${issue.number}: ${result.title}`);
return;
}
// Fallback if formatErrorIssue doesn't exist
const title = prNumber ? `[Verifier Error] PR #${prNumber} verification failed` : '[Verifier Error] Verification failed';
const body = `## Verifier Error\n\nThe verifier encountered an error while checking acceptance criteria.\n\n**PR:** ${prUrl || `#${prNumber}` || 'unknown'}\n**Run:** ${runUrl}\n\nPlease review the workflow run logs for details.`;
const { data: issue } = await github.rest.issues.create({
Expand Down
Loading