Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 188 additions & 0 deletions .github/scripts/__tests__/verifier-issue-formatter.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,127 @@ Error classification and recovery.
assert.ok(result.body.includes('Verifier confirmed these criteria were met'));
assert.ok(result.body.includes('✓ First criterion'));
});

describe('hasSubstantiveContent property', () => {
it('returns false when all tasks and criteria are placeholders', () => {
const verifierOutput = 'Verdict: PASS\n\nEverything looks good.';
const prBody = `## Tasks
- [ ] Tasks section missing from source issue

## Acceptance Criteria
- [ ] Acceptance Criteria section missing from source issue`;

const result = formatFollowUpIssue({
verifierOutput,
prBody,
issues: [],
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, false);
});

it('returns true when there are real tasks', () => {
const verifierOutput = 'Verdict: PASS\n\nEverything looks good.';
const prBody = `## Tasks
- [ ] Implement feature A
- [ ] Add tests

## Acceptance Criteria
- [ ] Acceptance Criteria section missing from source issue`;

const result = formatFollowUpIssue({
verifierOutput,
prBody,
issues: [],
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns true when there are real criteria', () => {
const verifierOutput = 'Verdict: PASS\n\nEverything looks good.';
const prBody = `## Tasks
- [ ] Tasks section missing from source issue

## Acceptance Criteria
- [ ] Feature works correctly
- [ ] Tests pass`;

const result = formatFollowUpIssue({
verifierOutput,
prBody,
issues: [],
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns true when verifier has gaps', () => {
const verifierOutput = `Verdict: FAIL

Blocking gaps:
- Missing test coverage
- API returns wrong status code`;
const prBody = `## Tasks
- [ ] Tasks section missing from source issue

## Acceptance Criteria
- [ ] Acceptance Criteria section missing from source issue`;

const result = formatFollowUpIssue({
verifierOutput,
prBody,
issues: [],
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns true when verifier has unmet criteria', () => {
const verifierOutput = `Verdict: FAIL

## Criteria Status
- [ ] First criterion - NOT MET (missing implementation)
- [ ] Second criterion - NOT MET (no tests)`;
const prBody = `## Tasks
- [x] Done

## Acceptance Criteria
- [ ] First criterion
- [ ] Second criterion`;

const result = formatFollowUpIssue({
verifierOutput,
prBody,
issues: [],
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns false when only some placeholders mixed with empty content', () => {
const verifierOutput = 'Verdict: PASS\n\nLooks good.';
const prBody = `## Tasks
- [ ] Tasks section missing from source issue

## Acceptance Criteria
- [ ] n/a`;

const result = formatFollowUpIssue({
verifierOutput,
prBody,
issues: [],
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, false);
});
});
});

describe('formatSimpleFollowUpIssue', () => {
Expand Down Expand Up @@ -476,5 +597,72 @@ Something went wrong.`;
assert.ok(result.title.includes('Verifier failure'));
assert.ok(!result.title.includes('PR #'));
});

describe('hasSubstantiveContent property', () => {
it('returns true when there are verifier gaps', () => {
const output = `Verdict: FAIL

Blocking gaps:
- Missing test coverage
- API error handling incomplete`;

const result = formatSimpleFollowUpIssue({
verifierOutput: output,
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns true when there are unmet criteria', () => {
const output = `Verdict: FAIL

## Criteria Status
- [ ] First criterion - NOT MET
- [ ] Second criterion - NOT MET`;

const result = formatSimpleFollowUpIssue({
verifierOutput: output,
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns true when there is verifier output', () => {
const output = `Verdict: FAIL

Something went wrong with the verification.`;

const result = formatSimpleFollowUpIssue({
verifierOutput: output,
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns true even for minimal verifier output', () => {
const output = 'Verdict: FAIL';

const result = formatSimpleFollowUpIssue({
verifierOutput: output,
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, true);
});

it('returns false for empty verifier output', () => {
const output = '';

const result = formatSimpleFollowUpIssue({
verifierOutput: output,
prNumber: 123,
});

assert.equal(result.hasSubstantiveContent, false);
});
});
});
});
14 changes: 8 additions & 6 deletions .github/scripts/verifier_issue_formatter.js
Original file line number Diff line number Diff line change
Expand Up @@ -510,11 +510,12 @@ function formatFollowUpIssue({

// Determine if this issue has substantive content worth creating
// Skip if we have no real tasks/criteria (just defaults) and no verifier gaps
const hasSubstantiveContent =
const hasSubstantiveContent = Boolean(
(newTasks.length > 0 && !newTasks.every(t => isPlaceholderContent(t))) ||
(refinedUnmetCriteria.length > 0 && !refinedUnmetCriteria.every(c => isPlaceholderContent(c))) ||
findings.gaps.length > 0 ||
findings.unmetCriteria.length > 0;
findings.unmetCriteria.length > 0
);

return {
title,
Expand Down Expand Up @@ -581,12 +582,13 @@ function formatSimpleFollowUpIssue({
? `Verifier failure for PR #${prNumber}`
: 'Verifier failure on merged commit';

// Check if there's substantive content to report:
// parsed findings (gaps/unmet criteria) or non-empty verifier output
const hasSubstantiveContent =
// Simple format always has substantive content (verifier output)
// since we only use it when we have actual verifier output to display
const hasSubstantiveContent = Boolean(
findings.gaps.length > 0 ||
findings.unmetCriteria.length > 0 ||
(verifierOutput && verifierOutput.trim().length > 0);
(verifierOutput && verifierOutput.trim().length > 0)
);

return {
title,
Expand Down
Loading