diff --git a/.github/scripts/__tests__/verifier-issue-formatter.test.js b/.github/scripts/__tests__/verifier-issue-formatter.test.js index 9a8bb37cb..c23ae8fa1 100644 --- a/.github/scripts/__tests__/verifier-issue-formatter.test.js +++ b/.github/scripts/__tests__/verifier-issue-formatter.test.js @@ -426,6 +426,127 @@ Error classification and recovery. assert.ok(result.body.includes('Verifier confirmed these criteria were met')); assert.ok(result.body.includes('✓ First criterion')); }); + + describe('hasSubstantiveContent property', () => { + it('returns false when all tasks and criteria are placeholders', () => { + const verifierOutput = 'Verdict: PASS\n\nEverything looks good.'; + const prBody = `## Tasks +- [ ] Tasks section missing from source issue + +## Acceptance Criteria +- [ ] Acceptance Criteria section missing from source issue`; + + const result = formatFollowUpIssue({ + verifierOutput, + prBody, + issues: [], + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, false); + }); + + it('returns true when there are real tasks', () => { + const verifierOutput = 'Verdict: PASS\n\nEverything looks good.'; + const prBody = `## Tasks +- [ ] Implement feature A +- [ ] Add tests + +## Acceptance Criteria +- [ ] Acceptance Criteria section missing from source issue`; + + const result = formatFollowUpIssue({ + verifierOutput, + prBody, + issues: [], + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns true when there are real criteria', () => { + const verifierOutput = 'Verdict: PASS\n\nEverything looks good.'; + const prBody = `## Tasks +- [ ] Tasks section missing from source issue + +## Acceptance Criteria +- [ ] Feature works correctly +- [ ] Tests pass`; + + const result = formatFollowUpIssue({ + verifierOutput, + prBody, + issues: [], + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns true when verifier has gaps', () => { + const verifierOutput = `Verdict: FAIL + +Blocking gaps: +- Missing test coverage +- API returns wrong status code`; + const prBody = `## Tasks +- [ ] Tasks section missing from source issue + +## Acceptance Criteria +- [ ] Acceptance Criteria section missing from source issue`; + + const result = formatFollowUpIssue({ + verifierOutput, + prBody, + issues: [], + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns true when verifier has unmet criteria', () => { + const verifierOutput = `Verdict: FAIL + +## Criteria Status +- [ ] First criterion - NOT MET (missing implementation) +- [ ] Second criterion - NOT MET (no tests)`; + const prBody = `## Tasks +- [x] Done + +## Acceptance Criteria +- [ ] First criterion +- [ ] Second criterion`; + + const result = formatFollowUpIssue({ + verifierOutput, + prBody, + issues: [], + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns false when only some placeholders mixed with empty content', () => { + const verifierOutput = 'Verdict: PASS\n\nLooks good.'; + const prBody = `## Tasks +- [ ] Tasks section missing from source issue + +## Acceptance Criteria +- [ ] n/a`; + + const result = formatFollowUpIssue({ + verifierOutput, + prBody, + issues: [], + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, false); + }); + }); }); describe('formatSimpleFollowUpIssue', () => { @@ -476,5 +597,72 @@ Something went wrong.`; assert.ok(result.title.includes('Verifier failure')); assert.ok(!result.title.includes('PR #')); }); + + describe('hasSubstantiveContent property', () => { + it('returns true when there are verifier gaps', () => { + const output = `Verdict: FAIL + +Blocking gaps: +- Missing test coverage +- API error handling incomplete`; + + const result = formatSimpleFollowUpIssue({ + verifierOutput: output, + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns true when there are unmet criteria', () => { + const output = `Verdict: FAIL + +## Criteria Status +- [ ] First criterion - NOT MET +- [ ] Second criterion - NOT MET`; + + const result = formatSimpleFollowUpIssue({ + verifierOutput: output, + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns true when there is verifier output', () => { + const output = `Verdict: FAIL + +Something went wrong with the verification.`; + + const result = formatSimpleFollowUpIssue({ + verifierOutput: output, + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns true even for minimal verifier output', () => { + const output = 'Verdict: FAIL'; + + const result = formatSimpleFollowUpIssue({ + verifierOutput: output, + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, true); + }); + + it('returns false for empty verifier output', () => { + const output = ''; + + const result = formatSimpleFollowUpIssue({ + verifierOutput: output, + prNumber: 123, + }); + + assert.equal(result.hasSubstantiveContent, false); + }); + }); }); }); diff --git a/.github/scripts/verifier_issue_formatter.js b/.github/scripts/verifier_issue_formatter.js index bc1206050..d125a6fc6 100644 --- a/.github/scripts/verifier_issue_formatter.js +++ b/.github/scripts/verifier_issue_formatter.js @@ -510,11 +510,12 @@ function formatFollowUpIssue({ // Determine if this issue has substantive content worth creating // Skip if we have no real tasks/criteria (just defaults) and no verifier gaps - const hasSubstantiveContent = + const hasSubstantiveContent = Boolean( (newTasks.length > 0 && !newTasks.every(t => isPlaceholderContent(t))) || (refinedUnmetCriteria.length > 0 && !refinedUnmetCriteria.every(c => isPlaceholderContent(c))) || findings.gaps.length > 0 || - findings.unmetCriteria.length > 0; + findings.unmetCriteria.length > 0 + ); return { title, @@ -581,12 +582,13 @@ function formatSimpleFollowUpIssue({ ? `Verifier failure for PR #${prNumber}` : 'Verifier failure on merged commit'; - // Check if there's substantive content to report: - // parsed findings (gaps/unmet criteria) or non-empty verifier output - const hasSubstantiveContent = + // Simple format always has substantive content (verifier output) + // since we only use it when we have actual verifier output to display + const hasSubstantiveContent = Boolean( findings.gaps.length > 0 || findings.unmetCriteria.length > 0 || - (verifierOutput && verifierOutput.trim().length > 0); + (verifierOutput && verifierOutput.trim().length > 0) + ); return { title,