diff --git a/.github/scripts/issue_format.py b/.github/scripts/issue_format.py index e233e7092..d29ec02c4 100644 --- a/.github/scripts/issue_format.py +++ b/.github/scripts/issue_format.py @@ -290,12 +290,21 @@ def validate(body: str) -> Report: acceptance_at = _find(body, REQUIRED["Acceptance Criteria"]) if acceptance_at is not None: acceptance = _section_text(body, acceptance_at) - if not GATE.search(acceptance): + acceptance_prose = _without_fenced_code(acceptance) + if not GATE.search(acceptance_prose): report.problems.append( "`Acceptance Criteria` names no test, runnable command or observable " "verification gate — Definition of Ready / Quality Bar §2 requires one." ) - prose = re.sub(r"(? str: SAFE_VERIFY_COMMAND_RE = re.compile( r"^(?:" r"(?:" - r"python(?:3)?\s+-m\s+(?:pytest|unittest)\b" + r"(?:python(?:3)?\s+-m\s+)?pytest\b|python(?:3)?\s+-m\s+unittest\b" r"|node\s+--test\b" r"|(?:npm|pnpm|yarn)\s+(?:run\s+)?(?:test|vitest|jest|playwright)\b" r"|(?:make|just|cargo)\s+(?:test|check)\b" diff --git a/templates/consumer-repo/.github/scripts/issue_format.py b/templates/consumer-repo/.github/scripts/issue_format.py index e233e7092..d29ec02c4 100644 --- a/templates/consumer-repo/.github/scripts/issue_format.py +++ b/templates/consumer-repo/.github/scripts/issue_format.py @@ -290,12 +290,21 @@ def validate(body: str) -> Report: acceptance_at = _find(body, REQUIRED["Acceptance Criteria"]) if acceptance_at is not None: acceptance = _section_text(body, acceptance_at) - if not GATE.search(acceptance): + acceptance_prose = _without_fenced_code(acceptance) + if not GATE.search(acceptance_prose): report.problems.append( "`Acceptance Criteria` names no test, runnable command or observable " "verification gate — Definition of Ready / Quality Bar §2 requires one." ) - prose = re.sub(r"(? Acceptance Criteria block references **no** test, smoke test, or verification > gate at all (a conservative string check for a test path/id, a runner command > like `pytest` / `gh workflow run` / `npm test` / `curl`, or a `smoke` / -> `verif` token). An acceptance section of pure adjectives will not pass. +> `verif` token). The qualifying gate must appear in normal Acceptance Criteria +> prose, not inside a Markdown fenced code block. An acceptance section of pure +> adjectives will not pass. #### The deliberate-break pattern (recommended worked form) diff --git a/templates/consumer-repo/scripts/langchain/issue_formatter.py b/templates/consumer-repo/scripts/langchain/issue_formatter.py index 8ca4f766d..2c22182d4 100755 --- a/templates/consumer-repo/scripts/langchain/issue_formatter.py +++ b/templates/consumer-repo/scripts/langchain/issue_formatter.py @@ -177,7 +177,7 @@ def _strip_reuse_marker(text: str) -> str: SAFE_VERIFY_COMMAND_RE = re.compile( r"^(?:" r"(?:" - r"python(?:3)?\s+-m\s+(?:pytest|unittest)\b" + r"(?:python(?:3)?\s+-m\s+)?pytest\b|python(?:3)?\s+-m\s+unittest\b" r"|node\s+--test\b" r"|(?:npm|pnpm|yarn)\s+(?:run\s+)?(?:test|vitest|jest|playwright)\b" r"|(?:make|just|cargo)\s+(?:test|check)\b" diff --git a/tests/scripts/test_issue_format.py b/tests/scripts/test_issue_format.py index d5985526e..f7298530a 100644 --- a/tests/scripts/test_issue_format.py +++ b/tests/scripts/test_issue_format.py @@ -145,17 +145,64 @@ def test_subjective_words_in_inline_code_are_ignored_but_prose_is_rejected( assert report.ok is expected_ok -def test_acceptance_gate_inside_fence_and_code_path_adjective_are_valid() -> None: - validator = _validator() +@pytest.mark.parametrize( + "validator_path", + [ + Path(".github/scripts/issue_format.py"), + Path("templates/consumer-repo/.github/scripts/issue_format.py"), + ], +) +def test_acceptance_gate_inside_fence_does_not_satisfy_validation( + validator_path: Path, +) -> None: + validator = _validator(validator_path) report = validator.validate( VALID_CONTEXT + "## Tasks\n- [ ] Update `tests/fast/test_api.py`\n\n" + "## Acceptance Criteria\nRun the regression suite:\n" + "```sh\npython -m pytest tests/fast/test_api.py -q\n```\n" ) + assert not report.ok + assert "names no test" in report.problems[0] + + +@pytest.mark.parametrize( + "validator_path", + [ + Path(".github/scripts/issue_format.py"), + Path("templates/consumer-repo/.github/scripts/issue_format.py"), + ], +) +def test_acceptance_path_component_is_not_subjective_prose(validator_path: Path) -> None: + validator = _validator(validator_path) + report = validator.validate( + VALID_CONTEXT + + "## Tasks\n- [ ] Update `tests/fast/test_api.py`\n\n" + + "## Acceptance Criteria\n- [ ] Run pytest tests/fast/test_api.py and confirm it passes.\n" + ) assert report.ok +@pytest.mark.parametrize( + "validator_path", + [ + Path(".github/scripts/issue_format.py"), + Path("templates/consumer-repo/.github/scripts/issue_format.py"), + ], +) +def test_acceptance_slash_separated_subjective_prose_is_rejected( + validator_path: Path, +) -> None: + validator = _validator(validator_path) + report = validator.validate( + VALID_CONTEXT + + "## Tasks\n- [ ] Update `tests/fast/test_api.py`\n\n" + + "## Acceptance Criteria\n- [ ] Run pytest tests/fast/test_api.py and make the response fast/performant.\n" + ) + assert not report.ok + assert "subjective wording" in report.problems[0] + + def test_runner_and_curl_are_accepted_gates() -> None: validator = _validator() report = validator.validate( diff --git a/tests/scripts/test_issue_formatter.py b/tests/scripts/test_issue_formatter.py index 309d64f96..cbd764d27 100644 --- a/tests/scripts/test_issue_formatter.py +++ b/tests/scripts/test_issue_formatter.py @@ -956,6 +956,7 @@ def test_safe_verify_command_requires_runnable_unittest_and_gh_args() -> None: issue_formatter.SAFE_VERIFY_COMMAND_RE.match("python -m unittest tests.test_x") is not None ) assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("unittest") is None + assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("pytest") is not None assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("go check") is None assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("dotnet check") is None assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("dotnet test") is not None