Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions .github/scripts/issue_format.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,12 +290,21 @@ def validate(body: str) -> Report:
acceptance_at = _find(body, REQUIRED["Acceptance Criteria"])
if acceptance_at is not None:
acceptance = _section_text(body, acceptance_at)
if not GATE.search(acceptance):
acceptance_prose = _without_fenced_code(acceptance)
if not GATE.search(acceptance_prose):
Comment thread
stranske marked this conversation as resolved.
report.problems.append(
"`Acceptance Criteria` names no test, runnable command or observable "
"verification gate — Definition of Ready / Quality Bar §2 requires one."
)
prose = re.sub(r"(?<!`)`(?!`)[^`\n]*`", "", _without_fenced_code(acceptance))
prose = re.sub(r"(?<!`)`(?!`)[^`\n]*`", "", acceptance_prose)
# Only discard tokens that are demonstrably file paths. A broad
# slash-separated-word pattern would also erase subjective prose such
# as "fast/performant" before the adjective check sees it.
prose = re.sub(
r"(?<![\w/])(?:[\w.-]+/)+[\w.-]+\.[A-Za-z0-9]{1,10}\b",
"",
prose,
)
hits = [word for word in BANNED_ADJECTIVES if re.search(rf"\b{word}\b", prose, re.I)]
if hits:
report.problems.append(
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/agents-issue-optimizer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -579,9 +579,9 @@ jobs:
import json
with open('/tmp/format_result.json') as f:
result = json.load(f)
formatted = result['formatted_body']
formatted = result.get('formatted_body', '')
if not formatted:
print('ERROR: No formatted body returned')
print('ERROR: ' + (result.get('error') or 'No formatted body returned'))
import sys
sys.exit(1)
with open('/tmp/formatted_body.md', 'w') as f:
Expand Down
4 changes: 2 additions & 2 deletions config/template-drift-allowlist.txt
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ reason = Intentional divergence re-baselined 2026-06-30: root and consumer guard
[pair.11]
main = .github/workflows/agents-issue-optimizer.yml
template = templates/consumer-repo/.github/workflows/agents-issue-optimizer.yml
main_sha256 = 178a3fa311c8f92a24d8ab5fc7aa24f6f975c906361d2b2040ed9a4208edca93
template_sha256 = c4f57ffe386c7e261ac3549ceb9058c8e7c4a5699af3666ccdc038723a023fe4
main_sha256 = f0bc17d390f7b9f1c0398903929a5987c8b6a0c944a5ea5ca0291445df854c44
template_sha256 = ceef0f57e91084daa86cf6876d89f42350a08678d0a424b459d83dacf2437f0c
reason = Intentional divergence re-baselined 2026-08-10: root remains in-tree (scripts/langchain + .github/scripts/issue_format.py); consumer vendors those via Workflows sparse-checkout under workflows-scripts/. Shared behavioral contract includes safe linear verify-hint validation, event-safe optimizer run names, cancel-safe agents:format release even if the initial check fails, and nonzero failure for unexpected validator stderr. Do not align wholesale — that would strip consumer action pins/token setup.

[pair.12]
Expand Down
2 changes: 1 addition & 1 deletion scripts/langchain/issue_formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def _strip_reuse_marker(text: str) -> str:
SAFE_VERIFY_COMMAND_RE = re.compile(
r"^(?:"
r"(?:"
r"python(?:3)?\s+-m\s+(?:pytest|unittest)\b"
r"(?:python(?:3)?\s+-m\s+)?pytest\b|python(?:3)?\s+-m\s+unittest\b"
r"|node\s+--test\b"
r"|(?:npm|pnpm|yarn)\s+(?:run\s+)?(?:test|vitest|jest|playwright)\b"
r"|(?:make|just|cargo)\s+(?:test|check)\b"
Expand Down
13 changes: 11 additions & 2 deletions templates/consumer-repo/.github/scripts/issue_format.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,12 +290,21 @@ def validate(body: str) -> Report:
acceptance_at = _find(body, REQUIRED["Acceptance Criteria"])
if acceptance_at is not None:
acceptance = _section_text(body, acceptance_at)
if not GATE.search(acceptance):
acceptance_prose = _without_fenced_code(acceptance)
if not GATE.search(acceptance_prose):
report.problems.append(
"`Acceptance Criteria` names no test, runnable command or observable "
"verification gate — Definition of Ready / Quality Bar §2 requires one."
)
prose = re.sub(r"(?<!`)`(?!`)[^`\n]*`", "", _without_fenced_code(acceptance))
prose = re.sub(r"(?<!`)`(?!`)[^`\n]*`", "", acceptance_prose)
# Only discard tokens that are demonstrably file paths. A broad
# slash-separated-word pattern would also erase subjective prose such
# as "fast/performant" before the adjective check sees it.
prose = re.sub(
r"(?<![\w/])(?:[\w.-]+/)+[\w.-]+\.[A-Za-z0-9]{1,10}\b",
"",
prose,
)
hits = [word for word in BANNED_ADJECTIVES if re.search(rf"\b{word}\b", prose, re.I)]
if hits:
report.problems.append(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -521,9 +521,9 @@ jobs:
import json
with open('/tmp/format_result.json') as f:
result = json.load(f)
formatted = result['formatted_body']
formatted = result.get('formatted_body', '')
if not formatted:
print('ERROR: No formatted body returned')
print('ERROR: ' + (result.get('error') or 'No formatted body returned'))
import sys
sys.exit(1)
with open('/tmp/formatted_body.md', 'w') as f:
Expand Down
7 changes: 5 additions & 2 deletions templates/consumer-repo/docs/AGENT_ISSUE_FORMAT.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ A qualifying criterion names one of:

- a specific test path / test id (e.g.
`tests/test_verdict_policy.py::test_select_verdict_worst_policy`), **or**
- a specific runnable command and its expected observable result (e.g.
- a specific runnable command written in normal Acceptance Criteria prose (not
inside a Markdown fenced code block) and its expected observable result (e.g.
`gh workflow run selftest-ci.yml` → the run log shows a non-zero collected
count for the named test files), **or**
- a documented live-verification step tied to behavior a human or agent can
Expand All @@ -166,7 +167,9 @@ intuitive / polished) are rejected — replace with a measurable check.
> Acceptance Criteria block references **no** test, smoke test, or verification
> gate at all (a conservative string check for a test path/id, a runner command
> like `pytest` / `gh workflow run` / `npm test` / `curl`, or a `smoke` /
> `verif` token). An acceptance section of pure adjectives will not pass.
> `verif` token). The qualifying gate must appear in normal Acceptance Criteria
> prose, not inside a Markdown fenced code block. An acceptance section of pure
> adjectives will not pass.

#### The deliberate-break pattern (recommended worked form)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def _strip_reuse_marker(text: str) -> str:
SAFE_VERIFY_COMMAND_RE = re.compile(
r"^(?:"
r"(?:"
r"python(?:3)?\s+-m\s+(?:pytest|unittest)\b"
r"(?:python(?:3)?\s+-m\s+)?pytest\b|python(?:3)?\s+-m\s+unittest\b"
r"|node\s+--test\b"
r"|(?:npm|pnpm|yarn)\s+(?:run\s+)?(?:test|vitest|jest|playwright)\b"
r"|(?:make|just|cargo)\s+(?:test|check)\b"
Expand Down
51 changes: 49 additions & 2 deletions tests/scripts/test_issue_format.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,17 +145,64 @@ def test_subjective_words_in_inline_code_are_ignored_but_prose_is_rejected(
assert report.ok is expected_ok


def test_acceptance_gate_inside_fence_and_code_path_adjective_are_valid() -> None:
validator = _validator()
@pytest.mark.parametrize(
"validator_path",
[
Path(".github/scripts/issue_format.py"),
Path("templates/consumer-repo/.github/scripts/issue_format.py"),
],
)
def test_acceptance_gate_inside_fence_does_not_satisfy_validation(
validator_path: Path,
) -> None:
validator = _validator(validator_path)
report = validator.validate(
VALID_CONTEXT
+ "## Tasks\n- [ ] Update `tests/fast/test_api.py`\n\n"
+ "## Acceptance Criteria\nRun the regression suite:\n"
+ "```sh\npython -m pytest tests/fast/test_api.py -q\n```\n"
)
assert not report.ok
assert "names no test" in report.problems[0]


@pytest.mark.parametrize(
"validator_path",
[
Path(".github/scripts/issue_format.py"),
Path("templates/consumer-repo/.github/scripts/issue_format.py"),
],
)
def test_acceptance_path_component_is_not_subjective_prose(validator_path: Path) -> None:
validator = _validator(validator_path)
report = validator.validate(
VALID_CONTEXT
+ "## Tasks\n- [ ] Update `tests/fast/test_api.py`\n\n"
+ "## Acceptance Criteria\n- [ ] Run pytest tests/fast/test_api.py and confirm it passes.\n"
)
assert report.ok


@pytest.mark.parametrize(
"validator_path",
[
Path(".github/scripts/issue_format.py"),
Path("templates/consumer-repo/.github/scripts/issue_format.py"),
],
)
def test_acceptance_slash_separated_subjective_prose_is_rejected(
validator_path: Path,
) -> None:
validator = _validator(validator_path)
report = validator.validate(
VALID_CONTEXT
+ "## Tasks\n- [ ] Update `tests/fast/test_api.py`\n\n"
+ "## Acceptance Criteria\n- [ ] Run pytest tests/fast/test_api.py and make the response fast/performant.\n"
)
assert not report.ok
assert "subjective wording" in report.problems[0]


def test_runner_and_curl_are_accepted_gates() -> None:
validator = _validator()
report = validator.validate(
Expand Down
1 change: 1 addition & 0 deletions tests/scripts/test_issue_formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -956,6 +956,7 @@ def test_safe_verify_command_requires_runnable_unittest_and_gh_args() -> None:
issue_formatter.SAFE_VERIFY_COMMAND_RE.match("python -m unittest tests.test_x") is not None
)
assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("unittest") is None
assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("pytest") is not None
assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("go check") is None
assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("dotnet check") is None
assert issue_formatter.SAFE_VERIFY_COMMAND_RE.match("dotnet test") is not None
Expand Down
Loading