From 19f0595900929823e21aff170fcbc814782b0ae8 Mon Sep 17 00:00:00 2001 From: stranske Date: Wed, 31 Dec 2025 14:26:23 +0000 Subject: [PATCH] fix: add @slow marker to integration tests and improve coverage prompts - Add @slow marker to 3 integration tests that run external tools - Update all keepalive prompts to use targeted coverage verification: - Use -m 'not slow' to skip slow tests - Use --cov=scripts/specific_module for faster feedback - Run specific test file when available - Prevents Codex timeout issues during coverage verification --- .github/codex/prompts/keepalive_next_task.md | 9 ++++++++- .github/templates/keepalive-instruction.md | 9 ++++++++- agents/codex-prompt.md | 9 ++++++++- .../.github/codex/prompts/keepalive_next_task.md | 9 ++++++++- .../.github/templates/keepalive-instruction.md | 9 ++++++++- tests/workflows/test_autofix_full_pipeline.py | 1 + tests/workflows/test_autofix_pipeline.py | 1 + tests/workflows/test_autofix_pipeline_diverse.py | 1 + 8 files changed, 43 insertions(+), 5 deletions(-) diff --git a/.github/codex/prompts/keepalive_next_task.md b/.github/codex/prompts/keepalive_next_task.md index ed4365bc2..172bbb80c 100644 --- a/.github/codex/prompts/keepalive_next_task.md +++ b/.github/codex/prompts/keepalive_next_task.md @@ -16,12 +16,19 @@ Your objective is to satisfy the **Acceptance Criteria** by completing each **Ta **COVERAGE TASKS - SPECIAL RULES:** If a task mentions "coverage" or a percentage target (e.g., "≥95%", "to 95%"), you MUST: -1. Run `pytest tests/ --cov=scripts --cov-report=term-missing` after adding tests +1. After adding tests, run TARGETED coverage verification to avoid timeouts: + - For a specific script like `scripts/foo.py`, run: + `pytest tests/scripts/test_foo.py --cov=scripts/foo --cov-report=term-missing -m "not slow"` + - If no matching test file exists, run: + `pytest tests/ --cov=scripts/foo --cov-report=term-missing -m "not slow" -x` 2. Find the specific script in the coverage output table 3. Verify the `Cover` column shows the target percentage or higher 4. Only mark the task complete if the actual coverage meets the target 5. If coverage is below target, add more tests until it meets the target +IMPORTANT: Always use `-m "not slow"` to skip slow integration tests that may timeout. +IMPORTANT: Use targeted `--cov=scripts/specific_module` instead of `--cov=scripts` for faster feedback. + A coverage task is NOT complete just because you added tests. It is complete ONLY when the coverage command output confirms the target is met. **The Tasks and Acceptance Criteria are provided in the appendix below.** Work through them in order. diff --git a/.github/templates/keepalive-instruction.md b/.github/templates/keepalive-instruction.md index 941efb5f4..ec5b791a9 100644 --- a/.github/templates/keepalive-instruction.md +++ b/.github/templates/keepalive-instruction.md @@ -30,12 +30,19 @@ After: `- [x] Add validation for user input` **COVERAGE TASKS - SPECIAL RULES:** If a task mentions "coverage" or a percentage target (e.g., "≥95%", "to 95%"), you MUST: -1. Run `pytest tests/ --cov=scripts --cov-report=term-missing` after adding tests +1. After adding tests, run TARGETED coverage verification to avoid timeouts: + - For a specific script like `scripts/foo.py`, run: + `pytest tests/scripts/test_foo.py --cov=scripts/foo --cov-report=term-missing -m "not slow"` + - If no matching test file exists, run: + `pytest tests/ --cov=scripts/foo --cov-report=term-missing -m "not slow" -x` 2. Find the specific script in the coverage output table 3. Verify the `Cover` column shows the target percentage or higher 4. Only mark the task complete if the actual coverage meets the target 5. If coverage is below target, add more tests until it meets the target +IMPORTANT: Always use `-m "not slow"` to skip slow integration tests that may timeout. +IMPORTANT: Use targeted `--cov=scripts/specific_module` instead of `--cov=scripts` for faster feedback. + A coverage task is NOT complete just because you added tests. It is complete ONLY when the coverage command output confirms the target is met. **CONTEXT TIP:** diff --git a/agents/codex-prompt.md b/agents/codex-prompt.md index b450d17d8..a73a699b4 100644 --- a/agents/codex-prompt.md +++ b/agents/codex-prompt.md @@ -119,12 +119,19 @@ Your objective is to satisfy the **Acceptance Criteria** by completing each **Ta **COVERAGE TASKS - SPECIAL RULES:** If a task mentions "coverage" or a percentage target (e.g., "≥95%", "to 95%"), you MUST: -1. Run `pytest tests/ --cov=scripts --cov-report=term-missing` after adding tests +1. After adding tests, run TARGETED coverage verification to avoid timeouts: + - For a specific script like `scripts/foo.py`, run: + `pytest tests/scripts/test_foo.py --cov=scripts/foo --cov-report=term-missing -m "not slow"` + - If no matching test file exists, run: + `pytest tests/ --cov=scripts/foo --cov-report=term-missing -m "not slow" -x` 2. Find the specific script in the coverage output table 3. Verify the `Cover` column shows the target percentage or higher 4. Only mark the task complete if the actual coverage meets the target 5. If coverage is below target, add more tests until it meets the target +IMPORTANT: Always use `-m "not slow"` to skip slow integration tests that may timeout. +IMPORTANT: Use targeted `--cov=scripts/specific_module` instead of `--cov=scripts` for faster feedback. + A coverage task is NOT complete just because you added tests. It is complete ONLY when the coverage command output confirms the target is met. **The Tasks and Acceptance Criteria are provided in the appendix below.** Work through them in order. diff --git a/templates/consumer-repo/.github/codex/prompts/keepalive_next_task.md b/templates/consumer-repo/.github/codex/prompts/keepalive_next_task.md index ed4365bc2..172bbb80c 100644 --- a/templates/consumer-repo/.github/codex/prompts/keepalive_next_task.md +++ b/templates/consumer-repo/.github/codex/prompts/keepalive_next_task.md @@ -16,12 +16,19 @@ Your objective is to satisfy the **Acceptance Criteria** by completing each **Ta **COVERAGE TASKS - SPECIAL RULES:** If a task mentions "coverage" or a percentage target (e.g., "≥95%", "to 95%"), you MUST: -1. Run `pytest tests/ --cov=scripts --cov-report=term-missing` after adding tests +1. After adding tests, run TARGETED coverage verification to avoid timeouts: + - For a specific script like `scripts/foo.py`, run: + `pytest tests/scripts/test_foo.py --cov=scripts/foo --cov-report=term-missing -m "not slow"` + - If no matching test file exists, run: + `pytest tests/ --cov=scripts/foo --cov-report=term-missing -m "not slow" -x` 2. Find the specific script in the coverage output table 3. Verify the `Cover` column shows the target percentage or higher 4. Only mark the task complete if the actual coverage meets the target 5. If coverage is below target, add more tests until it meets the target +IMPORTANT: Always use `-m "not slow"` to skip slow integration tests that may timeout. +IMPORTANT: Use targeted `--cov=scripts/specific_module` instead of `--cov=scripts` for faster feedback. + A coverage task is NOT complete just because you added tests. It is complete ONLY when the coverage command output confirms the target is met. **The Tasks and Acceptance Criteria are provided in the appendix below.** Work through them in order. diff --git a/templates/consumer-repo/.github/templates/keepalive-instruction.md b/templates/consumer-repo/.github/templates/keepalive-instruction.md index 941efb5f4..ec5b791a9 100644 --- a/templates/consumer-repo/.github/templates/keepalive-instruction.md +++ b/templates/consumer-repo/.github/templates/keepalive-instruction.md @@ -30,12 +30,19 @@ After: `- [x] Add validation for user input` **COVERAGE TASKS - SPECIAL RULES:** If a task mentions "coverage" or a percentage target (e.g., "≥95%", "to 95%"), you MUST: -1. Run `pytest tests/ --cov=scripts --cov-report=term-missing` after adding tests +1. After adding tests, run TARGETED coverage verification to avoid timeouts: + - For a specific script like `scripts/foo.py`, run: + `pytest tests/scripts/test_foo.py --cov=scripts/foo --cov-report=term-missing -m "not slow"` + - If no matching test file exists, run: + `pytest tests/ --cov=scripts/foo --cov-report=term-missing -m "not slow" -x` 2. Find the specific script in the coverage output table 3. Verify the `Cover` column shows the target percentage or higher 4. Only mark the task complete if the actual coverage meets the target 5. If coverage is below target, add more tests until it meets the target +IMPORTANT: Always use `-m "not slow"` to skip slow integration tests that may timeout. +IMPORTANT: Use targeted `--cov=scripts/specific_module` instead of `--cov=scripts` for faster feedback. + A coverage task is NOT complete just because you added tests. It is complete ONLY when the coverage command output confirms the target is met. **CONTEXT TIP:** diff --git a/tests/workflows/test_autofix_full_pipeline.py b/tests/workflows/test_autofix_full_pipeline.py index 3b69b9631..b89ca5c8a 100644 --- a/tests/workflows/test_autofix_full_pipeline.py +++ b/tests/workflows/test_autofix_full_pipeline.py @@ -25,6 +25,7 @@ def _run( return result +@pytest.mark.slow @pytest.mark.integration def test_autofix_pipeline_resolves_lint_and_typing( tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/workflows/test_autofix_pipeline.py b/tests/workflows/test_autofix_pipeline.py index 9801cf4a0..956b3da2a 100644 --- a/tests/workflows/test_autofix_pipeline.py +++ b/tests/workflows/test_autofix_pipeline.py @@ -25,6 +25,7 @@ def _run(cmd: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: return result +@pytest.mark.slow @pytest.mark.integration def test_autofix_pipeline_fixes_trivial_ruff_issue( tmp_path: Path, monkeypatch: pytest.MonkeyPatch diff --git a/tests/workflows/test_autofix_pipeline_diverse.py b/tests/workflows/test_autofix_pipeline_diverse.py index ce8b069f0..c71233399 100644 --- a/tests/workflows/test_autofix_pipeline_diverse.py +++ b/tests/workflows/test_autofix_pipeline_diverse.py @@ -35,6 +35,7 @@ def _run( return result +@pytest.mark.slow @pytest.mark.integration def test_autofix_pipeline_handles_diverse_errors( tmp_path: Path, monkeypatch: pytest.MonkeyPatch