Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ jobs:
if: needs.changes.outputs.integration == 'true' || needs.changes.outputs.python == 'true' || needs.changes.outputs.workflows == 'true'
runs-on: ubuntu-latest
# Coverage runs in dedicated shards below, and the Python 3.12
# slow/integration/performance suite now runs in a separate parallel job.
# slow/integration suite now runs in a separate parallel job.
timeout-minutes: 75
strategy:
fail-fast: false
Expand Down Expand Up @@ -535,9 +535,9 @@ jobs:
run: |
uv sync --extra all-ci

- name: Run slow, integration, and performance tests
- name: Run slow and integration tests
run: |
uv run pytest tests -n auto -m "slow or integration or performance" --tb=short --durations=20
uv run pytest tests -n auto -m "(slow or integration) and not performance" --tb=short --durations=20

coverage:
name: Coverage (${{ matrix.shard }})
Expand Down
12 changes: 4 additions & 8 deletions tests/test_perf_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,7 @@ def test_python_ci_windows_matrix_shards_main_and_workflow_prs() -> None:
assert "--modelaudit-shard-index ${{ matrix.shard-index }}" in exhaustive_run


def test_python_ci_runs_slow_suite_in_a_separate_job() -> None:
def test_python_ci_keeps_performance_out_of_the_xdist_slow_suite() -> None:
workflow = _load_workflow("test.yml")
jobs = _jobs(workflow)

Expand All @@ -545,13 +545,9 @@ def test_python_ci_runs_slow_suite_in_a_separate_job() -> None:
"(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-slow-tests'))"
)
slow_steps = slow_job["steps"]
assert (
'-m "slow or integration or performance"'
in _step_by_name(
slow_steps,
"Run slow, integration, and performance tests",
)["run"]
)
slow_run = _step_by_name(slow_steps, "Run slow and integration tests")["run"]
assert "pytest tests -n auto" in slow_run
assert '-m "(slow or integration) and not performance"' in slow_run

fast_steps = jobs["test"]["steps"]
fast_step_names = {step.get("name") for step in fast_steps}
Expand Down
3 changes: 2 additions & 1 deletion tests/test_regular_scan_hash.py
Original file line number Diff line number Diff line change
Expand Up @@ -1512,7 +1512,8 @@ def mock_hash(path: str, *, deadline: float | None = None) -> str:
# files_scanned should include both files
assert result.files_scanned == 2

def test_hash_generation_performance(self, tmp_path):
@pytest.mark.performance
Comment thread
mldangelo-oai marked this conversation as resolved.
def test_hash_generation_performance(self, tmp_path: Path) -> None:
"""Test that hash generation doesn't significantly impact performance."""
import time

Expand Down
8 changes: 8 additions & 0 deletions tests/test_xdist_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,14 @@ def test_nightly_runs_unsharded_performance_and_rust_once_and_fails_closed() ->
assert all("continue-on-error" not in step for step in job["steps"])


def test_wall_clock_hash_generation_uses_dedicated_performance_lane() -> None:
from tests.test_regular_scan_hash import TestHashGenerationEdgeCases

marks = getattr(TestHashGenerationEdgeCases.test_hash_generation_performance, "pytestmark", ())

assert any(mark.name == "performance" for mark in marks)


def test_nodeid_sharding_rejects_non_positive_count() -> None:
root_conftest = _load_root_conftest()

Expand Down