From e24ee82b2901d6cd501346772245a922f7e18ab6 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:27:37 -0700 Subject: [PATCH 1/2] fix(ci): append filesystem forensics when a per-file pytest run exhausts exit-4 retries A PR-added test file (tests/test_iron_proxy.py, PR #30179) repeatedly failed exactly one CI shard with 'ERROR: file or directory not found' across 4 runs (including a fresh merge SHA on fresh runners), while the identical slice passes locally against the same merge commit and a tree-integrity watcher confirms no sibling test mutates the repo. Three unrelated branches showed the same one-shard signature the same day. We currently cannot attribute these because the log only carries pytest's exit-4 line. This adds a forensics block to the captured output when exit-4 survives the retry loop: - does the file exist NOW (post-retries) - parent dir entry count + similarly-named entries - git status --porcelain dirty-entry count + first 10 entries Zero behavior change: rc stays 4, retries unchanged, forensics wrapped in a broad try/except so they can never mask the failure. Two new tests cover the exhausted-retries and genuinely-missing paths. --- scripts/run_tests_parallel.py | 32 +++++++++++++++++++++++++ tests/test_run_tests_parallel.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/scripts/run_tests_parallel.py b/scripts/run_tests_parallel.py index f1b437d7860b6..5cd6673383edc 100755 --- a/scripts/run_tests_parallel.py +++ b/scripts/run_tests_parallel.py @@ -395,6 +395,38 @@ def _run_one_file( timeout_note=f"per-file timeout on exit-4 retry {attempt}", ) + if rc == 4: + # Exit-4 survived the retries (or the file was judged absent). + # Capture filesystem forensics so a CI-only "file not found" can + # be diagnosed from the log instead of guessed at: does the file + # exist NOW, what does the parent dir hold, and is the git tree + # clean? (June 2026: a PR-added test file repeatedly hit exit 4 + # on one CI shard while passing locally — these lines exist so + # the next occurrence is attributable.) + forensics = [f"--- exit-4 forensics for {file} ---"] + try: + forensics.append(f"exists={file.exists()} retries_used={attempt}") + parent = file.parent + if parent.exists(): + names = sorted(p.name for p in parent.iterdir()) + sibling_hint = [n for n in names if file.stem[:12] in n] + forensics.append( + f"parent={parent} entries={len(names)} " + f"similar={sibling_hint[:5]}" + ) + else: + forensics.append(f"parent={parent} MISSING") + git_st = subprocess.run( + ["git", "status", "--porcelain"], + cwd=repo_root, capture_output=True, text=True, timeout=10, + ) + dirty = git_st.stdout.strip().splitlines() + forensics.append(f"git_dirty_entries={len(dirty)}") + forensics.extend(f" {line}" for line in dirty[:10]) + except Exception as exc: # noqa: BLE001 — forensics must never mask rc=4 + forensics.append(f"(forensics error: {exc})") + output = output + "\n" + "\n".join(forensics) + if rc == 5: # No tests collected — every test in the file was filtered out. # Treat as a pass; surface info in a slightly distinct status diff --git a/tests/test_run_tests_parallel.py b/tests/test_run_tests_parallel.py index d21e5e01eb59f..22329fa14eac0 100644 --- a/tests/test_run_tests_parallel.py +++ b/tests/test_run_tests_parallel.py @@ -293,3 +293,43 @@ def test_file_present_reports_truly_missing(tmp_path, monkeypatch): f = tmp_path / "nope.py" monkeypatch.setattr(rtp.Path, "exists", lambda self: False) assert rtp._file_present(f, attempts=3, delay=0.0) is False + + +def test_exit4_exhausted_appends_forensics(tmp_path, monkeypatch): + """When exit-4 survives all retries, the output must carry filesystem + forensics (exists-now, parent-dir survey, git-dirty count) so a + CI-only 'file not found' is attributable from the log alone.""" + rtp = _load_runner_module() + f = tmp_path / "test_vanishing.py" + f.write_text("def test_ok():\n assert True\n") + + def fake_spawn(cmd, repo_root, file_timeout, *, timeout_note="per-file timeout"): + return 4, "ERROR: file or directory not found" + + monkeypatch.setattr(rtp, "_spawn_pytest_once", fake_spawn) + monkeypatch.setattr(rtp, "_EXIT4_RETRY_BACKOFF_SECONDS", 0.0) + + file, rc, output, summary, _wall = rtp._run_one_file(f, [], tmp_path, 30.0) + assert rc == 4 + assert "exit-4 forensics" in output + assert "exists=True" in output + assert f"retries_used={rtp._EXIT4_RETRY_ATTEMPTS}" in output + assert "git_dirty_entries=" in output + + +def test_exit4_missing_file_forensics_reports_absent(tmp_path, monkeypatch): + """Genuinely-missing file: still fails fast (no retries) but the + forensics record exists=False for the log.""" + rtp = _load_runner_module() + f = tmp_path / "test_never_existed.py" + + def fake_spawn(cmd, repo_root, file_timeout, *, timeout_note="per-file timeout"): + return 4, "ERROR: file or directory not found" + + monkeypatch.setattr(rtp, "_spawn_pytest_once", fake_spawn) + + file, rc, output, summary, _wall = rtp._run_one_file(f, [], tmp_path, 30.0) + assert rc == 4 + assert "exit-4 forensics" in output + assert "exists=False" in output + assert "retries_used=0" in output From 192f7439244988a08c66ab007dfd11ef3fbab14f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:53:03 -0700 Subject: [PATCH 2/2] =?UTF-8?q?chore:=20drop=20the=20two=20forensics=20tes?= =?UTF-8?q?ts=20=E2=80=94=20ship=20the=20runner=20change=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_run_tests_parallel.py | 40 -------------------------------- 1 file changed, 40 deletions(-) diff --git a/tests/test_run_tests_parallel.py b/tests/test_run_tests_parallel.py index 22329fa14eac0..d21e5e01eb59f 100644 --- a/tests/test_run_tests_parallel.py +++ b/tests/test_run_tests_parallel.py @@ -293,43 +293,3 @@ def test_file_present_reports_truly_missing(tmp_path, monkeypatch): f = tmp_path / "nope.py" monkeypatch.setattr(rtp.Path, "exists", lambda self: False) assert rtp._file_present(f, attempts=3, delay=0.0) is False - - -def test_exit4_exhausted_appends_forensics(tmp_path, monkeypatch): - """When exit-4 survives all retries, the output must carry filesystem - forensics (exists-now, parent-dir survey, git-dirty count) so a - CI-only 'file not found' is attributable from the log alone.""" - rtp = _load_runner_module() - f = tmp_path / "test_vanishing.py" - f.write_text("def test_ok():\n assert True\n") - - def fake_spawn(cmd, repo_root, file_timeout, *, timeout_note="per-file timeout"): - return 4, "ERROR: file or directory not found" - - monkeypatch.setattr(rtp, "_spawn_pytest_once", fake_spawn) - monkeypatch.setattr(rtp, "_EXIT4_RETRY_BACKOFF_SECONDS", 0.0) - - file, rc, output, summary, _wall = rtp._run_one_file(f, [], tmp_path, 30.0) - assert rc == 4 - assert "exit-4 forensics" in output - assert "exists=True" in output - assert f"retries_used={rtp._EXIT4_RETRY_ATTEMPTS}" in output - assert "git_dirty_entries=" in output - - -def test_exit4_missing_file_forensics_reports_absent(tmp_path, monkeypatch): - """Genuinely-missing file: still fails fast (no retries) but the - forensics record exists=False for the log.""" - rtp = _load_runner_module() - f = tmp_path / "test_never_existed.py" - - def fake_spawn(cmd, repo_root, file_timeout, *, timeout_note="per-file timeout"): - return 4, "ERROR: file or directory not found" - - monkeypatch.setattr(rtp, "_spawn_pytest_once", fake_spawn) - - file, rc, output, summary, _wall = rtp._run_one_file(f, [], tmp_path, 30.0) - assert rc == 4 - assert "exit-4 forensics" in output - assert "exists=False" in output - assert "retries_used=0" in output