diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ae0060e..0461e0941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Prevent Windows cache identity probes from creating locked temporary files inside scanned directories. - Preserve locked Windows cache probes reached through directory aliases while clearing stale scan results. - Place cross-volume Windows cache identity probes near the volume root instead of the nearest ancestor of the scanned path, so a probe can no longer appear inside a directory tree that a concurrent scan is walking. +- Preserve macOS scan-result cache entries during unrelated temporary-file churn while rejecting replaced source files and directories. ## [0.2.52](https://github.com/promptfoo/modelaudit/compare/v0.2.51...v0.2.52) (2026-07-22) diff --git a/modelaudit/cache/scan_results_cache.py b/modelaudit/cache/scan_results_cache.py index c6b601ea6..6c5db2c56 100644 --- a/modelaudit/cache/scan_results_cache.py +++ b/modelaudit/cache/scan_results_cache.py @@ -4,6 +4,7 @@ import json import logging import os +import select import stat import struct import sys @@ -11,7 +12,7 @@ import threading import time from collections.abc import Callable, Iterable -from contextlib import suppress +from contextlib import ExitStack, suppress from contextvars import ContextVar from dataclasses import asdict, dataclass from importlib.machinery import ( @@ -217,6 +218,87 @@ def __del__(self) -> None: self.close() +class _DarwinPathMonitor: + """Observe vnode replacement without rejecting unrelated directory writes.""" + + def __init__(self, file_path: str, ancestor_identity: tuple[AncestorEntry, ...]) -> None: + select_module: Any = select + self._queue: Any = None + self._descriptors: list[int] = [] + self._descriptor_stack: ExitStack | None = None + + try: + self._queue = select_module.kqueue() + vnode_filter = select_module.KQ_FILTER_VNODE + event_flags = select_module.KQ_EV_ADD | select_module.KQ_EV_CLEAR + change_flags = ( + select_module.KQ_NOTE_DELETE + | select_module.KQ_NOTE_RENAME + | select_module.KQ_NOTE_REVOKE + | select_module.KQ_NOTE_ATTRIB + ) + descriptor_flags = getattr(os, "O_EVTONLY", os.O_RDONLY) | getattr(os, "O_CLOEXEC", 0) + descriptor_flags |= getattr(os, "O_NOFOLLOW", 0) + paths = [os.path.abspath(file_path), *(entry[0] for entry in ancestor_identity)] + with ExitStack() as descriptor_stack: + events = [] + opened_descriptors = [] + for path in dict.fromkeys(paths): + watched_path = _DARWIN_STABLE_SYMLINK_ALIASES.get(path, path) + descriptor = os.open(watched_path, descriptor_flags) + try: + descriptor_stack.callback(os.close, descriptor) + except BaseException: + with suppress(OSError): + os.close(descriptor) + raise + opened_descriptors.append(descriptor) + events.append( + select_module.kevent( + descriptor, + filter=vnode_filter, + flags=event_flags, + fflags=change_flags, + ) + ) + self._queue.control(events, 0, 0) + transferred_stack = descriptor_stack.pop_all() + try: + self._descriptor_stack = transferred_stack + self._descriptors = opened_descriptors + except BaseException: + with suppress(OSError): + transferred_stack.close() + raise + except BaseException: + self.close() + raise + + def changed(self) -> bool: + if self._queue is None: + return True + try: + return bool(self._queue.control(None, max(len(self._descriptors), 1), 0)) + except (OSError, ValueError): + return True + + def close(self) -> None: + queue = getattr(self, "_queue", None) + self._queue = None + descriptor_stack = getattr(self, "_descriptor_stack", None) + self._descriptor_stack = None + if queue is not None: + with suppress(OSError): + queue.close() + if descriptor_stack is not None: + with suppress(OSError): + descriptor_stack.close() + self._descriptors = [] + + def __del__(self) -> None: + self.close() + + class _WindowsPathLockMonitor: """Prevent file and ancestor replacement while a Windows scan is in flight.""" @@ -280,12 +362,12 @@ def __del__(self) -> None: class AncestorIdentity(tuple[AncestorEntry, ...]): - monitor: _AncestorPathMonitor | _WindowsPathLockMonitor | None + monitor: _AncestorPathMonitor | _DarwinPathMonitor | _WindowsPathLockMonitor | None def __new__( cls, entries: tuple[AncestorEntry, ...] | list[AncestorEntry], - monitor: _AncestorPathMonitor | _WindowsPathLockMonitor | None = None, + monitor: _AncestorPathMonitor | _DarwinPathMonitor | _WindowsPathLockMonitor | None = None, ) -> "AncestorIdentity": identity = super().__new__(cls, entries) identity.monitor = monitor @@ -610,6 +692,10 @@ def _get_cached_result_with_identity( logger.debug(f"Cache lookup failed for {file_path}: {e}") self._record_cache_miss("error") return None, file_identity + except BaseException: + if file_identity is not None: + self.release_ancestor_identity(file_identity[-1]) + raise def get_cached_result_by_key( self, @@ -1076,7 +1162,7 @@ def _capture_file_identity_leased( time.sleep(0.01) continue raise - except Exception: + except BaseException: self.release_ancestor_identity(monitored_ancestor_identity) raise @@ -1329,13 +1415,15 @@ def _monitor_ancestor_identity(file_path: str, identity: AncestorIdentity) -> An if not identity: return identity try: - monitor: _AncestorPathMonitor | _WindowsPathLockMonitor | None = None + monitor: _AncestorPathMonitor | _DarwinPathMonitor | _WindowsPathLockMonitor | None = None platform_name = getattr(sys, "platform", "") if platform_name.startswith("linux"): monitor = _AncestorPathMonitor( file_path, tuple(identity), ) + elif platform_name == "darwin": + monitor = _DarwinPathMonitor(file_path, tuple(identity)) elif platform_name == "win32": monitor = _WindowsPathLockMonitor(file_path, tuple(identity)) if monitor is None: diff --git a/tests/cache/test_cache_correctness.py b/tests/cache/test_cache_correctness.py index 2b9b1ab10..c84c28a27 100644 --- a/tests/cache/test_cache_correctness.py +++ b/tests/cache/test_cache_correctness.py @@ -9,7 +9,7 @@ import threading import time import zipfile -from collections.abc import Iterator +from collections.abc import Callable, Iterator from importlib.abc import MetaPathFinder from importlib.machinery import ( BYTECODE_SUFFIXES, @@ -1809,6 +1809,452 @@ def test_cache_identity_allows_darwin_private_var_and_tmp_aliases(tmp_path: Path assert ancestor_identity +@pytest.mark.skipif(sys.platform != "darwin", reason="requires Darwin vnode monitoring") +def test_darwin_cache_identity_ignores_unrelated_sibling_churn(tmp_path: Path) -> None: + file_path = _make_cacheable_file(tmp_path) + cache = ScanResultsCache(str(tmp_path / "cache")) + identity = _identity_kwargs(cache, str(file_path)) + + (tmp_path / "unrelated.cache").write_bytes(b"unrelated") + + assert cache.store_result(str(file_path), {"success": True}, **identity) is True + + +@pytest.mark.skipif(sys.platform != "darwin", reason="requires Darwin vnode monitoring") +def test_darwin_cache_identity_rejects_restored_ancestor_mode_change(tmp_path: Path) -> None: + model_directory = tmp_path / "models" + model_directory.mkdir() + file_path = _make_cacheable_file(model_directory) + cache = ScanResultsCache(str(tmp_path / "cache")) + identity = _identity_kwargs(cache, str(file_path)) + original_mode = stat.S_IMODE(model_directory.stat().st_mode) + + model_directory.chmod(original_mode ^ stat.S_IXUSR) + model_directory.chmod(original_mode) + + assert cache.store_result(str(file_path), {"success": True}, **identity) is False + + +@pytest.mark.skipif(sys.platform != "darwin", reason="requires Darwin vnode monitoring") +def test_darwin_cache_identity_rejects_restored_file_replacement(tmp_path: Path) -> None: + file_path = _make_cacheable_file(tmp_path) + replacement = _make_cacheable_file(tmp_path, name="replacement.cache") + original_backup = tmp_path / "original.cache" + cache = ScanResultsCache(str(tmp_path / "cache")) + identity = _identity_kwargs(cache, str(file_path)) + + file_path.replace(original_backup) + replacement.replace(file_path) + file_path.replace(replacement) + original_backup.replace(file_path) + + assert cache.store_result(str(file_path), {"success": True}, **identity) is False + + +@pytest.mark.skipif(sys.platform != "darwin", reason="requires Darwin vnode monitoring") +def test_darwin_cache_identity_rejects_restored_ancestor_replacement(tmp_path: Path) -> None: + model_directory = tmp_path / "models" + model_directory.mkdir() + file_path = _make_cacheable_file(model_directory) + replacement_directory = tmp_path / "replacement" + replacement_directory.mkdir() + _make_cacheable_file(replacement_directory) + original_backup = tmp_path / "original" + cache = ScanResultsCache(str(tmp_path / "cache")) + identity = _identity_kwargs(cache, str(file_path)) + + model_directory.replace(original_backup) + replacement_directory.replace(model_directory) + model_directory.replace(replacement_directory) + original_backup.replace(model_directory) + + assert cache.store_result(str(file_path), {"success": True}, **identity) is False + + +def test_cache_identity_selects_darwin_path_monitor( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class StubDarwinPathMonitor: + def __init__(self, file_path: str, ancestor_identity: tuple[Any, ...]) -> None: + self.file_path = file_path + self.ancestor_identity = ancestor_identity + self.closed = False + + def changed(self) -> bool: + return False + + def close(self) -> None: + self.closed = True + + file_path = _make_cacheable_file(tmp_path) + cache = ScanResultsCache(str(tmp_path / "cache")) + monkeypatch.setattr(scan_results_cache_module.sys, "platform", "darwin") + monkeypatch.setattr(scan_results_cache_module, "_DarwinPathMonitor", StubDarwinPathMonitor) + + identity = cache.capture_file_identity(str(file_path))[-1] + monitor = cast(StubDarwinPathMonitor, identity.monitor) + assert isinstance(monitor, StubDarwinPathMonitor) + assert monitor.file_path == str(file_path) + assert monitor.ancestor_identity == tuple(identity) + + cache.release_ancestor_identity(identity) + + assert monitor.closed is True + + +def test_identity_capture_closes_darwin_monitor_on_retained_keyboard_interrupt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + created_monitors: list[Any] = [] + + class StubDarwinPathMonitor: + def __init__(self, _file_path: str, _ancestor_identity: tuple[Any, ...]) -> None: + self.closed = False + created_monitors.append(self) + + def changed(self) -> bool: + return False + + def close(self) -> None: + self.closed = True + + def interrupt_hash(_path: str, _file_stat: os.stat_result) -> str: + raise KeyboardInterrupt("identity hashing interrupted") + + file_path = _make_cacheable_file(tmp_path) + cache = ScanResultsCache(str(tmp_path / "cache")) + monkeypatch.setattr(scan_results_cache_module.sys, "platform", "darwin") + monkeypatch.setattr(scan_results_cache_module, "_DarwinPathMonitor", StubDarwinPathMonitor) + monkeypatch.setattr(cache.hasher, "hash_file_with_stat", interrupt_hash) + + with pytest.raises(KeyboardInterrupt, match="identity hashing interrupted") as interruption: + cache.capture_file_identity(str(file_path)) + + assert interruption.traceback is not None + assert len(created_monitors) == 1 + assert created_monitors[0].closed is True + + +def test_cache_lookup_closes_darwin_monitor_on_retained_keyboard_interrupt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + created_monitors: list[Any] = [] + + class StubDarwinPathMonitor: + def __init__(self, _file_path: str, _ancestor_identity: tuple[Any, ...]) -> None: + self.closed = False + created_monitors.append(self) + + def changed(self) -> bool: + return False + + def close(self) -> None: + self.closed = True + + def interrupt_cache_key(*_args: Any, **_kwargs: Any) -> tuple[str, str]: + raise KeyboardInterrupt("cache key generation interrupted") + + file_path = _make_cacheable_file(tmp_path) + cache = ScanResultsCache(str(tmp_path / "cache")) + monkeypatch.setattr(scan_results_cache_module.sys, "platform", "darwin") + monkeypatch.setattr(scan_results_cache_module, "_DarwinPathMonitor", StubDarwinPathMonitor) + monkeypatch.setattr(cache, "_generate_cache_key_material", interrupt_cache_key) + + with pytest.raises(KeyboardInterrupt, match="cache key generation interrupted") as interruption: + cache.get_cached_result_with_identity(str(file_path)) + + assert interruption.traceback is not None + assert len(created_monitors) == 1 + assert created_monitors[0].closed is True + + +def _stub_darwin_select(queue: Any) -> type[Any]: + class StubDarwinSelect: + KQ_FILTER_VNODE = 1 + KQ_EV_ADD = 2 + KQ_EV_CLEAR = 4 + KQ_NOTE_DELETE = 8 + KQ_NOTE_RENAME = 16 + KQ_NOTE_REVOKE = 32 + KQ_NOTE_ATTRIB = 64 + + @staticmethod + def kqueue() -> Any: + return queue + + @staticmethod + def kevent(*args: Any, **kwargs: Any) -> tuple[tuple[Any, ...], dict[str, Any]]: + return args, kwargs + + return StubDarwinSelect + + +def test_darwin_path_monitor_reports_file_and_ancestor_attribute_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class AttributeAwareQueue: + def __init__(self) -> None: + self.registered_events: list[Any] = [] + + def control(self, changes: Any, _max_events: int, _timeout: int) -> list[Any]: + if changes is not None: + self.registered_events.extend(changes) + return [] + if any(event[1]["fflags"] & 64 for event in self.registered_events): + return [object()] + return [] + + def close(self) -> None: + return None + + file_path = _make_cacheable_file(tmp_path) + queue = AttributeAwareQueue() + select_stub = _stub_darwin_select(queue) + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + + def open_path(_path: str, _flags: int) -> int: + descriptor = 100 + len(opened_descriptors) + opened_descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(scan_results_cache_module, "select", select_stub) + monkeypatch.setattr(scan_results_cache_module.os, "open", open_path) + monkeypatch.setattr(scan_results_cache_module.os, "close", closed_descriptors.append) + + monitor = scan_results_cache_module._DarwinPathMonitor( + str(file_path), + ((str(tmp_path), 0, 0, 0, 0, 0),), + ) + + assert len(queue.registered_events) == 2 + assert all(event[1]["fflags"] & select_stub.KQ_NOTE_ATTRIB for event in queue.registered_events) + assert monitor.changed() is True + monitor.close() + assert sorted(closed_descriptors) == opened_descriptors + + +def test_darwin_path_monitor_normalizes_relative_file_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class StubQueue: + def control(self, _changes: Any, _max_events: int, _timeout: int) -> list[Any]: + return [] + + def close(self) -> None: + return None + + file_path = _make_cacheable_file(tmp_path) + opened_paths: list[str] = [] + closed_descriptors: list[int] = [] + + def open_path(path: str, _flags: int) -> int: + opened_paths.append(path) + return 100 + len(opened_paths) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(scan_results_cache_module, "select", _stub_darwin_select(StubQueue())) + monkeypatch.setattr(scan_results_cache_module.os, "open", open_path) + monkeypatch.setattr(scan_results_cache_module.os, "close", closed_descriptors.append) + + monitor = scan_results_cache_module._DarwinPathMonitor( + file_path.name, + ((str(tmp_path), 0, 0, 0, 0, 0),), + ) + + assert opened_paths[0] == str(file_path.resolve()) + monitor.close() + assert sorted(closed_descriptors) == [101, 102] + + +def test_darwin_path_monitor_closes_descriptors_when_registration_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FailingQueue: + closed = False + + def control(self, _changes: Any, _max_events: int, _timeout: int) -> list[Any]: + raise OSError("registration failed") + + def close(self) -> None: + self.closed = True + + file_path = _make_cacheable_file(tmp_path) + queue = FailingQueue() + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + + def open_path(_path: str, _flags: int) -> int: + descriptor = 100 + len(opened_descriptors) + opened_descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(scan_results_cache_module, "select", _stub_darwin_select(queue)) + monkeypatch.setattr(scan_results_cache_module.os, "open", open_path) + monkeypatch.setattr(scan_results_cache_module.os, "close", closed_descriptors.append) + + with pytest.raises(OSError, match="registration failed"): + scan_results_cache_module._DarwinPathMonitor( + str(file_path), + ((str(tmp_path), 0, 0, 0, 0, 0),), + ) + + assert queue.closed is True + assert sorted(closed_descriptors) == opened_descriptors + + +def test_darwin_path_monitor_closes_queue_when_path_setup_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class StubQueue: + closed = False + + def control(self, _changes: Any, _max_events: int, _timeout: int) -> list[Any]: + return [] + + def close(self) -> None: + self.closed = True + + queue = StubQueue() + monkeypatch.setattr(scan_results_cache_module, "select", _stub_darwin_select(queue)) + + def fail_abspath(_path: str) -> str: + raise FileNotFoundError("cwd disappeared") + + monkeypatch.setattr(scan_results_cache_module.os.path, "abspath", fail_abspath) + + with pytest.raises(FileNotFoundError, match="cwd disappeared"): + scan_results_cache_module._DarwinPathMonitor("relative.bin", ()) + + assert queue.closed is True + + +def test_darwin_path_monitor_closes_queue_on_retained_keyboard_interrupt_during_path_setup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class StubQueue: + closed = False + + def control(self, _changes: Any, _max_events: int, _timeout: int) -> list[Any]: + return [] + + def close(self) -> None: + self.closed = True + + queue = StubQueue() + monkeypatch.setattr(scan_results_cache_module, "select", _stub_darwin_select(queue)) + + def interrupt_abspath(_path: str) -> str: + raise KeyboardInterrupt("path setup interrupted") + + with monkeypatch.context() as path_patch: + path_patch.setattr(scan_results_cache_module.os.path, "abspath", interrupt_abspath) + with pytest.raises(KeyboardInterrupt, match="path setup interrupted") as interruption: + scan_results_cache_module._DarwinPathMonitor("relative.bin", ()) + + assert interruption.traceback is not None + assert queue.closed is True + + +def test_darwin_path_monitor_closes_descriptor_when_callback_registration_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class StubQueue: + closed = False + + def control(self, _changes: Any, _max_events: int, _timeout: int) -> list[Any]: + return [] + + def close(self) -> None: + self.closed = True + + class FailingCallbackStack: + def __enter__(self) -> FailingCallbackStack: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def callback(self, _callback: Callable[[int], None], _descriptor: int) -> None: + raise MemoryError("callback registration failed") + + file_path = _make_cacheable_file(tmp_path) + queue = StubQueue() + closed_descriptors: list[int] = [] + monkeypatch.setattr(scan_results_cache_module, "select", _stub_darwin_select(queue)) + monkeypatch.setattr(scan_results_cache_module, "ExitStack", FailingCallbackStack) + monkeypatch.setattr(scan_results_cache_module.os, "open", lambda _path, _flags: 101) + monkeypatch.setattr(scan_results_cache_module.os, "close", closed_descriptors.append) + + with pytest.raises(MemoryError, match="callback registration failed"): + scan_results_cache_module._DarwinPathMonitor(str(file_path), ()) + + assert queue.closed is True + assert closed_descriptors == [101] + + +def test_darwin_path_monitor_does_not_double_close_when_stack_transfer_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class StubQueue: + closed = False + + def control(self, _changes: Any, _max_events: int, _timeout: int) -> list[Any]: + return [] + + def close(self) -> None: + self.closed = True + + class FailingTransferStack: + def __init__(self) -> None: + self.callbacks: list[tuple[Callable[[int], None], int]] = [] + + def __enter__(self) -> FailingTransferStack: + return self + + def __exit__(self, *_args: object) -> None: + for callback, descriptor in reversed(self.callbacks): + callback(descriptor) + + def callback(self, callback: Callable[[int], None], descriptor: int) -> None: + self.callbacks.append((callback, descriptor)) + + def pop_all(self) -> FailingTransferStack: + raise MemoryError("stack transfer failed") + + file_path = _make_cacheable_file(tmp_path) + queue = StubQueue() + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + + def open_path(_path: str, _flags: int) -> int: + descriptor = 101 + len(opened_descriptors) + opened_descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(scan_results_cache_module, "select", _stub_darwin_select(queue)) + monkeypatch.setattr(scan_results_cache_module, "ExitStack", FailingTransferStack) + monkeypatch.setattr(scan_results_cache_module.os, "open", open_path) + monkeypatch.setattr(scan_results_cache_module.os, "close", closed_descriptors.append) + + with pytest.raises(MemoryError, match="stack transfer failed"): + scan_results_cache_module._DarwinPathMonitor( + str(file_path), + ((str(tmp_path), 0, 0, 0, 0, 0),), + ) + + assert queue.closed is True + assert sorted(closed_descriptors) == opened_descriptors + + def test_cache_path_component_rejects_windows_reparse_point( tmp_path: Path, monkeypatch: pytest.MonkeyPatch,