diff --git a/CHANGELOG.md b/CHANGELOG.md index c8f674f18..80ebdd4be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Place cross-volume Windows cache identity probes near the volume root instead of the nearest ancestor of the scanned path, so a probe can no longer appear inside a directory tree that a concurrent scan is walking. - Keep published scan-result cache entries readable when concurrent or interrupted hits update access metadata. - Preserve macOS scan-result cache entries during unrelated temporary-file churn while rejecting replaced source files and directories. +- Keep macOS scan-result caching enabled when hashing updates a model file's access time. ## [0.2.52](https://github.com/promptfoo/modelaudit/compare/v0.2.51...v0.2.52) (2026-07-22) diff --git a/modelaudit/cache/scan_results_cache.py b/modelaudit/cache/scan_results_cache.py index 95aefb5ed..0bdd8fcae 100644 --- a/modelaudit/cache/scan_results_cache.py +++ b/modelaudit/cache/scan_results_cache.py @@ -226,6 +226,10 @@ def __init__(self, file_path: str, ancestor_identity: tuple[AncestorEntry, ...]) self._queue: Any = None self._descriptors: list[int] = [] self._descriptor_stack: ExitStack | None = None + self._file_descriptor: int | None = None + self._attribute_flag = 0 + self._event_error_flag = 0 + self._unsafe_event_seen = False try: self._queue = select_module.kqueue() @@ -237,6 +241,8 @@ def __init__(self, file_path: str, ancestor_identity: tuple[AncestorEntry, ...]) | select_module.KQ_NOTE_REVOKE | select_module.KQ_NOTE_ATTRIB ) + self._attribute_flag = select_module.KQ_NOTE_ATTRIB + self._event_error_flag = getattr(select_module, "KQ_EV_ERROR", 0) descriptor_flags = getattr(os, "O_EVTONLY", os.O_RDONLY) | getattr(os, "O_CLOEXEC", 0) descriptor_flags |= getattr(os, "O_NOFOLLOW", 0) paths = [os.path.abspath(file_path), *(entry[0] for entry in ancestor_identity)] @@ -266,6 +272,7 @@ def __init__(self, file_path: str, ancestor_identity: tuple[AncestorEntry, ...]) try: self._descriptor_stack = transferred_stack self._descriptors = opened_descriptors + self._file_descriptor = opened_descriptors[0] except BaseException: with suppress(OSError): transferred_stack.close() @@ -274,8 +281,29 @@ def __init__(self, file_path: str, ancestor_identity: tuple[AncestorEntry, ...]) self.close() raise - def changed(self) -> bool: + def discard_validated_file_attribute_event(self) -> None: + """Discard only pending pure file attribute events after identity validation.""" if self._queue is None: + self._unsafe_event_seen = True + return + + try: + events = self._queue.control(None, max(len(self._descriptors), 1), 0) + except (OSError, ValueError): + self._unsafe_event_seen = True + return + + for event in events: + if ( + getattr(event, "ident", None) != self._file_descriptor + or getattr(event, "fflags", None) != self._attribute_flag + or (self._event_error_flag and getattr(event, "flags", 0) & self._event_error_flag) + ): + self._unsafe_event_seen = True + return + + def changed(self) -> bool: + if self._unsafe_event_seen or self._queue is None: return True try: return bool(self._queue.control(None, max(len(self._descriptors), 1), 0)) @@ -285,6 +313,7 @@ def changed(self) -> bool: def close(self) -> None: queue = getattr(self, "_queue", None) self._queue = None + self._file_descriptor = None descriptor_stack = getattr(self, "_descriptor_stack", None) self._descriptor_stack = None if queue is not None: @@ -929,14 +958,29 @@ def store_result( if self._get_file_change_token(file_path, file_stat) != expected_change_token: logger.debug("Skipping cache store for %s: file change token changed during scan", file_path) return False + if not self._ancestor_identity_matches_for_store( + expected_ancestor_identity, + self._capture_ancestor_identity(file_path), + ): + logger.debug("Skipping cache store for %s: ancestor path changed during scan", file_path) + return False + self._discard_validated_file_attribute_event(expected_ancestor_identity) + settled_pre_hash_stat = os.stat(file_path) + if not self._stat_matches(settled_pre_hash_stat, expected_file_stat): + logger.debug("Skipping cache store for %s: file metadata changed while starting store", file_path) + return False + if self._get_file_change_token(file_path, settled_pre_hash_stat) != expected_change_token: + logger.debug("Skipping cache store for %s: file changed while starting store", file_path) + return False if self._ancestor_monitor_changed( expected_ancestor_identity ) or not self._ancestor_identity_matches_for_store( expected_ancestor_identity, self._capture_ancestor_identity(file_path), ): - logger.debug("Skipping cache store for %s: ancestor path changed during scan", file_path) + logger.debug("Skipping cache store for %s: ancestor path changed while starting store", file_path) return False + file_stat = settled_pre_hash_stat verified_current_hash = self.hasher.hash_file_with_stat(file_path, file_stat) if verified_current_hash != expected_file_hash: @@ -949,15 +993,29 @@ def store_result( if self._get_file_change_token(file_path, post_hash_stat) != expected_change_token: logger.debug("Skipping cache store for %s: file changed during verification", file_path) return False + if not self._ancestor_identity_matches_for_store( + expected_ancestor_identity, + self._capture_ancestor_identity(file_path), + ): + logger.debug("Skipping cache store for %s: ancestor path changed during verification", file_path) + return False + self._discard_validated_file_attribute_event(expected_ancestor_identity) + settled_post_hash_stat = os.stat(file_path) + if not self._stat_matches(settled_post_hash_stat, expected_file_stat): + logger.debug("Skipping cache store for %s: file metadata changed while settling monitor", file_path) + return False + if self._get_file_change_token(file_path, settled_post_hash_stat) != expected_change_token: + logger.debug("Skipping cache store for %s: file changed while settling monitor", file_path) + return False if self._ancestor_monitor_changed( expected_ancestor_identity ) or not self._ancestor_identity_matches_for_store( expected_ancestor_identity, self._capture_ancestor_identity(file_path), ): - logger.debug("Skipping cache store for %s: ancestor path changed during verification", file_path) + logger.debug("Skipping cache store for %s: ancestor path changed while settling monitor", file_path) return False - file_stat = post_hash_stat + file_stat = settled_post_hash_stat version_info = self._get_version_info(version_context) if version_info is None: @@ -1162,7 +1220,6 @@ def _capture_file_identity_leased( if ( not self._stat_matches(initial_stat, verified_stat) or initial_change_token != verified_change_token - or self._ancestor_monitor_changed(monitored_ancestor_identity) or not self._ancestor_identity_matches_for_store( monitored_ancestor_identity, verified_ancestor_identity, @@ -1170,7 +1227,28 @@ def _capture_file_identity_leased( ): raise ValueError(f"File changed while capturing cache identity: {file_path}") - return verified_stat, content_hash, verified_change_token, monitored_ancestor_identity + self._discard_validated_file_attribute_event(monitored_ancestor_identity) + settled_stat = os.stat(file_path) + settled_change_token = self._get_file_change_token(file_path, settled_stat) + settled_ancestor_identity = self._capture_ancestor_identity(file_path) + + if ( + not self._stat_matches(initial_stat, settled_stat) + or initial_change_token != settled_change_token + or self._ancestor_monitor_changed(monitored_ancestor_identity) + or not self._ancestor_identity_matches_for_store( + monitored_ancestor_identity, + settled_ancestor_identity, + ) + ): + raise ValueError(f"File changed while settling cache identity monitor: {file_path}") + + return ( + settled_stat, + content_hash, + settled_change_token, + monitored_ancestor_identity, + ) except ValueError as exc: self.release_ancestor_identity(monitored_ancestor_identity) if str(exc).startswith("File changed while"): @@ -1405,6 +1483,12 @@ def _ancestor_identity_matches_for_store(expected: AncestorIdentity, current: An def _ancestor_monitor_changed(identity: AncestorIdentity) -> bool: return identity.monitor is not None and identity.monitor.changed() + @staticmethod + def _discard_validated_file_attribute_event(identity: AncestorIdentity) -> None: + discard_event = getattr(identity.monitor, "discard_validated_file_attribute_event", None) + if callable(discard_event): + discard_event() + def _file_identity_matches(self, file_path: str, expected: ScannedFileIdentity) -> bool: expected_stat, _expected_hash, expected_change_token, expected_ancestor_identity = expected try: diff --git a/tests/cache/test_cache_correctness.py b/tests/cache/test_cache_correctness.py index 14c4ce358..a028c55a2 100644 --- a/tests/cache/test_cache_correctness.py +++ b/tests/cache/test_cache_correctness.py @@ -1835,6 +1835,19 @@ def test_darwin_cache_identity_rejects_restored_ancestor_mode_change(tmp_path: P assert cache.store_result(str(file_path), {"success": True}, **identity) is False +@pytest.mark.skipif(sys.platform != "darwin", reason="requires Darwin vnode monitoring") +def test_darwin_cache_identity_rejects_restored_file_mode_change(tmp_path: Path) -> None: + file_path = _make_cacheable_file(tmp_path) + cache = ScanResultsCache(str(tmp_path / "cache")) + identity = _identity_kwargs(cache, str(file_path)) + original_mode = stat.S_IMODE(file_path.stat().st_mode) + + file_path.chmod(original_mode ^ stat.S_IXUSR) + file_path.chmod(original_mode) + + assert cache.store_result(str(file_path), {"success": True}, **identity) is False + + @pytest.mark.skipif(sys.platform != "darwin", reason="requires Darwin vnode monitoring") def test_darwin_cache_identity_rejects_restored_file_replacement(tmp_path: Path) -> None: file_path = _make_cacheable_file(tmp_path) @@ -1992,20 +2005,227 @@ def kevent(*args: Any, **kwargs: Any) -> tuple[tuple[Any, ...], dict[str, Any]]: return StubDarwinSelect -def test_darwin_path_monitor_reports_file_and_ancestor_attribute_changes( +def _install_portable_darwin_monitor( + file_path: Path, + queue: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + original_open = os.open + original_darwin_monitor = scan_results_cache_module._DarwinPathMonitor + + def open_regular_file(_path: str, _flags: int) -> int: + return original_open(file_path, os.O_RDONLY) + + def create_darwin_monitor(monitored_path: str, ancestor_identity: tuple[Any, ...]) -> Any: + with monkeypatch.context() as constructor_patch: + constructor_patch.setattr(scan_results_cache_module.os, "open", open_regular_file) + return original_darwin_monitor(monitored_path, ancestor_identity) + + monkeypatch.setattr(scan_results_cache_module, "select", _stub_darwin_select(queue)) + monkeypatch.setattr(scan_results_cache_module.sys, "platform", "darwin") + monkeypatch.setattr(scan_results_cache_module, "_DARWIN_STABLE_SYMLINK_ALIASES", {}) + monkeypatch.setattr(scan_results_cache_module, "_DarwinPathMonitor", create_darwin_monitor) + + +def test_darwin_path_monitor_rejects_file_mode_change_in_final_store_window( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + file_path = _make_cacheable_file(tmp_path) + cache_directory = tmp_path / "cache" + cache = ScanResultsCache(str(cache_directory)) + original_stat = os.stat + mode_changed = False + + class StatWithMode: + def __init__(self, result: os.stat_result, mode: int) -> None: + self._result = result + self.st_mode = mode + + def __getattr__(self, name: str) -> Any: + return getattr(self._result, name) + + def stat_with_mode_change(path: Any, *args: Any, **kwargs: Any) -> os.stat_result: + result = original_stat(path, *args, **kwargs) + try: + normalized_path = os.path.abspath(os.fsdecode(path)) + except TypeError: + return result + if normalized_path != str(file_path): + return result + mode = result.st_mode | stat.S_IXUSR if mode_changed else result.st_mode & ~stat.S_IXUSR + return cast(os.stat_result, StatWithMode(result, mode)) + + class FinalWindowAttributeQueue: + def __init__(self) -> None: + self.registered_events: list[Any] = [] + self.triggered = False + + def control(self, changes: Any, _max_events: int, _timeout: int) -> list[Any]: + nonlocal mode_changed + if changes is not None: + self.registered_events.extend(changes) + return [] + if not self.triggered and any(cache_directory.rglob("*.tmp")): + self.triggered = True + mode_changed = True + file_event = self.registered_events[0] + if file_event[1]["fflags"] & 64: + return [SimpleNamespace(ident=file_event[0][0], fflags=64, flags=0)] + return [] + + def close(self) -> None: + return None + + queue = FinalWindowAttributeQueue() + _install_portable_darwin_monitor(file_path, queue, monkeypatch) + monkeypatch.setattr(scan_results_cache_module.os, "stat", stat_with_mode_change) + identity = _identity_kwargs(cache, str(file_path)) + scan_result = {"success": True, "metadata": {"executable": False}} + + stored = cache.store_result(str(file_path), scan_result, **identity) + cached_result = cache.get_cached_result(str(file_path)) + + assert queue.triggered is True + assert { + "stored": stored, + "actual_executable": bool(stat.S_IMODE(os.stat(file_path).st_mode) & stat.S_IXUSR), + "cached_executable": cached_result["metadata"]["executable"] if cached_result else None, + "cache_hit": cached_result is not None, + } == { + "stored": False, + "actual_executable": True, + "cached_executable": None, + "cache_hit": False, + } + + +def test_darwin_path_monitor_allows_read_between_capture_and_store( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + file_path = _make_cacheable_file(tmp_path) + cache = ScanResultsCache(str(tmp_path / "cache")) + + class ReadAttributeQueue: + def __init__(self) -> None: + self.registered_events: list[Any] = [] + self.pending_file_attributes = 0 + self.drained_file_attributes = 0 + + def schedule_file_attribute(self) -> None: + self.pending_file_attributes += 1 + + def control(self, changes: Any, _max_events: int, _timeout: int) -> list[Any]: + if changes is not None: + self.registered_events.extend(changes) + return [] + if self.pending_file_attributes: + self.pending_file_attributes -= 1 + self.drained_file_attributes += 1 + file_event = self.registered_events[0] + return [SimpleNamespace(ident=file_event[0][0], fflags=64, flags=0)] + return [] + + def close(self) -> None: + return None + + queue = ReadAttributeQueue() + _install_portable_darwin_monitor(file_path, queue, monkeypatch) + original_hash = cache.hasher.hash_file_with_stat + + def hash_with_file_attribute(path: str, file_stat: os.stat_result) -> str: + result = original_hash(path, file_stat) + queue.schedule_file_attribute() + return result + + with monkeypatch.context() as hash_patch: + hash_patch.setattr(cache.hasher, "hash_file_with_stat", hash_with_file_attribute) + identity = _identity_kwargs(cache, str(file_path)) + file_path.read_bytes() + queue.schedule_file_attribute() + scan_result = {"success": True, "metadata": {"scanner_read": True}} + try: + stored = cache.store_result(str(file_path), scan_result, **identity) + finally: + cache.release_ancestor_identity(identity["expected_ancestor_identity"]) + + cached_result = cache.get_cached_result(str(file_path)) + + assert stored is True + assert cached_result == scan_result + assert queue.drained_file_attributes == 3 + assert queue.pending_file_attributes == 0 + + +def test_darwin_path_monitor_ignores_file_access_time_attribute_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FileAccessQueue: + def __init__(self) -> None: + self.registered_events: list[Any] = [] + self.pending = True + + def control(self, changes: Any, _max_events: int, _timeout: int) -> list[Any]: + if changes is not None: + self.registered_events.extend(changes) + return [] + if self.pending: + self.pending = False + file_event = self.registered_events[0] + return [SimpleNamespace(ident=file_event[0][0], fflags=64, flags=0)] + return [] + + def close(self) -> None: + return None + + file_path = _make_cacheable_file(tmp_path) + queue = FileAccessQueue() + select_stub = _stub_darwin_select(queue) + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + + def open_path(_path: str, _flags: int) -> int: + descriptor = 100 + len(opened_descriptors) + opened_descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(scan_results_cache_module, "select", select_stub) + monkeypatch.setattr(scan_results_cache_module.os, "open", open_path) + monkeypatch.setattr(scan_results_cache_module.os, "close", closed_descriptors.append) + + monitor = scan_results_cache_module._DarwinPathMonitor( + str(file_path), + ((str(tmp_path), 0, 0, 0, 0, 0),), + ) + + assert len(queue.registered_events) == 2 + assert queue.registered_events[0][1]["fflags"] & select_stub.KQ_NOTE_ATTRIB + assert queue.registered_events[1][1]["fflags"] & select_stub.KQ_NOTE_ATTRIB + monitor.discard_validated_file_attribute_event() + assert monitor.changed() is False + monitor.close() + assert sorted(closed_descriptors) == opened_descriptors + + +def test_darwin_path_monitor_reports_ancestor_attribute_changes( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: class AttributeAwareQueue: def __init__(self) -> None: self.registered_events: list[Any] = [] + self.pending = True def control(self, changes: Any, _max_events: int, _timeout: int) -> list[Any]: if changes is not None: self.registered_events.extend(changes) return [] - if any(event[1]["fflags"] & 64 for event in self.registered_events): - return [object()] + if self.pending: + self.pending = False + ancestor_event = self.registered_events[1] + return [SimpleNamespace(ident=ancestor_event[0][0], fflags=64, flags=0)] return [] def close(self) -> None: @@ -2032,7 +2252,63 @@ def open_path(_path: str, _flags: int) -> int: ) assert len(queue.registered_events) == 2 - assert all(event[1]["fflags"] & select_stub.KQ_NOTE_ATTRIB for event in queue.registered_events) + assert queue.registered_events[0][1]["fflags"] & select_stub.KQ_NOTE_ATTRIB + assert queue.registered_events[1][1]["fflags"] & select_stub.KQ_NOTE_ATTRIB + monitor.discard_validated_file_attribute_event() + assert monitor.changed() is True + monitor.close() + assert sorted(closed_descriptors) == opened_descriptors + + +def test_darwin_path_monitor_preserves_non_attribute_file_events( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class MixedFileEventQueue: + def __init__(self) -> None: + self.registered_events: list[Any] = [] + self.pending = True + + def control(self, changes: Any, _max_events: int, _timeout: int) -> list[Any]: + if changes is not None: + self.registered_events.extend(changes) + return [] + if self.pending: + self.pending = False + file_event = self.registered_events[0] + return [ + SimpleNamespace( + ident=file_event[0][0], + fflags=select_stub.KQ_NOTE_ATTRIB | select_stub.KQ_NOTE_DELETE, + flags=0, + ) + ] + return [] + + def close(self) -> None: + return None + + file_path = _make_cacheable_file(tmp_path) + queue = MixedFileEventQueue() + select_stub = _stub_darwin_select(queue) + opened_descriptors: list[int] = [] + closed_descriptors: list[int] = [] + + def open_path(_path: str, _flags: int) -> int: + descriptor = 100 + len(opened_descriptors) + opened_descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(scan_results_cache_module, "select", select_stub) + monkeypatch.setattr(scan_results_cache_module.os, "open", open_path) + monkeypatch.setattr(scan_results_cache_module.os, "close", closed_descriptors.append) + + monitor = scan_results_cache_module._DarwinPathMonitor( + str(file_path), + ((str(tmp_path), 0, 0, 0, 0, 0),), + ) + + monitor.discard_validated_file_attribute_event() assert monitor.changed() is True monitor.close() assert sorted(closed_descriptors) == opened_descriptors