refactor(natives): split grep and fs_cache into responsibility modules

crates/atomic-natives/src/fs_cache.rs

@@ -1,4 +1,112 @@
-// @generated split wrapper for Atomic native filesystem-cache parity with oh-my-pi.
-// DO NOT EDIT: part files preserve the upstream single-module scope used by native grep/glob.
-include!("fs_cache/part_01.rs");
-include!("fs_cache/part_02.rs");
+// DO NOT EDIT: copied from can1357/oh-my-pi commit 15b5c1397fc059673e3b0bcbc50b074e6dc1f9d8 for Atomic issue #1483 parity.
+// Shared filesystem scan cache for discovery tools (glob, fd).
+//
+// Provides a TTL-based cache of scanned directory entries, with:
+// - Global policy (no per-call TTL tuning)
+// - Explicit invalidation for agent file mutations
+// - Empty-result fast recheck to avoid stale negatives
+//
+// # Policy Configuration (environment overrides)
+// - `FS_SCAN_CACHE_TTL_MS`       – default `1000`
+// - `FS_SCAN_EMPTY_RECHECK_MS`   – default `200`
+// - `FS_SCAN_CACHE_MAX_ENTRIES`   – default `16`
+
+use std::path::PathBuf;
+
+use napi_derive::napi;
+
+// ═══════════════════════════════════════════════════════════════════════════
+// Public types (re-exported by glob for backward compatibility)
+// ═══════════════════════════════════════════════════════════════════════════
+
+/// Resolved filesystem entry kind for glob filters and match metadata.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[napi]
+pub enum FileType {
+	/// Regular file.
+	File = 1,
+	/// Directory.
+	Dir = 2,
+	/// Symbolic link.
+	Symlink = 3,
+}
+
+/// A single filesystem entry from a directory scan.
+#[derive(Clone)]
+#[napi(object)]
+pub struct GlobMatch {
+	/// Relative path from the search root, using forward slashes.
+	pub path: String,
+	/// Resolved filesystem type for the match.
+	pub file_type: FileType,
+	/// Modification time in milliseconds since Unix epoch (from
+	/// `symlink_metadata`).
+	pub mtime: Option<f64>,
+	/// File size in bytes for regular files.
+	pub size: Option<f64>,
+}
+
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+pub enum ScanDetail {
+	Minimal,
+	Full,
+}
+
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
+pub struct ScanOptions {
+	pub include_hidden: bool,
+	pub use_gitignore: bool,
+	pub skip_node_modules: bool,
+	pub follow_links: bool,
+	pub detail: ScanDetail,
+}
+
+/// Invalidate the filesystem scan cache.
+///
+/// When called with a path, removes entries for roots containing that path.
+/// When called without a path, clears the entire cache.
+///
+/// Intended to be called after agent file mutations (write, edit, rename,
+/// delete).
+#[napi]
+pub fn invalidate_fs_scan_cache(path: Option<String>) {
+	match path {
+		Some(p) => {
+			let candidate = PathBuf::from(&p);
+			let absolute = if candidate.is_absolute() {
+				candidate
+			} else if let Ok(cwd) = std::env::current_dir() {
+				cwd.join(candidate)
+			} else {
+				PathBuf::from(&p)
+			};
+			let target = std::fs::canonicalize(&absolute)
+				.or_else(|_| {
+					absolute
+						.parent()
+						.and_then(|parent| std::fs::canonicalize(parent).ok())
+						.and_then(|parent| absolute.file_name().map(|name| parent.join(name)))
+						.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::NotFound))
+				})
+				.unwrap_or(absolute);
+			invalidate_path(&target);
+		},
+		None => invalidate_all(),
+	}
+}
+
+mod cache;
+mod paths;
+mod policy;
+#[cfg(test)]
+mod test_util;
+mod walker;
+
+pub use cache::{ScanResult, force_rescan, get_or_scan, invalidate_all, invalidate_path};
+pub use paths::{
+	classify_file_type, contains_component, normalize_relative_path, resolve_search_path,
+	should_skip_path,
+};
+pub use policy::{cache_ttl_ms, empty_recheck_ms, grep_workers, max_cache_entries};
+pub use walker::build_walker;
+pub(crate) use walker::collect_entry;

crates/atomic-natives/src/fs_cache/cache.rs

@@ -0,0 +1,237 @@
+// TTL cache, epochs, and invalidation for filesystem scans.
+
+use std::{
+	path::{Path, PathBuf},
+	sync::{
+		LazyLock,
+		atomic::{AtomicU64, Ordering},
+	},
+	time::{Duration, Instant},
+};
+
+use dashmap::DashMap;
+use napi::bindgen_prelude::Result;
+
+use super::{
+	GlobMatch, ScanDetail, ScanOptions, cache_ttl_ms, max_cache_entries, walker::collect_entries,
+};
+use crate::task;
+
+// ═══════════════════════════════════════════════════════════════════════════
+// Cache internals
+// ═══════════════════════════════════════════════════════════════════════════
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+struct CacheKey {
+	root: PathBuf,
+	include_hidden: bool,
+	use_gitignore: bool,
+	skip_node_modules: bool,
+	detail: ScanDetail,
+}
+
+#[derive(Clone)]
+struct CacheEntry {
+	created_at: Instant,
+	epoch: u64,
+	entries: Vec<GlobMatch>,
+}
+
+static FS_CACHE: LazyLock<DashMap<CacheKey, CacheEntry>> = LazyLock::new(DashMap::new);
+static FS_CACHE_EPOCH: AtomicU64 = AtomicU64::new(0);
+
+fn cache_epoch() -> u64 {
+	FS_CACHE_EPOCH.load(Ordering::Acquire)
+}
+
+fn bump_cache_epoch() {
+	FS_CACHE_EPOCH.fetch_add(1, Ordering::AcqRel);
+}
+
+/// Result of a cache-aware scan, including the age of the cached data.
+pub struct ScanResult {
+	/// Scanned filesystem entries.
+	pub entries: Vec<GlobMatch>,
+	/// How old the cached data is in milliseconds (0 = freshly scanned).
+	pub cache_age_ms: u64,
+}
+
+fn evict_oldest() {
+	while FS_CACHE.len() > max_cache_entries() {
+		let Some(oldest_key) = FS_CACHE
+			.iter()
+			.min_by_key(|entry| entry.value().created_at)
+			.map(|entry| entry.key().clone())
+		else {
+			break;
+		};
+		FS_CACHE.remove(&oldest_key);
+	}
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// Cache API
+// ═══════════════════════════════════════════════════════════════════════════
+
+/// Returns scanned entries using the global TTL cache policy.
+///
+/// The returned [`ScanResult::cache_age_ms`] lets callers implement
+/// empty-result fast recheck: if a query produces zero matches and the cache is
+/// older than [`empty_recheck_ms()`](super::empty_recheck_ms), call
+/// [`force_rescan`] before returning empty.
+pub fn get_or_scan(
+	root: &Path,
+	options: ScanOptions,
+	ct: &task::CancelToken,
+) -> Result<ScanResult> {
+	let ttl = cache_ttl_ms();
+	if ttl == 0 {
+		// Caching disabled – always scan fresh.
+		let entries = collect_entries(root, options, ct)?;
+		return Ok(ScanResult { entries, cache_age_ms: 0 });
+	}
+
+	let key = CacheKey {
+		root: root.to_path_buf(),
+		include_hidden: options.include_hidden,
+		use_gitignore: options.use_gitignore,
+		skip_node_modules: options.skip_node_modules,
+		detail: options.detail,
+	};
+
+	let now = Instant::now();
+	if let Some(entry) = FS_CACHE.get(&key) {
+		let current_epoch = cache_epoch();
+		let age = now.duration_since(entry.created_at);
+		if entry.epoch == current_epoch && age < Duration::from_millis(ttl) {
+			return Ok(ScanResult {
+				entries: entry.entries.clone(),
+				cache_age_ms: age.as_millis() as u64,
+			});
+		}
+		drop(entry);
+		FS_CACHE.remove(&key);
+	}
+
+	let scan_epoch = cache_epoch();
+	let entries = collect_entries(root, options, ct)?;
+	FS_CACHE.insert(
+		key,
+		CacheEntry { created_at: Instant::now(), epoch: scan_epoch, entries: entries.clone() },
+	);
+	evict_oldest();
+	Ok(ScanResult { entries, cache_age_ms: 0 })
+}
+
+/// Force a fresh scan, replacing any existing cache entry.
+///
+/// Use when a cached query produced zero matches and the cache was old enough
+/// to warrant a recheck. When `store` is false, the fresh scan result is
+/// returned without repopulating the cache.
+pub fn force_rescan(
+	root: &Path,
+	options: ScanOptions,
+	store: bool,
+	ct: &task::CancelToken,
+) -> Result<Vec<GlobMatch>> {
+	let key = CacheKey {
+		root: root.to_path_buf(),
+		include_hidden: options.include_hidden,
+		use_gitignore: options.use_gitignore,
+		skip_node_modules: options.skip_node_modules,
+		detail: options.detail,
+	};
+	bump_cache_epoch();
+	FS_CACHE.remove(&key);
+
+	let scan_epoch = cache_epoch();
+	let entries = collect_entries(root, options, ct)?;
+	if store {
+		FS_CACHE.insert(
+			key,
+			CacheEntry { created_at: Instant::now(), epoch: scan_epoch, entries: entries.clone() },
+		);
+		evict_oldest();
+	}
+	Ok(entries)
+}
+
+// ═══════════════════════════════════════════════════════════════════════════
+// Invalidation
+// ═══════════════════════════════════════════════════════════════════════════
+
+/// Invalidate cache entries whose root contains `target`.
+///
+/// Removes any cache entry whose root is a prefix of (or equal to) `target`,
+/// because a file mutation under that root makes the scan stale.
+pub fn invalidate_path(target: &Path) {
+	bump_cache_epoch();
+	let keys_to_remove: Vec<CacheKey> = FS_CACHE
+		.iter()
+		.filter(|entry| target.starts_with(&entry.key().root))
+		.map(|entry| entry.key().clone())
+		.collect();
+	for key in keys_to_remove {
+		FS_CACHE.remove(&key);
+	}
+}
+
+/// Clear the entire scan cache.
+pub fn invalidate_all() {
+	bump_cache_epoch();
+	FS_CACHE.clear();
+}
+
+#[cfg(test)]
+mod tests {
+	use std::fs;
+
+	use crate::fs_cache::test_util::TempDirGuard;
+
+	#[test]
+	fn force_rescan_respects_skip_node_modules() {
+		let root = TempDirGuard::new();
+		// Create a nested node_modules with many files
+		for i in 0..100 {
+			let pkg_dir = root.path().join(format!("node_modules/pkg-{i}"));
+			fs::create_dir_all(&pkg_dir).unwrap();
+			fs::write(pkg_dir.join("index.js"), "x").unwrap();
+		}
+		fs::write(root.path().join("app.js"), "ok").unwrap();
+
+		let ct = crate::task::CancelToken::default();
+
+		// With skip: should only get app.js
+		let entries = super::force_rescan(
+			root.path(),
+			super::ScanOptions {
+				include_hidden: true,
+				use_gitignore: false,
+				skip_node_modules: true,
+				follow_links: false,
+				detail: super::ScanDetail::Full,
+			},
+			false,
+			&ct,
+		)
+		.unwrap();
+		assert_eq!(entries.len(), 1, "skip=true got: {}", entries.len());
+		assert_eq!(entries[0].path, "app.js");
+
+		// Without skip: should get app.js + 100 node_modules files + directories
+		let entries = super::force_rescan(
+			root.path(),
+			super::ScanOptions {
+				include_hidden: true,
+				use_gitignore: false,
+				skip_node_modules: false,
+				follow_links: false,
+				detail: super::ScanDetail::Full,
+			},
+			false,
+			&ct,
+		)
+		.unwrap();
+		assert!(entries.len() > 100, "skip=false got: {}", entries.len());
+	}
+}