diff --git a/crates/fs/src/fs_watcher.rs b/crates/fs/src/fs_watcher.rs index 99664ae74802e2..c844b7afe0300d 100644 --- a/crates/fs/src/fs_watcher.rs +++ b/crates/fs/src/fs_watcher.rs @@ -844,7 +844,27 @@ impl GlobalWatcher { drop(state); match self.watch(path.as_path(), mode) { Ok(()) => {} - Err(error) if mode == WatcherMode::Native && is_max_files_watch_error(&error) => { + Err(error) + if mode == WatcherMode::Native && is_native_watch_saturation_error(&error) => + { + // On macOS all native paths share one FSEventStream, and a watch + // registration tears the stream down before rebuilding it with the + // new path appended. When that rebuild fails (fd budget exhausted: + // each watched path costs ~11 descriptors), every previously + // working watch is left dead and the failed path stays in notify's + // path set, so any later registration would retry the same + // oversized set and fail too. Unwatching the failed path rebuilds + // the previous, known-good set, and the rescan tells the restored + // registrations to resync whatever changed while the stream was + // down (streams start at "now"; there is no event replay). + #[cfg(target_os = "macos")] + { + self.unwatch(path.as_path(), mode).log_err(); + self.enqueue( + mode, + Ok(Event::new(EventKind::Other).set_flag(notify::event::Flag::Rescan)), + ); + } self.start_native_watch_limit_cooldown(path.as_path()); return Ok(None); } @@ -1079,10 +1099,28 @@ impl GlobalWatcher { } } -fn is_max_files_watch_error(error: &anyhow::Error) -> bool { +/// Whether a native watch registration failed because the OS won't accept any +/// more watches from this process right now. +/// +/// On Linux this is inotify's `MaxFilesWatch` (`ENOSPC`/`EMFILE` on +/// `inotify_add_watch`). On macOS there is no distinct error kind: all native +/// paths share one `FSEventStream` costing ~11 file descriptors per watched +/// path, and when a rebuild of that stream exceeds the process fd budget, +/// `FSEventStreamStart` returns false and notify's FSEvents backend surfaces +/// it as a generic "unable to start FSEvent stream" error. Both mean the same +/// thing — the process has saturated the OS watcher capacity — so the caller +/// should roll back and back off via the cooldown rather than treat the path +/// as permanently unwatchable (which would silently stop delivering file +/// events, leaving buffers, the project panel, and git status stale). +fn is_native_watch_saturation_error(error: &anyhow::Error) -> bool { error .downcast_ref::() - .is_some_and(|error| matches!(&error.kind, notify::ErrorKind::MaxFilesWatch)) + .is_some_and(|error| match &error.kind { + notify::ErrorKind::MaxFilesWatch => true, + #[cfg(target_os = "macos")] + notify::ErrorKind::Generic(message) => message.contains("FSEvent"), + _ => false, + }) } static POLL_INTERVAL: LazyLock = LazyLock::new(|| { @@ -1160,6 +1198,7 @@ mod tests { watch_calls: Vec, unwatch_calls: Vec, fail_with_watch_limit: bool, + fail_with_fsevent_start: bool, } struct SharedFakeWatchBackend(Arc>); @@ -1172,6 +1211,11 @@ mod tests { if backend.fail_with_watch_limit { return Err(notify::Error::new(notify::ErrorKind::MaxFilesWatch)); } + if backend.fail_with_fsevent_start { + // Mirrors notify's macOS FSEvents backend when `FSEventStreamStart` + // returns false because the process has too many live streams. + return Err(notify::Error::generic("unable to start FSEvent stream")); + } backend.watched_paths.insert(path); Ok(()) } @@ -1329,6 +1373,66 @@ mod tests { assert_eq!(native_backend.watch_calls, &[first_path.to_path_buf()]); } + #[cfg(target_os = "macos")] + #[test] + fn fsevent_stream_start_failure_rolls_back_rescans_and_cools_down() { + // macOS has no distinct "too many watches" error kind: once the process + // exhausts its fd budget (~11 descriptors per path in the shared + // FSEventStream), the stream rebuild fails with a generic "unable to + // start FSEvent stream" — and because the rebuild tore down the previous + // stream first, every existing watch is dead at that point. The failed + // path must be rolled back (restoring the previous set), all native + // registrations told to rescan, and further registrations skipped for + // the cooldown instead of retrying the oversized set forever. + let native_backend = Arc::new(Mutex::new(FakeWatchBackend { + fail_with_fsevent_start: true, + ..Default::default() + })); + let (event_tx, event_rx) = async_channel::unbounded(); + let watcher = GlobalWatcher { + state: Mutex::new(WatcherState { + watchers: Default::default(), + native_path_registrations: Default::default(), + poll_path_registrations: Default::default(), + cooldown_until: None, + last_registration: Default::default(), + }), + native_watcher: Mutex::new(Some(Box::new(SharedFakeWatchBackend( + native_backend.clone(), + )))), + poll_watcher: Mutex::new(None), + event_tx, + }; + let first_path = Arc::::from(Path::new("/repo/first")); + let second_path = Arc::::from(Path::new("/repo/second")); + + let first_registration = watcher + .add(first_path.clone(), WatcherMode::Native, false, |_| {}) + .expect("fsevent start failure is handled as saturation, not an error"); + let second_registration = watcher + .add(second_path, WatcherMode::Native, false, |_| {}) + .expect("subsequent registration is skipped during cooldown"); + + assert!(first_registration.is_none()); + assert!(second_registration.is_none()); + + // The cooldown means only the first path ever reaches the OS backend; the + // second is skipped without a watch call. The failed path is unwatched to + // restore the previous stream. + { + let native_backend = native_backend.lock(); + assert_eq!(native_backend.watch_calls, &[first_path.to_path_buf()]); + assert_eq!(native_backend.unwatch_calls, &[first_path.to_path_buf()]); + } + + // All native registrations are told to rescan the window in which the + // stream was down. + let (mode, event) = event_rx.try_recv().expect("rescan event enqueued"); + assert_eq!(mode, WatcherMode::Native); + assert!(event.expect("rescan event is not an error").need_rescan()); + assert!(event_rx.is_empty(), "only one rescan per saturation"); + } + fn modify_event(path: &str) -> notify::Event { notify::Event { paths: vec![PathBuf::from(path)], diff --git a/crates/util/src/util.rs b/crates/util/src/util.rs index d4ebdfd5165f07..0d200a5ea20358 100644 --- a/crates/util/src/util.rs +++ b/crates/util/src/util.rs @@ -254,6 +254,55 @@ Error: Running Zed as root or via sudo is unsupported. } } +/// Raises the soft limit on open file descriptors to the maximum the OS allows. +/// +/// Processes launched from a GUI (Finder/launchd on macOS) inherit a soft limit +/// of only 256 open files (Apple's frameworks bump it to 2560 on first use of +/// some APIs), and Zed doesn't fit: language server pipes, sqlite, and sockets +/// aside, the macOS FSEvents machinery consumes roughly 11 descriptors per +/// watched path, so a language server registering a few hundred watched files +/// exhausts the budget. Once that happens, `FSEventStreamStart` fails and file +/// events silently stop, leaving buffers, git status, and the project panel +/// stale. Chromium (8192), the JVM, and Go all raise this limit at startup for +/// the same class of reasons. +#[cfg(unix)] +pub fn increase_open_file_limit() -> Result<()> { + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: `limit` is a valid rlimit struct for getrlimit to fill in. + if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) } != 0 { + return Err(anyhow::Error::from(std::io::Error::last_os_error()) + .context("getrlimit(RLIMIT_NOFILE)")); + } + + // macOS rejects rlim_cur values above OPEN_MAX even when rlim_max is + // RLIM_INFINITY (see setrlimit(2)). The libc crate doesn't expose OPEN_MAX, + // so use its value from . + #[cfg(target_os = "macos")] + let new_soft_limit = { + const OPEN_MAX: libc::rlim_t = 10240; + limit.rlim_max.min(OPEN_MAX) + }; + #[cfg(not(target_os = "macos"))] + let new_soft_limit = limit.rlim_max.min(65536); + + if limit.rlim_cur >= new_soft_limit { + return Ok(()); + } + + limit.rlim_cur = new_soft_limit; + // SAFETY: `limit` holds the values just read back from getrlimit, with only + // the soft limit raised (never above the hard limit). + if unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) } != 0 { + return Err(anyhow::Error::from(std::io::Error::last_os_error()) + .context("setrlimit(RLIMIT_NOFILE)")); + } + log::info!("raised open file soft limit to {new_soft_limit}"); + Ok(()) +} + #[cfg(unix)] fn load_shell_from_passwd() -> Result<()> { let buflen = match unsafe { libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) } { diff --git a/crates/zed/src/main.rs b/crates/zed/src/main.rs index eb933cc16e62b5..a963f136660efc 100644 --- a/crates/zed/src/main.rs +++ b/crates/zed/src/main.rs @@ -303,6 +303,9 @@ fn main() { } ztracing::init(); + #[cfg(unix)] + util::increase_open_file_limit().log_err(); + let version = option_env!("ZED_BUILD_ID"); let app_commit_sha = option_env!("ZED_COMMIT_SHA").map(|commit_sha| AppCommitSha::new(commit_sha.to_string()));