diff --git a/docs/runtime/environment-variables.mdx b/docs/runtime/environment-variables.mdx index bff70123a229..ec3f3eda98b7 100644 --- a/docs/runtime/environment-variables.mdx +++ b/docs/runtime/environment-variables.mdx @@ -215,6 +215,8 @@ Bun reads these environment variables to configure aspects of its behavior. | `FORCE_COLOR` | If `FORCE_COLOR=1`, then ANSI color output is forced on, even if `NO_COLOR` is set. | | `BUN_CONFIG_MAX_HTTP_REQUESTS` | Sets the maximum number of concurrent HTTP requests sent by fetch and `bun install`. Defaults to `256`. Lower it if you run into rate limits or connection issues. | | `BUN_CONFIG_NO_CLEAR_TERMINAL_ON_RELOAD` | If `BUN_CONFIG_NO_CLEAR_TERMINAL_ON_RELOAD=true`, then `bun --watch` does not clear the console on reload | +| `BUN_WATCHER_USE_POLLING` | If `BUN_WATCHER_USE_POLLING=1`, `--watch` and `--hot` poll the watched files with `stat()` instead of native filesystem events. Use it on Docker bind mounts, WSL `/mnt/*` paths, and network filesystems, where native events do not arrive. `0` keeps the native watcher. See [watch mode](/runtime/watch-mode#docker-wsl-and-network-filesystems). | +| `BUN_WATCHER_POLL_INTERVAL` | The interval, in milliseconds, between polls when `--watch` or `--hot` polls. Defaults to `100`. | | `DO_NOT_TRACK` | Disable uploading crash reports to `bun.report` on crash. On macOS & Windows, crash report uploads are enabled by default. Bun sends no other telemetry, though we plan to add some. If `DO_NOT_TRACK=1`, then auto-uploading crash reports and telemetry are both [disabled](https://do-not-track.dev/). | | `BUN_OPTIONS` | Prepends command-line arguments to any Bun execution. For example, `BUN_OPTIONS="--hot"` makes `bun run dev` behave like `bun --hot run dev`. | diff --git a/docs/runtime/watch-mode.mdx b/docs/runtime/watch-mode.mdx index 72860cdccb8c..1cf9c55ad6a6 100644 --- a/docs/runtime/watch-mode.mdx +++ b/docs/runtime/watch-mode.mdx @@ -82,6 +82,26 @@ bun --watch test signal, e.g. `bun --watch --watch-kill-signal SIGINT index.ts`. +### Docker, WSL, and network filesystems + +The native filesystem watcher APIs do not report a change that comes from the other side of a mount. The watch succeeds, and then no event arrives. This happens on: + +- Docker bind mounts of a Windows or macOS host directory +- WSL paths under `/mnt/c` +- NFS and SMB shares, and VM shared folders + +For these, `--watch` and `--hot` can poll: Bun calls `stat()` on each watched file and directory on an interval and reloads when the modification time, size, or inode of a file changes. On Linux, Bun turns polling on by itself when the directory you run it in is on a 9p, NFS, SMB, or Parallels filesystem, and prints a note. WSL `/mnt/*` paths are 9p, for example. For any other setup, set `BUN_WATCHER_USE_POLLING=1`: + +```bash terminal icon="terminal" +BUN_WATCHER_USE_POLLING=1 bun --watch index.ts +``` + +In a container, set it once with `ENV` in the Dockerfile or under `environment:` in the Compose file. + +`BUN_WATCHER_POLL_INTERVAL` sets the interval in milliseconds. The default is `100`. Polling uses more CPU than native events. `BUN_WATCHER_USE_POLLING=0` keeps the native watcher even where Bun would turn polling on. + +A watched directory changes when an entry is added, removed, or renamed. The dev server uses that to retry an import that failed, so it recovers when you create the missing file. + --- ## `--hot` mode diff --git a/src/bun_core/env_var.rs b/src/bun_core/env_var.rs index ece9747d32c2..bd4663f2a80e 100644 --- a/src/bun_core/env_var.rs +++ b/src/bun_core/env_var.rs @@ -129,6 +129,9 @@ platform_specific_new!(pub LIBRARY_PATH: string, posix = "LIBRARY_PATH", windows new!(pub BUN_TEST_DRAIN_EVENT_LOOP: boolean, "BUN_TEST_DRAIN_EVENT_LOOP", { default: false }); new!(pub BUN_TMPDIR: string, "BUN_TMPDIR", {}); new!(pub BUN_WATCHER_TRACE: string, "BUN_WATCHER_TRACE", {}); +// No default: unset lets `Watcher::init` pick polling from the filesystem type. +new!(pub BUN_WATCHER_USE_POLLING: boolean, "BUN_WATCHER_USE_POLLING", {}); +new!(pub BUN_WATCHER_POLL_INTERVAL: unsigned, "BUN_WATCHER_POLL_INTERVAL", {}); new!(pub CI: boolean, "CI", {}); new!(pub CI_COMMIT_SHA: string, "CI_COMMIT_SHA", {}); new!(pub CI_JOB_URL: string, "CI_JOB_URL", {}); diff --git a/src/bundler/bundle_v2.rs b/src/bundler/bundle_v2.rs index d19da17c6718..998eba15c6ae 100644 --- a/src/bundler/bundle_v2.rs +++ b/src/bundler/bundle_v2.rs @@ -4698,10 +4698,11 @@ pub mod bv2_impl { break 'add_watchers; } + let bun_watcher = this.bun_watcher_mut().unwrap(); // TODO: support explicit watchFiles array. this is not done // right now because DevServer requires a table to map // watched files and dirs to their respective dependants. - let fd = if bun_watcher::REQUIRES_FILE_DESCRIPTORS { + let fd = if bun_watcher.requires_file_descriptors() { let mut buf = bun_paths::path_buffer_pool::get(); // On kqueue platforms paths are already // posix-separated so `z()` alone suffices. @@ -4719,7 +4720,7 @@ pub mod bv2_impl { // Failures to watch are intentionally ignored. if !matches!( - this.bun_watcher_mut().unwrap().add_file::( + bun_watcher.add_file::( fd, &load.path, bun_wyhash::hash(load.path.as_ref()) as u32, diff --git a/src/jsc/hot_reloader.rs b/src/jsc/hot_reloader.rs index c06160e53efd..8cb09a7e2ac1 100644 --- a/src/jsc/hot_reloader.rs +++ b/src/jsc/hot_reloader.rs @@ -32,6 +32,14 @@ pub enum ImportWatcher { } impl ImportWatcher { + #[inline] + pub fn requires_file_descriptors(&self) -> bool { + match self { + ImportWatcher::Hot(w) | ImportWatcher::Watch(w) => w.requires_file_descriptors(), + ImportWatcher::None => false, + } + } + /// Look up the `package_json` column for `hash` under the watcher's /// mutex. /// diff --git a/src/runtime/bake/dev_server/mod.rs b/src/runtime/bake/dev_server/mod.rs index e8b18a04c1e0..846e7423a84e 100644 --- a/src/runtime/bake/dev_server/mod.rs +++ b/src/runtime/bake/dev_server/mod.rs @@ -1369,7 +1369,8 @@ impl DirectoryWatchStore { Ok(None) | Err(_) => None, }; - let (fd, owned_fd): (bun_sys::Fd, bool) = if bun_watcher::REQUIRES_FILE_DESCRIPTORS { + let requires_fds = self.dev_bun_watcher().requires_file_descriptors(); + let (fd, owned_fd): (bun_sys::Fd, bool) = if requires_fds { if let Some(fd) = cache_fd { (fd, false) } else { @@ -1402,7 +1403,7 @@ impl DirectoryWatchStore { (bun_sys::Fd::INVALID, false) }; let fd_guard = scopeguard::guard(fd, move |fd| { - if bun_watcher::REQUIRES_FILE_DESCRIPTORS && owned_fd { + if requires_fds && owned_fd { fd.close(); } }); diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index 2badada656dd..56716b83e185 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -3410,9 +3410,14 @@ fn transpile_source_code_inner( { break 'auto_watch; } + // SAFETY: `bun_watcher` is the `*mut ImportWatcher` + // set when `is_watcher_enabled()`; cast recovers the concrete + // type. + let watcher = + unsafe { &mut *(*jsc_vm).bun_watcher.cast::() }; // kqueue watchers need a file descriptor to receive event - // notifications on it; inotify/win32 watch by path. - let input_fd = if bun_watcher::REQUIRES_FILE_DESCRIPTORS { + // notifications on it; inotify/win32/polling watch by path. + let input_fd = if watcher.requires_file_descriptors() { let mut buf = bun_paths::path_buffer_pool::get(); if path.text.len() >= buf.len() { break 'auto_watch; @@ -3426,11 +3431,6 @@ fn transpile_source_code_inner( bun_sys::Fd::INVALID }; let hash = bun_watcher::Watcher::get_hash(path.text); - // SAFETY: `bun_watcher` is the `*mut ImportWatcher` - // set when `is_watcher_enabled()`; cast recovers the concrete - // type. - let watcher = - unsafe { &mut *(*jsc_vm).bun_watcher.cast::() }; let added = watcher.add_file::(input_fd, path.text, hash, bun_sys::Fd::INVALID, None); if !matches!(added, Ok(bun_watcher::FdOwnership::Watcher)) { diff --git a/src/watcher/INotifyWatcher.rs b/src/watcher/INotifyWatcher.rs index 23637c21ca76..9eb31ad3e5da 100644 --- a/src/watcher/INotifyWatcher.rs +++ b/src/watcher/INotifyWatcher.rs @@ -373,7 +373,7 @@ pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> bun_sys::Result<()> { use crate::watcher_impl::WatchItemColumns; let _flush = Output::flush_guard(); - let events = this.platform.read()?; + let events = this.platform.native_mut().read()?; if events.is_empty() { return Ok(()); } diff --git a/src/watcher/KEventWatcher.rs b/src/watcher/KEventWatcher.rs index c784b8636584..99460d0b179d 100644 --- a/src/watcher/KEventWatcher.rs +++ b/src/watcher/KEventWatcher.rs @@ -52,7 +52,7 @@ fn watch_event_from_kevent(kevent: &libc::kevent) -> WatchEvent { pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> bun_sys::Result<()> { let _flush = Output::flush_guard(); - let fd = this.platform.fd; + let fd = this.platform.native_mut().fd; let mut changelist: [libc::kevent; CHANGELIST_COUNT] = bun_core::ffi::zeroed(); diff --git a/src/watcher/PollingWatcher.rs b/src/watcher/PollingWatcher.rs new file mode 100644 index 000000000000..93177a823e27 --- /dev/null +++ b/src/watcher/PollingWatcher.rs @@ -0,0 +1,264 @@ +//! Stat-polling watch backend, for mounts that accept a native watch and never +//! report a change made on the other side (9p, NFS, SMB). See `Watcher::init`. + +use std::time::Duration; + +use bun_collections::{HashMap, IdentityContext}; +use bun_core::{Timespec, ZStr}; +use bun_sys::{self as sys, PosixStat}; + +use crate::watcher_impl::{ + Backend, HashType, MAX_COUNT, Op, WatchEvent, WatchItemColumns, WatchItemIndex, Watcher, +}; + +pub(crate) const DEFAULT_INTERVAL_MS: u64 = 100; + +/// Whether `root` is on a Linux filesystem where inotify misses changes made by +/// the host or another client. FUSE and overlayfs mostly work, so are not listed. +pub(crate) fn should_auto_poll(root: &[u8]) -> bool { + #[cfg(target_os = "linux")] + { + // include/uapi/linux/magic.h, fs/smb/client/cifsglob.h, fs/smb/common/smb2pdu.h + const V9FS_MAGIC: u32 = 0x0102_1997; + const NFS_SUPER_MAGIC: u32 = 0x6969; + const SMB_SUPER_MAGIC: u32 = 0x517B; + const CIFS_SUPER_MAGIC: u32 = 0xFF53_4D42; + const SMB2_SUPER_MAGIC: u32 = 0xFE53_4D42; + // Parallels shared folders (`prl_fs`). + const PRL_FS_MAGIC: u32 = 0x7C7C_6673; + + let mut buf = bun_paths::path_buffer_pool::get(); + if root.len() >= buf.len() { + return false; + } + buf[..root.len()].copy_from_slice(root); + buf[root.len()] = 0; + let Ok(st) = sys::statfs(ZStr::from_buf(&buf[..], root.len())) else { + return false; + }; + // Every magic fits in 32 bits; `f_type` is a 64-bit word on our targets. + matches!( + st.f_type as u32, + V9FS_MAGIC + | NFS_SUPER_MAGIC + | SMB_SUPER_MAGIC + | CIFS_SUPER_MAGIC + | SMB2_SUPER_MAGIC + | PRL_FS_MAGIC + ) + } + #[cfg(not(target_os = "linux"))] + { + let _ = root; + false + } +} + +#[derive(Clone, Copy, Default, PartialEq, Eq)] +struct Snapshot { + exists: bool, + mtime: Timespec, + size: u64, + ino: u64, +} + +#[derive(Clone, Copy, Default)] +struct Tracked { + last: Snapshot, + /// The previous poll missed the path. Only the second miss in a row is a + /// DELETE, so a two-step save (unlink or rename away, then create) is one WRITE. + missing_once: bool, +} + +struct Candidate { + index: WatchItemIndex, + hash: HashType, + /// The NUL-terminated path is `paths[path_start..][..path_len]`. + path_start: u32, + path_len: u32, + /// `None`: `stat` failed with an errno that does not mean "gone". + now: Option, +} + +pub(crate) struct PollingWatcher { + interval: Duration, + /// Keyed by `WatchItem.hash`. Guarded by `Watcher.mutex`. + tracked: HashMap>, + /// Watcher-thread scratch, reused by every cycle. + candidates: Vec, + paths: Vec, +} + +impl PollingWatcher { + pub(crate) fn new(interval_ms: u64) -> Self { + Self { + interval: Duration::from_millis(interval_ms), + tracked: HashMap::default(), + candidates: Vec::new(), + paths: Vec::new(), + } + } + + /// Takes the baseline when the path joins the watchlist, so a write that + /// lands before the first poll is a change. Caller holds `Watcher.mutex`. + pub(crate) fn register(&mut self, hash: HashType, path: &[u8]) { + let mut buf = bun_paths::path_buffer_pool::get(); + let snapshot = if path.len() < buf.len() { + buf[..path.len()].copy_from_slice(path); + buf[path.len()] = 0; + stat_path(ZStr::from_buf(&buf[..], path.len())) + } else { + None + }; + match snapshot { + Some(last) => { + self.tracked.insert( + hash, + Tracked { + last, + missing_once: false, + }, + ); + } + // Unknown for now. The first poll that can stat it sets the baseline. + None => { + self.tracked.remove(&hash); + } + } + } + + /// Caller holds `Watcher.mutex`. + pub(crate) fn unregister(&mut self, hash: HashType) { + self.tracked.remove(&hash); + } +} + +/// `None` for any errno other than ENOENT/ENOTDIR: a network mount returns ESTALE, +/// EIO or ETIMEDOUT for a file that is still there. The caller keeps the snapshot. +fn stat_path(path: &ZStr) -> Option { + match sys::stat(path) { + Ok(st) => { + let st = PosixStat::init(&st); + Some(Snapshot { + exists: true, + mtime: st.mtim, + size: st.size, + ino: st.ino, + }) + } + Err(err) => match err.get_errno() { + sys::E::ENOENT | sys::E::ENOTDIR => Some(Snapshot::default()), + _ => None, + }, + } +} + +/// One cycle: sleep, copy the watched paths out under the mutex, `stat` them +/// with the mutex released, then diff and dispatch under the mutex. +pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> sys::Result<()> { + let _flush = bun_core::output::flush_guard(); + + let Backend::Polling(poll) = &mut this.platform else { + unreachable!("polling::watch_loop_cycle on the native backend") + }; + + // `shutdown()` only clears `running`, so sleep in slices to notice it soon. + const SLICE: Duration = Duration::from_millis(20); + let mut remaining = poll.interval; + while !remaining.is_zero() { + if !this.running.load() { + return Ok(()); + } + let step = remaining.min(SLICE); + std::thread::sleep(step); + remaining -= step; + } + if !this.running.load() { + return Ok(()); + } + + let mut candidates = core::mem::take(&mut poll.candidates); + let mut paths = core::mem::take(&mut poll.paths); + candidates.clear(); + paths.clear(); + + // Other threads only append. Entries move only in `flush_evictions`, on this + // thread inside the dispatch below, so an index read here stays valid. + // Directories too: their mtime moves when an entry is added or removed. + { + let _guard = this.mutex.lock_guard(); + let file_paths = this.watchlist.items_file_path(); + let hashes = this.watchlist.items_hash(); + for (i, path) in file_paths.iter().enumerate() { + candidates.push(Candidate { + index: i as WatchItemIndex, + hash: hashes[i], + path_start: paths.len() as u32, + path_len: path.len() as u32, + now: None, + }); + paths.extend_from_slice(path); + paths.push(0); + } + } + + for c in &mut candidates { + let start = c.path_start as usize; + c.now = stat_path(ZStr::from_buf(&paths[start..], c.path_len as usize)); + } + + let mut event_count: usize = 0; + { + let _guard = this.mutex.lock_guard(); + let Backend::Polling(poll) = &mut this.platform else { + unreachable!() + }; + for c in &candidates { + let Some(now) = c.now else { continue }; + let tracked = poll.tracked.entry(c.hash).or_insert_with(|| Tracked { + last: now, + missing_once: false, + }); + + let op = if now.exists { + tracked.missing_once = false; + if now == tracked.last { + continue; + } + Op::WRITE + } else if !tracked.last.exists { + continue; + } else if !tracked.missing_once { + tracked.missing_once = true; + continue; + } else { + Op::DELETE + }; + + // The baseline moves only with an emitted event. The next poll + // finds a change that does not fit in this batch. + if event_count == MAX_COUNT { + break; + } + tracked.last = now; + tracked.missing_once = false; + this.watch_events[event_count] = WatchEvent { + index: c.index, + op, + name_off: 0, + name_len: 0, + }; + event_count += 1; + } + } + + if event_count > 0 { + this.dispatch_file_updates(event_count, 0); + } + + if let Backend::Polling(poll) = &mut this.platform { + poll.candidates = candidates; + poll.paths = paths; + } + Ok(()) +} diff --git a/src/watcher/Watcher.rs b/src/watcher/Watcher.rs index 177487c6fe4c..f027b55384bd 100644 --- a/src/watcher/Watcher.rs +++ b/src/watcher/Watcher.rs @@ -8,6 +8,7 @@ use bun_core::{ThreadLock, ZStr, feature_flags, output as Output, strings, zstr} use bun_sys::{self as sys, Fd}; use bun_threading::Mutex; +use crate::polling_watcher::{self as polling, PollingWatcher}; use crate::watcher_trace as WatcherTrace; // Android: same kernel inotify ABI as glibc/musl Linux, so list both. @@ -27,11 +28,6 @@ bun_core::define_scoped_log!(log, watcher, visible); pub const MAX_COUNT: usize = 128; -#[cfg(any(target_os = "macos", target_os = "freebsd"))] -pub const REQUIRES_FILE_DESCRIPTORS: bool = true; -#[cfg(not(any(target_os = "macos", target_os = "freebsd")))] -pub const REQUIRES_FILE_DESCRIPTORS: bool = false; - /// Open flags for an fd that exists only to receive kqueue VNODE events. /// Darwin has O_EVTONLY (no read/write access requested); FreeBSD has no /// equivalent, so the watch fd is a plain O_RDONLY. @@ -85,6 +81,32 @@ impl AnyResolveWatcher { // ideally, the constants above can be inlined pub(crate) type Platform = platform::Platform; +/// The native backend of this target, or stat polling. `Watcher::init` selects. +// One boxed `Watcher` per process: the size difference costs nothing. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Backend { + Native(Platform), + Polling(PollingWatcher), +} + +impl Backend { + /// For the native `watch_loop_cycle`, which `watch_loop` only runs on `Native`. + #[inline] + pub(crate) fn native_mut(&mut self) -> &mut Platform { + match self { + Backend::Native(p) => p, + Backend::Polling(_) => unreachable!("native_mut() on polling backend"), + } + } + + fn stop(&mut self) { + match self { + Backend::Native(p) => p.stop(), + Backend::Polling(_) => {} + } + } +} + /// `?[:0]u8` — name of a changed file inside a watched directory, borrowed /// from the platform's event buffer (inotify event names / kqueue udata). /// Ownership stays with the platform buffer for the duration of one @@ -100,7 +122,7 @@ pub struct Watcher { pub(crate) changed_filepaths: [ChangedFilePath; MAX_COUNT], /// The platform-specific implementation of the watcher - pub(crate) platform: Platform, + pub(crate) platform: Backend, pub watchlist: WatchList, pub mutex: Mutex, @@ -187,6 +209,29 @@ impl Watcher { unsafe { (*ctx_opaque.cast::()).on_watch_error(err) } } + let use_polling = match bun_core::env_var::BUN_WATCHER_USE_POLLING::get() { + Some(v) => v, + None if polling::should_auto_poll(top_level_dir) => { + // `--watch` prints this again in every reloaded process. + bun_core::note!( + "{} is on a filesystem that does not report file changes, so Bun polls the watched files. BUN_WATCHER_USE_POLLING=1 hides this note. BUN_WATCHER_USE_POLLING=0 uses native file events.", + bstr::BStr::new(top_level_dir), + ); + true + } + None => false, + }; + let platform = if use_polling { + let interval = bun_core::env_var::BUN_WATCHER_POLL_INTERVAL + .get() + .filter(|&ms| ms > 0) + .unwrap_or(polling::DEFAULT_INTERVAL_MS); + log!("using polling backend (interval={}ms)", interval); + Backend::Polling(PollingWatcher::new(interval)) + } else { + Backend::Native(Platform::new(top_level_dir)?) + }; + let this = Box::new(Watcher { watchlist: WatchList::default(), mutex: Mutex::default(), @@ -194,7 +239,7 @@ impl Watcher { ctx: ctx.cast::<()>(), on_file_update: on_file_update_wrapped::, on_error: on_error_wrapped::, - platform: Platform::new(top_level_dir)?, + platform, watch_events: vec![WatchEvent::default(); MAX_COUNT].into_boxed_slice(), changed_filepaths: [const { None }; MAX_COUNT], watchloop_handle: bun_core::AtomicCell::new(false), @@ -299,7 +344,9 @@ impl Watcher { if close_descriptors && me.running.load() { let fds = me.watchlist.items_fd(); for &fd in fds { - let _ = bun_sys::close(fd); + if fd.is_valid() { + let _ = bun_sys::close(fd); + } } } true @@ -317,6 +364,14 @@ impl Watcher { bun_wyhash::hash(filepath) as HashType } + /// The kqueue backend watches by fd, so callers open one per file. + /// inotify, Windows and polling watch by path and take `Fd::INVALID`. + #[inline] + pub fn requires_file_descriptors(&self) -> bool { + cfg!(any(target_os = "macos", target_os = "freebsd")) + && matches!(self.platform, Backend::Native(_)) + } + /// # Safety /// `this` must be the unique heap pointer returned from [`init`]. The /// watcher thread takes ownership: after `watch_loop` exits, this function @@ -370,7 +425,9 @@ impl Watcher { if self.close_descriptors.load() { let fds = self.watchlist.items_fd(); for &fd in fds { - let _ = bun_sys::close(fd); + if fd.is_valid() { + let _ = bun_sys::close(fd); + } } } owner_still_alive @@ -437,6 +494,9 @@ impl Watcher { if item == last_item || self.watchlist.len() <= item as usize { continue; } + if let Backend::Polling(p) = &mut self.platform { + p.unregister(self.watchlist.items_hash()[item as usize]); + } // Frees an owned `file_path`; the fd was closed in the first pass. drop(self.watchlist.swap_remove(item as usize)); @@ -463,7 +523,10 @@ impl Watcher { fn watch_loop(&mut self) -> sys::Result<()> { while self.running.load() { // individual platform implementation will call onFileUpdate - platform::watch_loop_cycle(self)?; + match self.platform { + Backend::Native(_) => platform::watch_loop_cycle(self)?, + Backend::Polling(_) => polling::watch_loop_cycle(self)?, + } } Ok(()) } @@ -489,6 +552,11 @@ impl Watcher { use libc::{EV_ADD, EV_CLEAR, EV_ENABLE, EVFILT_VNODE, kevent as KEvent}; use libc::{NOTE_DELETE, NOTE_RENAME, NOTE_WRITE}; + // The polling backend has no kqueue and watches by path. + let Backend::Native(platform::Platform { fd: kqueue_fd, .. }) = self.platform else { + return; + }; + // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/kqueue.2.html let mut event: KEvent = bun_core::ffi::zeroed(); @@ -508,7 +576,7 @@ impl Watcher { // Basically: // - We register the event here. // our while(true) loop above receives notification of changes to any of the events created here. - let _ = bun_sys::kevent(self.platform.fd, &[event], &mut [], None); + let _ = bun_sys::kevent(kqueue_fd, &[event], &mut [], None); } fn append_file_assume_capacity( @@ -547,10 +615,13 @@ impl Watcher { Cow::Borrowed(unsafe { bun_collections::detach_lifetime(file_path) }) }; + if let Backend::Polling(p) = &mut self.platform { + p.register(hash, file_path); + } #[cfg(any(target_os = "macos", target_os = "freebsd"))] self.add_file_descriptor_to_kqueue_without_checks(fd, watchlist_id); #[cfg(any(target_os = "linux", target_os = "android"))] - let eventlist_index = { + let eventlist_index = if let Backend::Native(p) = &mut self.platform { // inotify needs a trailing NUL. When // CLONE_FILE_PATH is true the caller's `file_path` is NOT NUL-terminated, // so we must copy into a NUL-terminated scratch buffer (mirrors the @@ -566,7 +637,9 @@ impl Watcher { // interned in `bun.fs.FileSystem` with a NUL sentinel at [len]. unsafe { ZStr::from_raw(file_path.as_ptr(), file_path.len()) } }; - self.platform.watch_path(slice)? + p.watch_path(slice)? + } else { + 0 }; self.watchlist.append_assume_capacity(WatchItem { @@ -601,7 +674,8 @@ impl Watcher { } } - let fd = if stored_fd.is_valid() { + // Polling stats the path and needs no descriptor. + let fd = if stored_fd.is_valid() || matches!(self.platform, Backend::Polling(_)) { stored_fd } else { bun_sys::open_a(file_path, 0, 0)? @@ -625,10 +699,13 @@ impl Watcher { #[cfg(any(target_os = "macos", target_os = "freebsd"))] let watchlist_id = self.watchlist.len(); + if let Backend::Polling(p) = &mut self.platform { + p.register(hash, file_path); + } #[cfg(any(target_os = "macos", target_os = "freebsd"))] self.add_file_descriptor_to_kqueue_without_checks(fd, watchlist_id); #[cfg(any(target_os = "linux", target_os = "android"))] - let eventlist_index = { + let eventlist_index = if let Backend::Native(p) = &mut self.platform { let mut buf = bun_paths::path_buffer_pool::get(); let path: &ZStr = if CLONE_FILE_PATH && !file_path.is_empty() @@ -648,9 +725,9 @@ impl Watcher { ZStr::from_buf(&buf[..], trailing_slash.len()) }; - self.platform - .watch_dir(path) - .map_err(|e| e.with_path(file_path))? + p.watch_dir(path).map_err(|e| e.with_path(file_path))? + } else { + 0 }; self.watchlist.append_assume_capacity(WatchItem { @@ -824,8 +901,7 @@ impl Watcher { } // Only open fd if we might need it - #[cfg(any(target_os = "macos", target_os = "freebsd"))] - let fd: Fd = { + let fd: Fd = if self.requires_file_descriptors() { let mut path_z = bun_paths::path_buffer_pool::get(); if file_path.len() >= path_z.len() { return false; @@ -839,9 +915,9 @@ impl Watcher { Ok(opened) => opened, Err(_) => return false, } + } else { + Fd::INVALID }; - #[cfg(not(any(target_os = "macos", target_os = "freebsd")))] - let fd: Fd = Fd::INVALID; let res = self.add_file::(fd, file_path, hash, Fd::INVALID, None); match res { diff --git a/src/watcher/WindowsWatcher.rs b/src/watcher/WindowsWatcher.rs index e39adde6e990..66cccc32ae90 100644 --- a/src/watcher/WindowsWatcher.rs +++ b/src/watcher/WindowsWatcher.rs @@ -3,7 +3,7 @@ use core::mem::size_of; use core::ptr; -use crate::watcher_impl::{Op, WatchEvent, WatchItemColumns, WatchItemIndex, Watcher}; +use crate::watcher_impl::{Backend, Op, WatchEvent, WatchItemColumns, WatchItemIndex, Watcher}; use bun_core::strings; use bun_paths::PathBuffer; use bun_paths::resolve_path::{ParentEqual, is_parent_or_equal}; @@ -383,17 +383,29 @@ pub(crate) enum Timeout { None = 0, } +/// `&mut this.platform.buf` for the native backend, with no `&mut Platform` in +/// between: that borrow also covers `Platform::watcher` and invalidates the +/// pointer a live `EventIterator` holds into it. +macro_rules! path_buf { + ($this:ident) => { + match $this.platform { + Backend::Native(WindowsWatcher { ref mut buf, .. }) => buf, + Backend::Polling(_) => unreachable!(), + } + }; +} + pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> bun_sys::Result<()> { // We re-borrow buf inside the inner loop instead of holding `&this.platform.buf` // across calls to `this.platform.next()`. - let base_idx = this.platform.base_idx; + let base_idx = this.platform.native_mut().base_idx; let mut event_id: usize = 0; // first wait has infinite timeout - we're waiting for the next event and don't want to spin let mut timeout = Timeout::Infinite; loop { - let mut iter = match this.platform.next(timeout)? { + let mut iter = match this.platform.native_mut().next(timeout)? { Some(it) => it, None => break, }; @@ -412,13 +424,13 @@ pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> bun_sys::Result<()> { // outer loop reiterates) — encapsulated by the `RawSlice` invariant. let filename: &[u16] = event.filename.slice(); let convert_res = - strings::copy_utf16_into_utf8(&mut this.platform.buf[base_idx..], filename); + strings::copy_utf16_into_utf8(&mut path_buf!(this)[base_idx..], filename); let eventpath_len = base_idx + convert_res.written as usize; bun_core::scoped_log!( watcher, "watcher update event: (filename: {}, action: {}", - bstr::BStr::new(&this.platform.buf[..eventpath_len]), + bstr::BStr::new(&path_buf!(this)[..eventpath_len]), <&'static str>::from(event.action) ); @@ -437,7 +449,7 @@ pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> bun_sys::Result<()> { // are released before we touch `this.watch_events` or hand the // whole `&mut Watcher` to `process_watch_event_batch`. let rel = { - let eventpath = &this.platform.buf[..eventpath_len]; + let eventpath = &path_buf!(this)[..eventpath_len]; let path = &this.watchlist.items_file_path()[item_idx]; let rel = is_parent_or_equal(path.as_ref(), eventpath); bun_core::scoped_log!( @@ -469,7 +481,7 @@ pub(crate) fn watch_loop_cycle(this: &mut Watcher) -> bun_sys::Result<()> { // then dereference a pointer with invalidated provenance — UB that MIRI flags. // The callee never touches `platform.watcher`, so re-deriving the pointer // here from the now-current `&mut Watcher` restores valid provenance. - iter.watcher = BackRef::new(&this.platform.watcher); + iter.watcher = BackRef::new(&this.platform.native_mut().watcher); // Reset event_id to start a new batch event_id = 0; } diff --git a/src/watcher/lib.rs b/src/watcher/lib.rs index 76619cf6836e..251cbd0c2b97 100644 --- a/src/watcher/lib.rs +++ b/src/watcher/lib.rs @@ -21,6 +21,9 @@ pub mod kevent_watcher; #[path = "WindowsWatcher.rs"] pub mod windows_watcher; +#[path = "PollingWatcher.rs"] +pub(crate) mod polling_watcher; + #[path = "WatcherTrace.rs"] pub(crate) mod watcher_trace; @@ -36,6 +39,6 @@ pub use error::{Error, Result}; pub use WatchItemKind as Kind; pub use watcher_impl::{ AnyResolveWatcher, ChangedFilePath, Event, FdOwnership, HashType, MAX_COUNT, - MAX_EVICTION_COUNT, Op, PackageJSON, REQUIRES_FILE_DESCRIPTORS, WATCH_OPEN_FLAGS, WatchEvent, - WatchItem, WatchItemColumns, WatchItemIndex, WatchItemKind, WatchList, Watcher, WatcherContext, + MAX_EVICTION_COUNT, Op, PackageJSON, WATCH_OPEN_FLAGS, WatchEvent, WatchItem, WatchItemColumns, + WatchItemIndex, WatchItemKind, WatchList, Watcher, WatcherContext, }; diff --git a/test/cli/watch/watch.test.ts b/test/cli/watch/watch.test.ts index d9315f4994c5..f28141df0de5 100644 --- a/test/cli/watch/watch.test.ts +++ b/test/cli/watch/watch.test.ts @@ -3,7 +3,7 @@ import { spawn } from "bun"; import { afterEach, expect, it } from "bun:test"; import { bunEnv, bunExe, isBroken, isLinux, isWindows, tempDir, tmpdirSync } from "harness"; import { readdirSync, rmSync } from "node:fs"; -import { join } from "node:path"; +import { basename, join } from "node:path"; let watchee: Subprocess; @@ -260,6 +260,152 @@ int pthread_create(pthread_t *t, const pthread_attr_t *a, void *(*f)(void *), vo expect(exitCode).not.toBe(0); }); +// https://github.com/oven-sh/bun/issues/5841 +// On a Docker bind mount from a Windows host or a WSL /mnt/c path, +// inotify_add_watch succeeds and the kernel never delivers an event. The shim +// reproduces that: the native watcher blocks in read() and nothing reloads. +// BUN_WATCHER_USE_POLLING=1 selects the backend that stats the watched files. +const INOTIFY_NEVER_FIRES_C = /* c */ ` +static int next_wd = 1; +int inotify_add_watch(int fd, const char *path, unsigned int mask) { + (void)fd; (void)path; (void)mask; + return next_wd++; +} +`; + +// Spawns `bun watchee.js` in `cwd` with the polling backend on. With +// `blindInotify`, `cwd` must contain shim.c, and the native watcher gets no event. +async function spawnPollingWatchee( + cwd: string, + mode: "--watch" | "--hot", + blindInotify: boolean, + extraEnv: Record = {}, +) { + const env: Record = { + ...bunEnv, + BUN_WATCHER_USE_POLLING: "1", + BUN_WATCHER_POLL_INTERVAL: "20", + ...extraEnv, + }; + if (blindInotify) { + const shimPath = join(cwd, "shim.so"); + await using ccProc = Bun.spawn({ + cmd: [cc!, "-shared", "-fPIC", "-o", shimPath, join(cwd, "shim.c")], + env: bunEnv, + stderr: "pipe", + stdout: "pipe", + }); + const [ccOut, ccErr, ccExit] = await Promise.all([ccProc.stdout.text(), ccProc.stderr.text(), ccProc.exited]); + if (ccExit !== 0) throw new Error(`shim compile failed: ${ccErr || ccOut}`); + env.LD_PRELOAD = bunEnv.LD_PRELOAD ? `${shimPath}:${bunEnv.LD_PRELOAD}` : shimPath; + } + return spawn({ + cwd, + cmd: [bunExe(), mode, "--no-clear-screen", "watchee.js"], + env, + stdout: "pipe", + stderr: "inherit", + stdin: "ignore", + }); +} + +async function expectPollingReloads(mode: "--watch" | "--hot", blindInotify: boolean) { + // --hot keeps one process, so the script must stay alive. The content + // changes length on every write: same-size writes inside one mtime tick + // are invisible to stat. + const source = (i: number) => + `console.log("tick ${i}");\n//${Buffer.alloc(i, "-").toString()}\n` + + (mode === "--hot" ? "setInterval(() => {}, 1 << 30);\n" : ""); + using dir = tempDir("watch-poll", { + ...(blindInotify ? { "shim.c": INOTIFY_NEVER_FIRES_C } : {}), + "watchee.js": source(0), + }); + const cwd = String(dir); + watchee = await spawnPollingWatchee(cwd, mode, blindInotify); + const { waitFor, release, output } = stdoutWaiter(watchee); + for (let i = 0; i < 3; i++) { + await waitFor(`tick ${i}\n`); + await Bun.write(join(cwd, "watchee.js"), source(i + 1)); + } + await waitFor("tick 3\n"); + release(); + expect(output()).toContain("tick 3\n"); + // The child's cwd is `dir`, and Windows cannot remove a directory that is + // the cwd of a live process. + watchee.kill("SIGKILL"); + await watchee.exited; +} + +// Consumers use a directory event to look at a directory again: the resolver +// drops its cached listing, the dev server retries an import that failed. The +// polling backend reports one when the mtime of a watched directory moves. +async function expectPollingSeesNewDirectoryEntry(blindInotify: boolean) { + using dir = tempDir("watch-poll-dir", { + ...(blindInotify ? { "shim.c": INOTIFY_NEVER_FIRES_C } : {}), + "watchee.js": `console.log("ready");\nsetInterval(() => {}, 1 << 30);\n`, + }); + // Not inside `dir`: creating the trace file would itself change `dir`. + using traceDir = tempDir("watch-poll-trace", {}); + const trace = join(String(traceDir), "trace.log"); + const cwd = String(dir); + watchee = await spawnPollingWatchee(cwd, "--hot", blindInotify, { BUN_WATCHER_TRACE: trace }); + const { waitFor, release } = stdoutWaiter(watchee); + await waitFor("ready\n"); + release(); + + // Not in the module graph, so only the directory can report it. + await Bun.write(join(cwd, "created.js"), "export {};\n"); + + // The trace has one JSON line per batch of events, keyed by watched path. + // The key of the directory ends with a separator. + const isWatchedDir = (path: string) => /[\\/]$/.test(path) && path.replace(/[\\/]+$/, "").endsWith(basename(cwd)); + let dirEvents: string[] = []; + while (dirEvents.length === 0) { + const text = await Bun.file(trace) + .text() + .catch(() => ""); + // Drop a last line that is still being written. + const lines = text + .slice(0, text.lastIndexOf("\n") + 1) + .split("\n") + .filter(Boolean); + dirEvents = lines.flatMap(line => + Object.entries(JSON.parse(line).files as Record) + .filter(([path]) => isWatchedDir(path)) + .flatMap(([, file]) => file.events), + ); + if (dirEvents.length === 0) await Bun.sleep(20); + } + expect(dirEvents).toContain("write"); + watchee.kill("SIGKILL"); + await watchee.exited; +} + +for (const mode of ["--watch", "--hot"] as const) { + it.skipIf(!isLinux || !cc)( + `${mode} with BUN_WATCHER_USE_POLLING=1 reloads when inotify never delivers an event`, + () => expectPollingReloads(mode, true), + 30000, + ); + + // No shim, so this also passes on the native backend. It is here to run the + // polling backend on macOS and Windows. + it(`${mode} with BUN_WATCHER_USE_POLLING=1 reloads`, () => expectPollingReloads(mode, false), 30000); +} + +it.skipIf(!isLinux || !cc)( + "BUN_WATCHER_USE_POLLING=1 reports a new entry in a watched directory when inotify never delivers an event", + () => expectPollingSeesNewDirectoryEntry(true), + 30000, +); + +// No shim: runs the directory poll on macOS and Windows too. +it( + "BUN_WATCHER_USE_POLLING=1 reports a new entry in a watched directory", + () => expectPollingSeesNewDirectoryEntry(false), + 30000, +); + // A script that registers a SIGTERM handler and then spins in synchronous // code must still restart on file change: the watcher thread posts the reload // to the JS thread first (so listeners can run), but forces the reload itself