From 020b00173e42f219083835233fcee12097fdd805 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Tue, 2 Jun 2026 23:10:40 -0600 Subject: [PATCH] refactor: Split process child module --- crates/turborepo-process/src/child.rs | 3097 +---------------- crates/turborepo-process/src/child/handle.rs | 648 ++++ crates/turborepo-process/src/child/io.rs | 315 ++ .../turborepo-process/src/child/shutdown.rs | 241 ++ crates/turborepo-process/src/child/state.rs | 92 + crates/turborepo-process/src/child/test.rs | 1413 ++++++++ .../turborepo-process/src/child/test_guard.rs | 80 + 7 files changed, 2961 insertions(+), 2925 deletions(-) create mode 100644 crates/turborepo-process/src/child/handle.rs create mode 100644 crates/turborepo-process/src/child/io.rs create mode 100644 crates/turborepo-process/src/child/shutdown.rs create mode 100644 crates/turborepo-process/src/child/state.rs create mode 100644 crates/turborepo-process/src/child/test.rs create mode 100644 crates/turborepo-process/src/child/test_guard.rs diff --git a/crates/turborepo-process/src/child.rs b/crates/turborepo-process/src/child.rs index 41cb057865720..9eb51c3f19fd5 100644 --- a/crates/turborepo-process/src/child.rs +++ b/crates/turborepo-process/src/child.rs @@ -15,2987 +15,234 @@ //! running these processes to completion, forwarding signals, and closing //! them when the manager is closed. -const CHILD_POLL_INTERVAL: Duration = Duration::from_micros(50); -const POST_EXIT_OUTPUT_DRAIN_TIMEOUT: Duration = Duration::from_millis(100); -#[cfg(unix)] -const PTY_PROCESS_GROUP_SIGINT_DELAY: Duration = Duration::from_secs(1); -#[cfg(any(unix, windows))] -const PROCESS_TREE_DRAIN_POLL_INTERVAL: Duration = Duration::from_millis(10); -#[cfg(windows)] -const WINDOWS_DESCENDANT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); +#[path = "child/io.rs"] +mod child_io; +mod handle; +mod shutdown; +mod state; +#[cfg(test)] +mod test; +#[cfg(test)] +mod test_guard; -use std::{ - fmt, - io::{self, BufRead, Read, Write}, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - }, - time::Duration, +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, }; -use portable_pty::{Child as PtyChild, MasterPty as PtyController, native_pty_system}; -use tokio::{ - io::{AsyncBufRead, AsyncBufReadExt, BufReader}, - process::Command as TokioCommand, - sync::{mpsc, watch}, -}; +use tokio::sync::watch; use tracing::{debug, trace}; -use super::{Command, PtySize}; - -// The atomic covers `cargo test`'s in-process parallelism; flock covers -// nextest's process-per-test parallelism. +#[cfg(unix)] +use self::handle::{TargetIdentity, process_group_matches_identity, signal_process_group}; #[cfg(test)] -static PTY_TEST_LOCK: AtomicBool = AtomicBool::new(false); +use self::test_guard::PtyTestGuard; +pub use self::{ + child_io::ChildStdin, + shutdown::{ChildExit, ShutdownStyle}, +}; +use self::{ + child_io::{ChildIO, ChildInput, ChildOutput}, + handle::{ChildHandle, SpawnResult}, + state::{ChildCommand, ChildCommandChannel, ChildStateManager}, +}; +use super::{Command, PtySize}; -#[cfg(test)] -#[derive(Debug)] -struct PtyTestGuard { +/// A child process that can be interacted with asynchronously. +/// +/// This is a wrapper around the `tokio::process::Child` struct, which provides +/// a cross platform interface for spawning and managing child processes. +#[derive(Clone, Debug)] +pub struct Child { + pid: Option, #[cfg(unix)] - file: std::fs::File, + target_identity: Option, + command_channel: ChildCommandChannel, + exit_channel: watch::Receiver>, + stdin: Arc>>, + output: Arc>>, + label: String, + shutdown_style: ShutdownStyle, + /// Flag indicating this child is being stopped as part of a shutdown of the + /// ProcessManager, rather than individually stopped. + closing: Arc, + #[cfg(test)] + _pty_test_guard: Option>, } -#[cfg(test)] -impl PtyTestGuard { - fn acquire() -> Self { - while PTY_TEST_LOCK - .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - std::thread::sleep(Duration::from_millis(10)); - } +impl Child { + /// Start a child process, returning a handle that can be used to interact + /// with it. The command will be started immediately. + #[tracing::instrument(skip(command), fields(command = command.label()))] + pub fn spawn( + command: Command, + shutdown_style: ShutdownStyle, + pty_size: Option, + ) -> std::io::Result { + let label = command.label(); + #[cfg(test)] + let pty_test_guard = pty_size.map(|_| Arc::new(PtyTestGuard::acquire())); + let SpawnResult { + handle: mut child, + io: ChildIO { stdin, output }, + controller, + } = if let Some(size) = pty_size { + ChildHandle::spawn_pty(command, size) + } else { + ChildHandle::spawn_normal(command) + }?; + let pid = child.pid(); #[cfg(unix)] - { - use std::{fs::OpenOptions, os::fd::AsRawFd}; - - let path = std::env::temp_dir().join("turborepo-process-pty.lock"); - let file = match OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(path) - { - Ok(file) => file, - Err(err) => { - PTY_TEST_LOCK.store(false, Ordering::Release); - panic!("failed to open PTY test lock: {err}"); - } - }; - let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) }; - if result != 0 { - PTY_TEST_LOCK.store(false, Ordering::Release); - panic!( - "failed to lock PTY test lock: {}", - io::Error::last_os_error() - ); - } - Self { file } - } - - #[cfg(not(unix))] - { - Self {} - } - } -} - -#[cfg(all(test, unix))] -impl Drop for PtyTestGuard { - fn drop(&mut self) { - use std::os::fd::AsRawFd; - - let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; - PTY_TEST_LOCK.store(false, Ordering::Release); - } -} - -#[cfg(all(test, not(unix)))] -impl Drop for PtyTestGuard { - fn drop(&mut self) { - PTY_TEST_LOCK.store(false, Ordering::Release); - } -} + let target_identity = child.target_identity; -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum ChildExit { - Finished(Option), - /// The child process exited during graceful shutdown. - Interrupted, - /// The child process was killed, it could either be explicitly killed or it - /// did not respond to an interrupt and was killed as a result - Killed, - /// The child process was killed by someone else. Note that on - /// windows, it is not possible to distinguish between whether - /// the process exited normally or was killed - KilledExternal, - Failed, -} + let (command_tx, mut command_rx) = ChildCommandChannel::new(); -#[derive(Debug, Clone, Copy)] -pub enum ShutdownStyle { - /// On Unix this sends SIGINT to the process group. On Windows, Turbo cannot - /// send a signal directly, so it waits for an externally delivered console - /// event or an explicit kill. - /// - /// `Graceful(Some(timeout))` escalates to `Kill` after `timeout` elapses. - /// `Graceful(None)` waits indefinitely until an explicit `Kill` command - /// arrives. - Graceful(Option), + // we use a watch channel to communicate the exit code back to the + // caller. we are interested in three cases: + // - the child process exits + // - the child process is killed (and doesn't have an exit code) + // - the child process fails somehow (some syscall fails) + let (exit_tx, exit_rx) = watch::channel(None); - Kill, -} + let _task = tokio::spawn(async move { + // On Windows it is important that this gets dropped once the child process + // exits + let controller = controller; + debug!("waiting for task: {pid:?}"); + let mut manager = ChildStateManager { + shutdown_style, + exit_tx, + shutdown_initiated: false, + }; + tokio::select! { + biased; + command = command_rx.recv() => { + manager.shutdown_initiated = true; + manager.handle_child_command(command, &mut command_rx, &mut child, controller).await; + } + status = child.wait() => { + drop(controller); + manager.handle_child_exit(status).await; + } + } -/// Child process stopped. -#[allow(dead_code)] -#[derive(Debug)] -pub struct ShutdownFailed; + debug!("child process stopped"); + }); -impl From for ShutdownFailed { - fn from(_: std::io::Error) -> Self { - ShutdownFailed + Ok(Self { + pid, + #[cfg(unix)] + target_identity, + command_channel: command_tx, + exit_channel: exit_rx, + stdin: Arc::new(Mutex::new(stdin)), + output: Arc::new(Mutex::new(output)), + label, + shutdown_style, + closing: Arc::new(AtomicBool::new(false)), + #[cfg(test)] + _pty_test_guard: pty_test_guard, + }) } -} - -struct ChildHandle { - pid: Option, - imp: ChildHandleImpl, - #[cfg(unix)] - shutdown_semantics: ShutdownSemantics, - #[cfg(unix)] - target_identity: Option, - #[cfg(unix)] - pty_controller_fd: Option, - #[cfg(windows)] - _job: Option, -} - -enum ChildHandleImpl { - Tokio(Option), - Pty(Box), -} - -#[cfg(unix)] -#[derive(Debug, Clone, Copy)] -enum GracefulInterruptTarget { - DirectChildWithProcessGroupFallback, - ProcessGroup, -} - -#[cfg(unix)] -#[derive(Debug, Clone, Copy)] -struct ShutdownSemantics { - // Who should receive the first graceful interrupt. - graceful_interrupt_target: GracefulInterruptTarget, - // Whether we should keep waiting on the process group after the direct child exits. - wait_for_process_group_after_child_exit: bool, -} -#[cfg(unix)] -impl ShutdownSemantics { - fn process_group() -> Self { - Self { - graceful_interrupt_target: GracefulInterruptTarget::ProcessGroup, - wait_for_process_group_after_child_exit: true, + /// Wait for the `Child` to exit, returning the exit code. + pub async fn wait(&mut self) -> Option { + trace!("watching exit channel of {}", self.label); + // If sending end of exit channel closed, then return last value in the channel + match self.exit_channel.changed().await { + Ok(()) => trace!("exit channel was updated"), + Err(_) => trace!("exit channel sender was dropped"), } + *self.exit_channel.borrow() } - fn direct_child_with_process_group_fallback() -> Self { - Self { - graceful_interrupt_target: GracefulInterruptTarget::DirectChildWithProcessGroupFallback, - wait_for_process_group_after_child_exit: true, - } + /// Perform a graceful shutdown of the `Child` process. + pub async fn stop(&mut self) -> Option { + self.shutdown(self.shutdown_style).await } -} - -#[cfg(unix)] -#[derive(Debug, Clone, Copy)] -struct TargetIdentity { - process_group_id: libc::pid_t, - session_id: libc::pid_t, -} -#[cfg(unix)] -fn target_identity(target_pid: libc::pid_t) -> io::Result { - let process_group_id = unsafe { libc::getpgid(target_pid) }; - if process_group_id == -1 { - return Err(io::Error::last_os_error()); + pub async fn shutdown(&mut self, shutdown_style: ShutdownStyle) -> Option { + // if this fails, it's because the channel is dropped (toctou) + // we can just ignore it + self.command_channel.shutdown(shutdown_style).await.ok(); + self.wait().await } - let session_id = unsafe { libc::getsid(target_pid) }; - if session_id == -1 { - return Err(io::Error::last_os_error()); + /// Kill the `Child` process immediately. + pub async fn kill(&mut self) -> Option { + // if this fails, it's because the channel is dropped (toctou) + // we can just ignore it + self.command_channel.kill().await.ok(); + self.wait().await } - Ok(TargetIdentity { - process_group_id, - session_id, - }) -} - -#[cfg(unix)] -fn process_group_matches_identity(target_pid: libc::pid_t, identity: TargetIdentity) -> bool { - let process_group_id = unsafe { libc::getpgid(target_pid) }; - if process_group_id != -1 { - if process_group_id != identity.process_group_id { - return false; - } - - let session_id = unsafe { libc::getsid(target_pid) }; - return session_id != -1 && session_id == identity.session_id; + pub fn pid(&self) -> Option { + self.pid } - let result = unsafe { libc::kill(-identity.process_group_id, 0) }; - result == 0 || io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) -} - -#[cfg(unix)] -fn signal_process_group(process_group_id: libc::pid_t, signal: libc::c_int) { - let _ = unsafe { libc::kill(-process_group_id, signal) }; -} - -#[cfg(unix)] -fn capture_target_identity(pid: Option) -> Option { - pid.and_then(|pid| match target_identity(pid as libc::pid_t) { - Ok(identity) => Some(identity), - Err(err) => { - debug!("failed to capture target identity for process {pid}: {err}"); - None - } - }) -} - -impl ChildHandle { - #[tracing::instrument(skip(command))] - pub fn spawn_normal(command: Command) -> io::Result { - #[cfg(windows)] - let command_for_fallback = command.clone(); - - let mut command = TokioCommand::from(command); - - // Create a new process group so we can send signals (e.g. SIGINT) to - // the child and all of its descendants via kill(-pgid, sig). - #[cfg(unix)] - command.process_group(0); + #[cfg(unix)] + fn cleanup_process_scope_after_success(&self) { + let Some(identity) = self.target_identity else { + return; + }; - #[cfg(windows)] - let job = match super::job_object::JobObject::new() { - Ok(job) => Some(job), - Err(err) => { - debug!("failed to create Windows JobObject: {err}"); - None - } + let Some(pid) = self.pid else { + return; }; - #[cfg(windows)] - if job.is_some() { - command.creation_flags( - windows_sys::Win32::System::Threading::CREATE_SUSPENDED - | windows_sys::Win32::System::Threading::CREATE_BREAKAWAY_FROM_JOB, + if process_group_matches_identity(pid as libc::pid_t, identity) { + debug!( + "cleaning up remaining process group after successful task: {}", + identity.process_group_id ); + signal_process_group(identity.process_group_id, libc::SIGKILL); } - - #[cfg(not(windows))] - let mut child = command.spawn()?; - - #[cfg(windows)] - let mut child = match command.spawn() { - Ok(child) => child, - Err(err) if job.is_some() => { - debug!("failed to spawn child with job breakaway: {err}"); - let mut fallback_command = TokioCommand::from(command_for_fallback); - fallback_command - .creation_flags(windows_sys::Win32::System::Threading::CREATE_SUSPENDED); - fallback_command.spawn()? - } - Err(err) => return Err(err), - }; - let pid = child.id(); - - #[cfg(unix)] - let target_identity = capture_target_identity(pid); - - #[cfg(windows)] - let job = job.and_then(|job| match child.raw_handle() { - Some(handle) => match job.assign_suspended_process(handle) { - Ok(true) => Some(job), - Ok(false) => None, - Err(err) => { - debug!("failed to resume suspended process after job assignment: {err}"); - child.start_kill().ok(); - None - } - }, - None => { - debug!("failed to get child process handle for job assignment"); - child.start_kill().ok(); - None - } - }); - - let stdin = child.stdin.take().map(ChildInput::Std); - let stdout = child - .stdout - .take() - .ok_or_else(|| io::Error::other("child process must be started with piped stdout"))?; - let stderr = child - .stderr - .take() - .ok_or_else(|| io::Error::other("child process must be started with piped stderr"))?; - - Ok(SpawnResult { - handle: Self { - pid, - imp: ChildHandleImpl::Tokio(Some(child)), - #[cfg(unix)] - shutdown_semantics: ShutdownSemantics::process_group(), - #[cfg(unix)] - target_identity, - #[cfg(unix)] - pty_controller_fd: None, - #[cfg(windows)] - _job: job, - }, - io: ChildIO { - stdin, - output: Some(ChildOutput::Std { stdout, stderr }), - }, - controller: None, - }) } - #[tracing::instrument(skip(command))] - pub fn spawn_pty(command: Command, size: PtySize) -> io::Result { - let keep_stdin_open = command.will_open_stdin(); - - let command = portable_pty::CommandBuilder::from(command); - let pty_system = native_pty_system(); - let size = portable_pty::PtySize { - rows: size.rows, - cols: size.cols, - pixel_width: 0, - pixel_height: 0, + #[cfg(windows)] + fn cleanup_process_scope_after_success(&self) { + let Some(pid) = self.pid else { + return; }; - let pair = pty_system - .openpty(size) - .map_err(|err| match err.downcast() { - Ok(err) => err, - Err(err) => io::Error::other(err), - })?; - let controller = pair.master; - let receiver = pair.slave; - - #[cfg(unix)] - { - use nix::sys::termios; - if let Some((file_desc, mut termios)) = controller - .as_raw_fd() - .and_then(|fd| Some(fd).zip(termios::tcgetattr(fd).ok())) - { - // We unset ECHOCTL to disable rendering of the closing of stdin - // as ^D - termios.local_flags &= !nix::sys::termios::LocalFlags::ECHOCTL; - if let Err(e) = nix::sys::termios::tcsetattr( - file_desc, - nix::sys::termios::SetArg::TCSANOW, - &termios, - ) { - debug!("unable to unset ECHOCTL: {e}"); - } - } + if let Err(err) = super::job_object::terminate_descendant_processes(pid) { + debug!("failed to clean up descendants after successful task {pid}: {err}"); } + } - let child = receiver - .spawn_command(command) - .map_err(|err| match err.downcast() { - Ok(err) => err, - Err(err) => io::Error::other(err), - })?; - - let pid = child.process_id(); - - #[cfg(unix)] - let target_identity = capture_target_identity(pid); - - #[cfg(windows)] - let job = pid.and_then(|pid| { - super::job_object::JobObject::new() - .and_then(|job| job.assign_pid(pid).map(|_| job)) - .map_err(|e| debug!("failed to set up job object for PTY process {pid}: {e}")) - .ok() - }); - - #[cfg(unix)] - let pty_controller_fd = controller.as_raw_fd(); - - let mut stdin = controller.take_writer().ok(); - let output = controller.try_clone_reader().ok().map(ChildOutput::Pty); - - // portable-pty 0.9.0 creates ConPTY with PSEUDOCONSOLE_INHERIT_CURSOR, - // which sends a Device Status Report (DSR) cursor position request - // (\x1b[6n) on the output pipe during initialization. ConPTY blocks - // until the host responds with a Cursor Position Report on stdin. - // Without this response the PTY hangs indefinitely. - // See https://github.com/vercel/turborepo/issues/11808 - #[cfg(windows)] - if let Some(ref mut writer) = stdin { - // Respond with cursor at position (1,1). The actual position - // doesn't matter — ConPTY just needs a valid CPR to unblock. - if let Err(e) = writer.write_all(b"\x1b[1;1R") { - debug!("failed to write ConPTY cursor position response: {e}"); - } - } + #[cfg(not(any(unix, windows)))] + fn cleanup_process_scope_after_success(&self) {} - // If we don't want to keep stdin open we take it here and it is immediately - // dropped resulting in a EOF being sent to the child process. - if !keep_stdin_open { - stdin.take(); + fn cleanup_if_successful(&self, status: Option) { + if status == Some(ChildExit::Finished(Some(0))) { + self.cleanup_process_scope_after_success(); } - - Ok(SpawnResult { - handle: Self { - pid, - imp: ChildHandleImpl::Pty(child), - #[cfg(unix)] - shutdown_semantics: ShutdownSemantics::direct_child_with_process_group_fallback(), - #[cfg(unix)] - target_identity, - #[cfg(unix)] - pty_controller_fd, - #[cfg(windows)] - _job: job, - }, - io: ChildIO { - stdin: stdin.map(ChildInput::Pty), - output, - }, - controller: Some(controller), - }) } - pub fn pid(&self) -> Option { - self.pid + pub(crate) fn has_exited(&self) -> bool { + self.exit_channel.borrow().is_some() } - #[cfg(unix)] - fn process_group_id(&self) -> Option { - self.target_identity - .map(|identity| identity.process_group_id) - .or(self.pid.map(|pid| pid as libc::pid_t)) + pub fn label(&self) -> &str { + &self.label } - #[cfg(unix)] - fn graceful_process_group_id(&self) -> Option { - self.pty_controller_fd - .and_then(|fd| match unsafe { libc::tcgetpgrp(fd) } { - process_group_id if process_group_id > 0 => Some(process_group_id), - _ => None, - }) - .or_else(|| self.process_group_id()) + /// Mark this child as being stopped as part of a ProcessManager shutdown + pub fn set_closing(&self) { + self.closing.store(true, Ordering::Release); } - #[cfg(unix)] - fn send_signal_to_process_group(&self, pid: libc::pid_t, signal: libc::c_int) { - let Some(process_group_id) = self.graceful_process_group_id() else { - debug!("missing process group id for child {pid}"); - return; - }; - - debug!("sending signal {signal} to process group -{process_group_id}"); - signal_process_group(process_group_id, signal); + /// Check if this child was stopped as part of a ProcessManager shutdown + pub fn is_closing(&self) -> bool { + self.closing.load(Ordering::Acquire) } +} - #[cfg(unix)] - fn send_graceful_interrupt(&self, pid: libc::pid_t) -> bool { - match self.shutdown_semantics.graceful_interrupt_target { - GracefulInterruptTarget::DirectChildWithProcessGroupFallback => { - debug!("sending SIGINT to child {pid}"); - if unsafe { libc::kill(pid, libc::SIGINT) } == -1 { - debug!("failed to send SIGINT to {pid}"); - } - false - } - GracefulInterruptTarget::ProcessGroup => { - self.send_signal_to_process_group(pid, libc::SIGINT); - true - } - } - } - - #[cfg(unix)] - fn send_fallback_graceful_interrupt(&self, pid: libc::pid_t) { - self.send_signal_to_process_group(pid, libc::SIGINT); - } - - #[cfg(unix)] - fn should_wait_for_process_group_after_child_exit(&self) -> bool { - self.shutdown_semantics - .wait_for_process_group_after_child_exit - } - - #[cfg(unix)] - fn has_running_process_group(&self, pid: libc::pid_t) -> bool { - if let Some(identity) = self.target_identity { - return process_group_matches_identity(pid, identity); - } - - let process_group_id = self.process_group_id().unwrap_or(pid); - - let result = unsafe { libc::kill(-process_group_id, 0) }; - result == 0 || io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) - } - - #[cfg(unix)] - fn kill_process_group(&self, pid: libc::pid_t) { - let process_group_id = self.process_group_id().unwrap_or(pid); - - debug!("killing process group {}", process_group_id); - signal_process_group(process_group_id, libc::SIGKILL); - } - - #[cfg(unix)] - async fn wait_for_process_group_exit( - &mut self, - pid: libc::pid_t, - deadline: Option, - command_rx: &mut mpsc::Receiver, - command_rx_open: &mut bool, - ) -> ChildExit { - if !self.should_wait_for_process_group_after_child_exit() { - return ChildExit::Interrupted; - } - - while self.has_running_process_group(pid) { - match deadline { - Some(deadline) => { - tokio::select! { - command = command_rx.recv(), if *command_rx_open => { - match command { - Some(ChildCommand::Kill) => { - debug!("graceful shutdown interrupted, killing process group"); - self.kill_process_group(pid); - return ChildExit::Killed; - } - Some(ChildCommand::Shutdown(_)) => {} - None => *command_rx_open = false, - } - } - _ = tokio::time::sleep_until(deadline) => { - debug!("graceful shutdown timed out, killing process group"); - self.kill_process_group(pid); - return ChildExit::Killed; - } - _ = tokio::time::sleep(PROCESS_TREE_DRAIN_POLL_INTERVAL) => {} - } - } - None => { - tokio::select! { - command = command_rx.recv(), if *command_rx_open => { - match command { - Some(ChildCommand::Kill) => { - debug!("graceful shutdown interrupted, killing process group"); - self.kill_process_group(pid); - return ChildExit::Killed; - } - Some(ChildCommand::Shutdown(_)) => {} - None => *command_rx_open = false, - } - } - _ = tokio::time::sleep(PROCESS_TREE_DRAIN_POLL_INTERVAL) => {} - } - } - } - } - - ChildExit::Interrupted - } - - #[cfg(windows)] - fn has_active_windows_job(&self) -> bool { - self._job - .as_ref() - .is_some_and(|job| match job.active_processes() { - Ok(active_processes) => active_processes > 0, - Err(err) => { - debug!("failed to query job object: {err}"); - false - } - }) - } - - #[cfg(windows)] - fn has_running_windows_descendants(&self) -> bool { - match self.pid { - Some(pid) => match super::job_object::has_descendant_processes(pid) { - Ok(has_descendants) => has_descendants, - Err(err) => { - debug!("failed to query descendant processes: {err}"); - false - } - }, - None => false, - } - } - - #[cfg(windows)] - fn terminate_windows_process_tree(&self) { - if let Some(job) = &self._job - && let Err(err) = job.terminate() - { - debug!("failed to terminate job object: {err}"); - } - - if let Some(pid) = self.pid - && let Err(err) = super::job_object::terminate_descendant_processes(pid) - { - debug!("failed to terminate descendant process tree: {err}"); - } - } - - #[cfg(windows)] - async fn wait_for_job_exit( - &mut self, - deadline: Option, - command_rx: &mut mpsc::Receiver, - command_rx_open: &mut bool, - ) -> ChildExit { - // PID snapshots are only a fallback for runners where Job Object - // assignment fails. After the parent exits they can match unrelated - // reused PIDs, so never let that path wait forever. - let descendant_drain_deadline = self - ._job - .is_none() - .then(|| tokio::time::Instant::now() + WINDOWS_DESCENDANT_DRAIN_TIMEOUT); - - loop { - let has_active_job = self.has_active_windows_job(); - let has_descendants = self._job.is_none() && self.has_running_windows_descendants(); - - if !has_active_job && !has_descendants { - break; - } - - tokio::select! { - command = command_rx.recv(), if *command_rx_open => { - match command { - Some(ChildCommand::Kill) => { - debug!("process tree drain interrupted, terminating job object"); - self.terminate_windows_process_tree(); - return ChildExit::Killed; - } - Some(ChildCommand::Shutdown(_)) => {} - None => *command_rx_open = false, - } - } - _ = async { - if let Some(deadline) = deadline { - tokio::time::sleep_until(deadline).await; - } - }, if deadline.is_some() => { - debug!("graceful shutdown timed out, terminating Windows process tree"); - self.terminate_windows_process_tree(); - return ChildExit::Killed; - } - _ = async { - if let Some(deadline) = descendant_drain_deadline { - tokio::time::sleep_until(deadline).await; - } - }, if has_descendants && descendant_drain_deadline.is_some() => { - debug!("timed out waiting for Windows descendant process tree after direct child exit"); - break; - } - _ = tokio::time::sleep(PROCESS_TREE_DRAIN_POLL_INTERVAL) => {} - } - } - - ChildExit::Interrupted - } - - /// Perform a `wait` syscall on the child until it exits - pub async fn wait(&mut self) -> io::Result> { - match &mut self.imp { - ChildHandleImpl::Tokio(child) => { - let result = match child { - Some(child) => child.wait().await.map(|status| status.code()), - None => Ok(None), - }; - - #[cfg(windows)] - if result.is_ok() { - // Drop the process handle before querying the Job Object so - // the exited direct child is not counted during tree drain. - child.take(); - } - - result - } - ChildHandleImpl::Pty(child) => { - // TODO: we currently poll the child to see if it has finished yet which is less - // than ideal - loop { - match child.try_wait() { - Ok(Some(status)) => { - // portable_pty maps the status of being killed by a signal to a 1 exit - // code. The only way to tell if the task - // exited normally with exit code 1 or got killed by a signal is to - // display it as the signal will be included - // in the message. - let exit_code = if status.exit_code() == 1 - && status.to_string().contains("Terminated by") - { - None - } else { - // This is safe as the portable_pty::ExitStatus's exit code is just - // converted from a i32 to an u32 before we get it - Some(status.exit_code() as i32) - }; - return Ok(exit_code); - } - Ok(None) => { - // child hasn't finished, we sleep for a short time - tokio::time::sleep(CHILD_POLL_INTERVAL).await; - } - Err(err) => return Err(err), - } - } - } - } - } - - pub async fn kill(&mut self) -> io::Result<()> { - #[cfg(unix)] - if let Some(process_group_id) = self.process_group_id() { - signal_process_group(process_group_id, libc::SIGKILL); - } - - match &mut self.imp { - ChildHandleImpl::Tokio(Some(child)) => child.kill().await, - ChildHandleImpl::Tokio(None) => Ok(()), - ChildHandleImpl::Pty(child) => { - let mut killer = child.clone_killer(); - tokio::task::spawn_blocking(move || killer.kill()) - .await - .map_err(|err| io::Error::other(format!("pty kill task failed: {err}")))? - } - } - } -} - -struct SpawnResult { - handle: ChildHandle, - io: ChildIO, - controller: Option>, -} - -struct ChildIO { - stdin: Option, - output: Option, -} - -enum ChildInput { - Std(tokio::process::ChildStdin), - Pty(Box), -} - -#[derive(Debug)] -pub struct ChildStdinGuard { - _stdin: ChildInput, -} - -pub enum ChildStdin { - Writable(Box), - Guard(ChildStdinGuard), -} - -impl fmt::Debug for ChildStdin { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Writable(_) => f.debug_tuple("Writable").finish(), - Self::Guard(guard) => f.debug_tuple("Guard").field(guard).finish(), - } - } -} - -enum ChildOutput { - Std { - stdout: tokio::process::ChildStdout, - stderr: tokio::process::ChildStderr, - }, - Pty(Box), -} - -impl fmt::Debug for ChildInput { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Std(arg0) => f.debug_tuple("Std").field(arg0).finish(), - Self::Pty(_) => f.debug_tuple("Pty").finish(), - } - } -} - -impl fmt::Debug for ChildOutput { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Std { stdout, stderr } => f - .debug_struct("Std") - .field("stdout", stdout) - .field("stderr", stderr) - .finish(), - Self::Pty(_) => f.debug_tuple("Pty").finish(), - } - } -} - -impl ShutdownStyle { - /// Process the shutdown style for the given child process. - /// - /// If an exit channel is provided, the exit code will be sent to the - /// channel when the child process exits. - async fn process( - &self, - child: &mut ChildHandle, - command_rx: &mut mpsc::Receiver, - ) -> ChildExit { - match self { - #[allow(unused)] - ShutdownStyle::Graceful(timeout) => { - // try ro run the command for the given timeout - #[cfg(unix)] - { - let Some(pid) = child.pid() else { - return ChildExit::Interrupted; - }; - - let pid = pid as libc::pid_t; - let mut process_group_interrupt_sent = child.send_graceful_interrupt(pid); - let process_group_interrupt_deadline = - tokio::time::Instant::now() + PTY_PROCESS_GROUP_SIGINT_DELAY; - debug!("waiting for child {}", pid); - - let deadline = timeout.map(|timeout| tokio::time::Instant::now() + timeout); - let mut command_rx_open = true; - - let exit = loop { - match deadline { - Some(deadline) => { - tokio::select! { - result = child.wait() => { - break match result { - Ok(_exit_code) => ChildExit::Interrupted, - Err(_) => ChildExit::Failed, - }; - } - _ = tokio::time::sleep_until(process_group_interrupt_deadline), if !process_group_interrupt_sent => { - child.send_fallback_graceful_interrupt(pid); - process_group_interrupt_sent = true; - } - command = command_rx.recv(), if command_rx_open => { - match command { - Some(ChildCommand::Kill) => { - debug!("graceful shutdown interrupted, killing child"); - break match child.kill().await { - Ok(_) => ChildExit::Killed, - Err(_) => ChildExit::Failed, - }; - } - Some(ChildCommand::Shutdown(_)) => {} - None => command_rx_open = false, - } - } - _ = tokio::time::sleep_until(deadline) => { - debug!("graceful shutdown timed out, killing child"); - break match child.kill().await { - Ok(_) => ChildExit::Killed, - Err(_) => ChildExit::Failed, - }; - } - } - } - None => { - tokio::select! { - result = child.wait() => { - break match result { - Ok(_exit_code) => ChildExit::Interrupted, - Err(_) => ChildExit::Failed, - }; - } - _ = tokio::time::sleep_until(process_group_interrupt_deadline), if !process_group_interrupt_sent => { - child.send_fallback_graceful_interrupt(pid); - process_group_interrupt_sent = true; - } - command = command_rx.recv(), if command_rx_open => { - match command { - Some(ChildCommand::Kill) => { - debug!("graceful shutdown interrupted, killing child"); - break match child.kill().await { - Ok(_) => ChildExit::Killed, - Err(_) => ChildExit::Failed, - }; - } - Some(ChildCommand::Shutdown(_)) => {} - None => command_rx_open = false, - } - } - } - } - } - }; - - if exit == ChildExit::Interrupted { - child - .wait_for_process_group_exit( - pid, - deadline, - command_rx, - &mut command_rx_open, - ) - .await - } else { - exit - } - } - - #[cfg(windows)] - { - // Windows consoles deliver Ctrl+C to attached child processes. - // Turbo can't send a targeted signal, so graceful shutdown - // waits for that external event when no timeout is provided. - let deadline = timeout.map(|timeout| tokio::time::Instant::now() + timeout); - let mut command_rx_open = true; - - let exit = loop { - match deadline { - Some(deadline) => { - tokio::select! { - result = child.wait() => { - break match result { - Ok(_exit_code) => ChildExit::Interrupted, - Err(_) => ChildExit::Failed, - }; - } - command = command_rx.recv(), if command_rx_open => { - match command { - Some(ChildCommand::Kill) => { - debug!("graceful shutdown interrupted, killing child"); - break match child.kill().await { - Ok(_) => ChildExit::Killed, - Err(_) => ChildExit::Failed, - }; - } - Some(ChildCommand::Shutdown(_)) => {} - None => command_rx_open = false, - } - } - _ = tokio::time::sleep_until(deadline) => { - debug!("graceful shutdown timed out, killing child"); - break match child.kill().await { - Ok(_) => ChildExit::Killed, - Err(_) => ChildExit::Failed, - }; - } - } - } - None => { - tokio::select! { - result = child.wait() => { - break match result { - Ok(_exit_code) => ChildExit::Interrupted, - Err(_) => ChildExit::Failed, - }; - } - command = command_rx.recv(), if command_rx_open => { - match command { - Some(ChildCommand::Kill) => { - debug!("graceful shutdown interrupted, killing child"); - break match child.kill().await { - Ok(_) => ChildExit::Killed, - Err(_) => ChildExit::Failed, - }; - } - Some(ChildCommand::Shutdown(_)) => {} - None => command_rx_open = false, - } - } - } - } - } - }; - - if exit == ChildExit::Interrupted { - child - .wait_for_job_exit(deadline, command_rx, &mut command_rx_open) - .await - } else { - exit - } - } - } - ShutdownStyle::Kill => match child.kill().await { - Ok(_) => ChildExit::Killed, - Err(_) => ChildExit::Failed, - }, - } - } -} - -/// The structure that holds logic regarding interacting with the underlying -/// child process -#[derive(Debug)] -struct ChildStateManager { - shutdown_style: ShutdownStyle, - exit_tx: watch::Sender>, - shutdown_initiated: bool, -} - -/// A child process that can be interacted with asynchronously. -/// -/// This is a wrapper around the `tokio::process::Child` struct, which provides -/// a cross platform interface for spawning and managing child processes. -#[derive(Clone, Debug)] -pub struct Child { - pid: Option, - #[cfg(unix)] - target_identity: Option, - command_channel: ChildCommandChannel, - exit_channel: watch::Receiver>, - stdin: Arc>>, - output: Arc>>, - label: String, - shutdown_style: ShutdownStyle, - /// Flag indicating this child is being stopped as part of a shutdown of the - /// ProcessManager, rather than individually stopped. - closing: Arc, - #[cfg(test)] - _pty_test_guard: Option>, -} - -#[derive(Clone, Debug)] -pub struct ChildCommandChannel(mpsc::Sender); - -impl ChildCommandChannel { - pub fn new() -> (Self, mpsc::Receiver) { - let (tx, rx) = mpsc::channel(1); - (ChildCommandChannel(tx), rx) - } - - pub async fn shutdown( - &self, - shutdown_style: ShutdownStyle, - ) -> Result<(), mpsc::error::SendError> { - self.0.send(ChildCommand::Shutdown(shutdown_style)).await - } - - pub async fn kill(&self) -> Result<(), mpsc::error::SendError> { - self.0.send(ChildCommand::Kill).await - } -} - -pub enum ChildCommand { - Shutdown(ShutdownStyle), - Kill, -} - -impl Child { - /// Start a child process, returning a handle that can be used to interact - /// with it. The command will be started immediately. - #[tracing::instrument(skip(command), fields(command = command.label()))] - pub fn spawn( - command: Command, - shutdown_style: ShutdownStyle, - pty_size: Option, - ) -> io::Result { - let label = command.label(); - #[cfg(test)] - let pty_test_guard = pty_size.map(|_| Arc::new(PtyTestGuard::acquire())); - let SpawnResult { - handle: mut child, - io: ChildIO { stdin, output }, - controller, - } = if let Some(size) = pty_size { - ChildHandle::spawn_pty(command, size) - } else { - ChildHandle::spawn_normal(command) - }?; - - let pid = child.pid(); - #[cfg(unix)] - let target_identity = child.target_identity; - - let (command_tx, mut command_rx) = ChildCommandChannel::new(); - - // we use a watch channel to communicate the exit code back to the - // caller. we are interested in three cases: - // - the child process exits - // - the child process is killed (and doesn't have an exit code) - // - the child process fails somehow (some syscall fails) - let (exit_tx, exit_rx) = watch::channel(None); - - let _task = tokio::spawn(async move { - // On Windows it is important that this gets dropped once the child process - // exits - let controller = controller; - debug!("waiting for task: {pid:?}"); - let mut manager = ChildStateManager { - shutdown_style, - exit_tx, - shutdown_initiated: false, - }; - tokio::select! { - biased; - command = command_rx.recv() => { - manager.shutdown_initiated = true; - manager.handle_child_command(command, &mut command_rx, &mut child, controller).await; - } - status = child.wait() => { - drop(controller); - manager.handle_child_exit(status).await; - } - } - - debug!("child process stopped"); - }); - - Ok(Self { - pid, - #[cfg(unix)] - target_identity, - command_channel: command_tx, - exit_channel: exit_rx, - stdin: Arc::new(Mutex::new(stdin)), - output: Arc::new(Mutex::new(output)), - label, - shutdown_style, - closing: Arc::new(AtomicBool::new(false)), - #[cfg(test)] - _pty_test_guard: pty_test_guard, - }) - } - - /// Wait for the `Child` to exit, returning the exit code. - pub async fn wait(&mut self) -> Option { - trace!("watching exit channel of {}", self.label); - // If sending end of exit channel closed, then return last value in the channel - match self.exit_channel.changed().await { - Ok(()) => trace!("exit channel was updated"), - Err(_) => trace!("exit channel sender was dropped"), - } - *self.exit_channel.borrow() - } - - /// Perform a graceful shutdown of the `Child` process. - pub async fn stop(&mut self) -> Option { - self.shutdown(self.shutdown_style).await - } - - pub async fn shutdown(&mut self, shutdown_style: ShutdownStyle) -> Option { - // if this fails, it's because the channel is dropped (toctou) - // we can just ignore it - self.command_channel.shutdown(shutdown_style).await.ok(); - self.wait().await - } - - /// Kill the `Child` process immediately. - pub async fn kill(&mut self) -> Option { - // if this fails, it's because the channel is dropped (toctou) - // we can just ignore it - self.command_channel.kill().await.ok(); - self.wait().await - } - - pub fn pid(&self) -> Option { - self.pid - } - - #[cfg(unix)] - fn cleanup_process_scope_after_success(&self) { - let Some(identity) = self.target_identity else { - return; - }; - - let Some(pid) = self.pid else { - return; - }; - - if process_group_matches_identity(pid as libc::pid_t, identity) { - debug!( - "cleaning up remaining process group after successful task: {}", - identity.process_group_id - ); - signal_process_group(identity.process_group_id, libc::SIGKILL); - } - } - - #[cfg(windows)] - fn cleanup_process_scope_after_success(&self) { - let Some(pid) = self.pid else { - return; - }; - - if let Err(err) = super::job_object::terminate_descendant_processes(pid) { - debug!("failed to clean up descendants after successful task {pid}: {err}"); - } - } - - #[cfg(not(any(unix, windows)))] - fn cleanup_process_scope_after_success(&self) {} - - fn cleanup_if_successful(&self, status: Option) { - if status == Some(ChildExit::Finished(Some(0))) { - self.cleanup_process_scope_after_success(); - } - } - - pub(crate) fn has_exited(&self) -> bool { - self.exit_channel.borrow().is_some() - } - - fn stdin_inner(&mut self) -> Option { - self.stdin - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - } - - fn outputs(&self) -> Option { - self.output - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - } - - pub fn stdin(&mut self) -> Option> { - let stdin = self.stdin_inner()?; - match stdin { - ChildInput::Std(_) => None, - ChildInput::Pty(stdin) => Some(stdin), - } - } - - pub fn take_stdin(&mut self) -> Option { - let stdin = self.stdin_inner()?; - match stdin { - ChildInput::Std(stdin) => Some(ChildStdin::Guard(ChildStdinGuard { - _stdin: ChildInput::Std(stdin), - })), - ChildInput::Pty(stdin) => Some(ChildStdin::Writable(stdin)), - } - } - - /// Wait for the `Child` to exit and pipe any stdout and stderr to the - /// provided writer. - #[tracing::instrument(skip_all)] - pub async fn wait_with_piped_outputs( - &mut self, - stdout_pipe: W, - ) -> Result, std::io::Error> { - match self.outputs() { - Some(ChildOutput::Std { stdout, stderr }) => { - self.wait_with_piped_async_outputs( - stdout_pipe, - Some(BufReader::new(stdout)), - Some(BufReader::new(stderr)), - ) - .await - } - Some(ChildOutput::Pty(output)) => { - // On Unix, drop stdin before reading so the master PTY writer - // sends EOT and releases its fd, allowing the reader to reach - // EOF once the controller is dropped after the child exits. - // - // On Windows, do NOT drop stdin here: ConPTY treats a closed - // stdin pipe as the session ending and immediately terminates - // the child process. - if !cfg!(windows) { - drop(self.stdin_inner()); - } - self.wait_with_piped_sync_output(stdout_pipe, std::io::BufReader::new(output)) - .await - } - None => Ok(self.wait().await), - } - } - - #[tracing::instrument(skip_all)] - async fn wait_with_piped_sync_output( - &mut self, - mut stdout_pipe: impl Write, - mut stdout_lines: R, - ) -> Result, std::io::Error> { - // TODO: in order to not impose that a stdout_pipe is Send we send the bytes - // across a channel - let (byte_tx, mut byte_rx) = mpsc::channel(48); - tokio::task::spawn_blocking(move || { - let mut buffer = [0; 1024]; - let mut last_byte = None; - loop { - match stdout_lines.read(&mut buffer) { - Ok(0) => { - if !matches!(last_byte, Some(b'\n')) { - // Ignore if this fails as we already are shutting down - byte_tx.blocking_send(vec![b'\n']).ok(); - } - break; - } - Ok(n) => { - let mut bytes = Vec::with_capacity(n); - bytes.extend_from_slice(&buffer[..n]); - last_byte = bytes.last().copied(); - if byte_tx.blocking_send(bytes).is_err() { - // A dropped receiver indicates that there was an issue writing to the - // pipe. We can stop reading output. - break; - } - } - Err(e) => return Err(e), - } - } - Ok(()) - }); - - let writer_fut = async { - let mut result = Ok(()); - while let Some(bytes) = byte_rx.recv().await { - if let Err(err) = stdout_pipe.write_all(&bytes) { - result = Err(err); - break; - } - } - result - }; - - let (status, write_result) = tokio::join!(self.wait(), writer_fut); - write_result?; - self.cleanup_if_successful(status); - - Ok(status) - } - - #[tracing::instrument(skip_all)] - async fn wait_with_piped_async_outputs( - &mut self, - mut stdout_pipe: impl Write, - mut stdout_lines: Option, - mut stderr_lines: Option, - ) -> Result, std::io::Error> { - async fn next_line( - stream: &mut Option, - buffer: &mut Vec, - ) -> Option> { - match stream { - Some(stream) => match stream.read_until(b'\n', buffer).await { - Ok(0) => { - trace!("reached EOF"); - None - } - Ok(_) => Some(Ok(())), - Err(e) => Some(Err(e)), - }, - None => None, - } - } - - let mut stdout_buffer = Vec::new(); - let mut stderr_buffer = Vec::new(); - - let mut is_exited = false; - let mut exit_status = None; - let mut draining_after_exit = false; - let mut drain_deadline = tokio::time::Instant::now() + POST_EXIT_OUTPUT_DRAIN_TIMEOUT; - loop { - tokio::select! { - Some(result) = next_line(&mut stdout_lines, &mut stdout_buffer) => { - trace!("processing stdout line"); - result?; - add_trailing_newline(&mut stdout_buffer); - stdout_pipe.write_all(&stdout_buffer)?; - stdout_buffer.clear(); - } - Some(result) = next_line(&mut stderr_lines, &mut stderr_buffer) => { - trace!("processing stderr line"); - result?; - add_trailing_newline(&mut stderr_buffer); - stdout_pipe.write_all(&stderr_buffer)?; - stderr_buffer.clear(); - } - status = self.wait(), if !is_exited => { - trace!("child process exited: {}", self.label()); - is_exited = true; - exit_status = status; - // We don't abort in the cases of a zero exit code as we could be - // caching this task and should read all the logs it produces. - if status == Some(ChildExit::Finished(Some(0))) { - continue; - } - - if self.is_closing() { - // During Turbo-initiated shutdown, give the pipe readers a - // short grace window to pull the child's final log lines. - draining_after_exit = true; - drain_deadline = tokio::time::Instant::now() + POST_EXIT_OUTPUT_DRAIN_TIMEOUT; - } else { - debug!("child process failed, skipping reading stdout/stderr"); - return Ok(status); - } - } - _ = tokio::time::sleep_until(drain_deadline), if draining_after_exit => { - trace!("post-exit output drain timed out"); - if !stdout_buffer.is_empty() { - add_trailing_newline(&mut stdout_buffer); - stdout_pipe.write_all(&stdout_buffer)?; - stdout_buffer.clear(); - } - if !stderr_buffer.is_empty() { - add_trailing_newline(&mut stderr_buffer); - stdout_pipe.write_all(&stderr_buffer)?; - stderr_buffer.clear(); - } - return Ok(exit_status); - } - else => { - trace!("flushing child stdout/stderr buffers"); - // In the case that both futures read a complete line - // the future not chosen in the select will return None if it's at EOF - // as the number of bytes read will be 0. - // We check and flush the buffers to avoid missing the last line of output. - if !stdout_buffer.is_empty() { - add_trailing_newline(&mut stdout_buffer); - stdout_pipe.write_all(&stdout_buffer)?; - stdout_buffer.clear(); - } - if !stderr_buffer.is_empty() { - add_trailing_newline(&mut stderr_buffer); - stdout_pipe.write_all(&stderr_buffer)?; - stderr_buffer.clear(); - } - break; - } - } - } - debug_assert!(stdout_buffer.is_empty(), "buffer should be empty"); - debug_assert!(stderr_buffer.is_empty(), "buffer should be empty"); - - let status = exit_status.or(self.wait().await); - self.cleanup_if_successful(status); - Ok(status) - } - - pub fn label(&self) -> &str { - &self.label - } - - /// Mark this child as being stopped as part of a ProcessManager shutdown - pub fn set_closing(&self) { - self.closing.store(true, Ordering::Release); - } - - /// Check if this child was stopped as part of a ProcessManager shutdown - pub fn is_closing(&self) -> bool { - self.closing.load(Ordering::Acquire) - } -} - -// Adds a trailing newline if necessary to the buffer -fn add_trailing_newline(buffer: &mut Vec) { - // If the line doesn't end with a newline, that indicates we hit a EOF. - // We add a newline so output from other tasks doesn't get written to the same - // line. - if buffer.last() != Some(&b'\n') { - buffer.push(b'\n'); - } -} - -impl ChildStateManager { - async fn handle_child_command( - &self, - command: Option, - command_rx: &mut mpsc::Receiver, - child: &mut ChildHandle, - controller: Option>, - ) { - let exit = match command.unwrap_or(ChildCommand::Shutdown(self.shutdown_style)) { - ChildCommand::Shutdown(shutdown_style) => { - debug!("stopping child process"); - shutdown_style.process(child, command_rx).await - } - ChildCommand::Kill => { - debug!("killing child process"); - ShutdownStyle::Kill.process(child, command_rx).await - } - }; - // ignore the send error, failure means the channel is dropped - trace!("sending child exit after shutdown"); - self.exit_tx.send(Some(exit)).ok(); - drop(controller); - } - - async fn handle_child_exit(&self, status: io::Result>) { - // If a shutdown was initiated we defer to the exit returned by - // `ShutdownStyle::process` as that will have information if the child - // responded to a SIGINT or a SIGKILL. The `wait` response this function - // gets in that scenario would make it appear that the child was killed by an - // external process. - if self.shutdown_initiated { - return; - } - - debug!("child process exited normally"); - // the child process exited - let child_exit = match status { - Ok(Some(c)) => ChildExit::Finished(Some(c)), - // if we hit this case, it means that the child process was killed - // by someone else, and we should report that it was killed - Ok(None) => ChildExit::KilledExternal, - Err(_e) => ChildExit::Failed, - }; - - // ignore the send error, the channel is dropped anyways - trace!("sending child exit"); - self.exit_tx.send(Some(child_exit)).ok(); - } -} - -#[cfg(test)] +#[cfg(test)] impl Child { // Helper method for checking if child is running fn is_running(&self) -> bool { !self.command_channel.0.is_closed() } } - -#[cfg(test)] -mod test { - use std::{ - assert_matches, fs, io, - sync::{Arc, Mutex}, - time::Duration, - }; - - use futures::{StreamExt, stream::FuturesUnordered}; - use test_case::test_case; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - sync::oneshot, - }; - use tracing_test::traced_test; - use turbopath::AbsoluteSystemPathBuf; - - use super::{Child, ChildInput, ChildOutput, ChildStdin, Command}; - use crate::{ - PtySize, - child::{ChildExit, ShutdownStyle}, - }; - - const STARTUP_DELAY: Duration = Duration::from_millis(500); - // We skip testing PTY usage on Windows - const TEST_PTY: bool = !cfg!(windows); - - struct ObservedOutput { - buffer: Arc>>, - ready_tx: Option>, - } - - impl ObservedOutput { - fn new() -> (Self, Arc>>, oneshot::Receiver<()>) { - let buffer = Arc::new(Mutex::new(Vec::new())); - let (ready_tx, ready_rx) = oneshot::channel(); - ( - Self { - buffer: buffer.clone(), - ready_tx: Some(ready_tx), - }, - buffer, - ready_rx, - ) - } - } - - impl io::Write for ObservedOutput { - fn write(&mut self, buf: &[u8]) -> io::Result { - let saw_ready = { - let mut buffer = self.buffer.lock().unwrap(); - buffer.extend_from_slice(buf); - String::from_utf8_lossy(&buffer).contains("ready") - }; - - if saw_ready && let Some(ready_tx) = self.ready_tx.take() { - ready_tx.send(()).ok(); - } - - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - const EOT: char = '\u{4}'; - - fn find_script_dir() -> AbsoluteSystemPathBuf { - let cwd = AbsoluteSystemPathBuf::cwd().unwrap(); - let mut root = cwd; - while !root.join_component(".git").exists() { - root = root.parent().unwrap().to_owned(); - } - root.join_components(&["crates", "turborepo-process", "test", "scripts"]) - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_pid(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - assert_matches!(child.pid(), Some(_)); - child.stop().await; - - let exit = child.wait().await; - assert_matches!(exit, Some(ChildExit::Killed)); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tracing_test::traced_test] - #[tokio::test] - async fn test_wait(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - let exit1 = child.wait().await; - let exit2 = child.wait().await; - assert_matches!(exit1, Some(ChildExit::Finished(Some(0)))); - assert_matches!(exit2, Some(ChildExit::Finished(Some(0)))); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_spawn(use_pty: bool) { - let cmd = { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd - }; - - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - assert!(child.is_running()); - - let code = tokio::time::timeout(Duration::from_secs(10), child.wait()) - .await - .expect("child wait should not hang after process exit"); - assert_eq!(code, Some(ChildExit::Finished(Some(0)))); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_stdout(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - { - let mut output = Vec::new(); - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => { - stdout - .read_to_end(&mut output) - .await - .expect("Failed to read stdout"); - } - ChildOutput::Pty(mut outputs) => { - outputs - .read_to_end(&mut output) - .expect("failed to read stdout"); - } - }; - - let output_str = String::from_utf8(output).expect("Failed to parse stdout"); - let trimmed_output = output_str.trim(); - let trimmed_output = trimmed_output.strip_prefix(EOT).unwrap_or(trimmed_output); - - assert_eq!(trimmed_output, "hello world"); - } - - let exit = child.wait().await; - - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_stdio(use_pty: bool) { - let script = find_script_dir().join_component("stdin_stdout.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let input = "hello world"; - // drop stdin to close the pipe - { - match child.stdin_inner().unwrap() { - ChildInput::Std(mut stdin) => stdin.write_all(input.as_bytes()).await.unwrap(), - ChildInput::Pty(mut stdin) => stdin.write_all(input.as_bytes()).unwrap(), - } - } - - let mut output = Vec::new(); - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => stdout.read_to_end(&mut output).await.unwrap(), - ChildOutput::Pty(mut stdout) => stdout.read_to_end(&mut output).unwrap(), - }; - - let output_str = String::from_utf8(output).expect("Failed to parse stdout"); - let trimmed_out = output_str.trim(); - let trimmed_out = trimmed_out.strip_prefix(EOT).unwrap_or(trimmed_out); - - assert!(trimmed_out.contains(input), "got: {trimmed_out}"); - - let exit = child.wait().await; - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - /// Regression test for #7834: proves that a child process can block - /// before producing any output if stdin is an open pipe instead of EOF. - /// - /// This models the v1.13 regression on Windows stream mode: - /// `tsx watch` received an open piped stdin and never started executing. - #[tokio::test] - async fn test_std_open_stdin_blocks_startup_until_eof() { - let script = find_script_dir().join_component("startup_after_stdin_eof.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let ChildOutput::Std { mut stdout, .. } = child.outputs().unwrap() else { - panic!("expected stdio child"); - }; - - let mut output = Vec::new(); - let result = - tokio::time::timeout(Duration::from_secs(1), stdout.read_to_end(&mut output)).await; - assert!( - result.is_err(), - "child should stay blocked while stdin is held open" - ); - assert!( - output.is_empty(), - "child should not produce output before stdin reaches EOF" - ); - - // Closing the parent's stdin pipe should unblock the child immediately. - drop(child.stdin_inner()); - - tokio::time::timeout(Duration::from_secs(5), stdout.read_to_end(&mut output)) - .await - .expect("child should finish reading after stdin is closed") - .expect("failed to read child output"); - - let exit = tokio::time::timeout(Duration::from_secs(5), child.wait()) - .await - .expect("child should exit after stdin is closed"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - - let output = String::from_utf8(output).unwrap().replace("\r\n", "\n"); - assert_eq!(output, "stdin bytes=0\nstarted\n"); - } - - /// Regression test for #7834: verifies the pre-v1.13 behavior where tasks - /// that do not need input start immediately when stdin is already at EOF. - #[tokio::test] - async fn test_std_null_stdin_allows_startup() { - let script = find_script_dir().join_component("startup_after_stdin_eof.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); - - let mut output = Vec::new(); - let exit = tokio::time::timeout( - Duration::from_secs(5), - child.wait_with_piped_outputs(&mut output), - ) - .await - .expect("child should not block when stdin is null") - .expect("failed to wait for child output"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - - let output = String::from_utf8(output).unwrap().replace("\r\n", "\n"); - assert_eq!(output, "stdin bytes=0\nstarted\n"); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_graceful_shutdown_timeout(use_pty: bool) { - let cmd = { - let script = find_script_dir().join_component("sleep_5_ignore.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd - }; - - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), - use_pty.then(PtySize::default), - ) - .unwrap(); - - let mut buf = vec![0; 4]; - // wait for the process to print "here" - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => { - stdout.read_exact(&mut buf).await.unwrap(); - } - ChildOutput::Pty(mut stdout) => { - stdout.read_exact(&mut buf).unwrap(); - } - }; - child.stop().await; - - let exit = child.wait().await; - // this should time out and be killed - assert_matches!(exit, Some(ChildExit::Killed)); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_graceful_shutdown(use_pty: bool) { - let cmd = { - let script = find_script_dir().join_component("sleep_5_interruptable.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd - }; - - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), - use_pty.then(PtySize::default), - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - // We need to read the child output otherwise the child will be unable to - // cleanly shut down as it waits for the receiving end of the PTY to read - // the output before exiting. - let mut output_child = child.clone(); - tokio::task::spawn(async move { - let mut output = Vec::new(); - output_child.wait_with_piped_outputs(&mut output).await.ok(); - }); - - child.stop().await; - let exit = child.wait().await; - - if cfg!(windows) { - assert_matches!(exit, Some(ChildExit::Killed)); - } else { - assert_matches!(exit, Some(ChildExit::Interrupted)); - } - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_graceful_shutdown_drains_final_output(use_pty: bool) { - let script = find_script_dir().join_component("graceful_sigint_output.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), - use_pty.then(PtySize::default), - ) - .unwrap(); - - let mut output_child = child.clone(); - let (mut observer, output, ready_rx) = ObservedOutput::new(); - let output_task = tokio::spawn(async move { - output_child - .wait_with_piped_outputs(&mut observer) - .await - .unwrap() - }); - - tokio::time::timeout(Duration::from_secs(2), ready_rx) - .await - .expect("timed out waiting for startup output") - .expect("ready notification channel closed unexpectedly"); - child.set_closing(); - child.stop().await; - let exit = output_task.await.unwrap(); - let output = String::from_utf8(output.lock().unwrap().clone()).unwrap(); - - assert!(output.contains("ready"), "missing startup output: {output}"); - - if cfg!(windows) { - assert_matches!(exit, Some(ChildExit::Killed)); - } else { - assert!( - output.contains("received SIGINT"), - "missing SIGINT receipt log: {output}" - ); - assert!( - output.contains("exiting after SIGINT"), - "missing SIGINT exit log: {output}" - ); - assert_matches!(exit, Some(ChildExit::Interrupted)); - } - } - - // Regression test: a wrapper process (simulating npm/pnpm) forwards SIGINT - // to its child. When turbo sends SIGINT to the process group, the child - // gets it twice — once from the group signal, once from the wrapper. - // For PTY children we now signal only the direct PID to avoid this. - #[cfg(unix)] - #[tokio::test] - #[traced_test] - async fn test_pty_child_receives_single_sigint() { - let script = find_script_dir().join_component("wrapper_count_sigints.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(2000))), - Some(PtySize::default()), - ) - .unwrap(); - - let mut output_child = child.clone(); - let (mut observer, output, ready_rx) = ObservedOutput::new(); - let output_task = tokio::spawn(async move { - output_child - .wait_with_piped_outputs(&mut observer) - .await - .unwrap() - }); - - tokio::time::timeout(Duration::from_secs(5), ready_rx) - .await - .expect("timed out waiting for ready") - .expect("ready channel closed"); - - child.set_closing(); - child.stop().await; - output_task.await.unwrap(); - - let output = String::from_utf8(output.lock().unwrap().clone()).unwrap(); - assert!( - output.contains("SIGINT_COUNT=1"), - "expected exactly one SIGINT, got output: {output}" - ); - assert!( - !output.contains("SIGINT_COUNT=2"), - "child received SIGINT twice: {output}" - ); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_detect_killed_someone_else(use_pty: bool) { - let cmd = { - let script = find_script_dir().join_component("sleep_5_interruptable.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd - }; - - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), - use_pty.then(PtySize::default), - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - #[cfg(unix)] - if let Some(pid) = child.pid() { - unsafe { - libc::kill(pid as i32, libc::SIGINT); - } - } - #[cfg(windows)] - if let Some(pid) = child.pid() { - unsafe { - println!("killing"); - windows_sys::Win32::System::Threading::TerminateProcess( - windows_sys::Win32::System::Threading::OpenProcess( - windows_sys::Win32::System::Threading::PROCESS_TERMINATE, - 0, - pid, - ), - 3, - ); - } - } - - let exit = child.wait().await; - - #[cfg(unix)] - assert_matches!(exit, Some(ChildExit::KilledExternal)); - #[cfg(not(unix))] - assert_matches!(exit, Some(ChildExit::Finished(Some(3)))); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_wait_with_output(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - let mut out = Vec::new(); - - let exit = child.wait_with_piped_outputs(&mut out).await.unwrap(); - - let out = String::from_utf8(out).unwrap(); - let trimmed_out = out.trim(); - let trimmed_out = trimmed_out.strip_prefix(EOT).unwrap_or(trimmed_out); - - assert_eq!(trimmed_out, "hello world"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_wait_with_single_output(use_pty: bool) { - let script = find_script_dir().join_component("hello_world_hello_moon.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - let mut buffer = Vec::new(); - - let exit = child.wait_with_piped_outputs(&mut buffer).await.unwrap(); - - let output = String::from_utf8(buffer).unwrap(); - - // There are no ordering guarantees so we just check that both logs made it - let expected_stdout = "hello world"; - let expected_stderr = "hello moon"; - assert!(output.contains(expected_stdout), "got: {output}"); - assert!(output.contains(expected_stderr), "got: {output}"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_wait_with_with_non_utf8_output(use_pty: bool) { - let script = find_script_dir().join_component("hello_non_utf8.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - let mut out = Vec::new(); - - let exit = child.wait_with_piped_outputs(&mut out).await.unwrap(); - - let expected = &[0, 159, 146, 150]; - let trimmed_out = out.trim_ascii(); - let trimmed_out = trimmed_out.strip_prefix(&[4]).unwrap_or(trimmed_out); - assert_eq!(trimmed_out, expected); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_no_newline(use_pty: bool) { - let script = find_script_dir().join_component("hello_no_line.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - let mut out = Vec::new(); - - let exit = child.wait_with_piped_outputs(&mut out).await.unwrap(); - - let output = String::from_utf8(out).unwrap(); - let trimmed_out = output.trim(); - let trimmed_out = trimmed_out.strip_prefix(EOT).unwrap_or(trimmed_out); - assert!( - output.ends_with('\n'), - "expected newline to be added: {output}" - ); - assert_eq!(trimmed_out, "look ma, no newline!"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - #[cfg(unix)] - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_kill_process_group(use_pty: bool) { - let mut cmd = Command::new("sh"); - cmd.args(["-c", "while true; do sleep 0.2; done"]); - let mut child = Child::spawn( - cmd, - // Bumping this to give ample time for the process to respond to the SIGINT to reduce - // flakiness inherent with sending and receiving signals. - ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), - use_pty.then(PtySize::default), - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - // We need to read the child output otherwise the child will be unable to - // cleanly shut down as it waits for the receiving end of the PTY to read - // the output before exiting. - let mut output_child = child.clone(); - tokio::task::spawn(async move { - let mut output = Vec::new(); - output_child.wait_with_piped_outputs(&mut output).await.ok(); - }); - - let exit = child.stop().await; - - // On Unix, shell scripts may not respond to SIGINT and will timeout, - // resulting in being killed rather than interrupted. - if cfg!(unix) { - assert_matches!(exit, Some(ChildExit::Killed) | Some(ChildExit::Interrupted)); - } else { - assert_matches!(exit, Some(ChildExit::Interrupted)); - } - } - - #[cfg(unix)] - #[tokio::test] - async fn test_orphan_process() { - let mut cmd = Command::new("sh"); - cmd.args(["-c", "echo hello; exec sleep 120"]); - let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let child_pid = child.pid().unwrap() as i32; - // We don't kill the process group to simulate what an external program might do - unsafe { - libc::kill(child_pid, libc::SIGKILL); - } - - let exit = child.wait().await; - assert_matches!(exit, Some(ChildExit::KilledExternal)); - - let mut output = Vec::new(); - match tokio::time::timeout( - Duration::from_millis(500), - child.wait_with_piped_outputs(&mut output), - ) - .await - { - Ok(exit_status) => { - assert_matches!(exit_status, Ok(Some(ChildExit::KilledExternal))); - } - Err(_) => panic!("expected wait_with_piped_outputs to exit after it was killed"), - } - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_graceful_shutdown_waits_for_force_kill(use_pty: bool) { - let script = find_script_dir().join_component("sleep_5_ignore.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_secs(5))), - use_pty.then(PtySize::default), - ) - .unwrap(); - - let mut buf = vec![0; 4]; - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => { - stdout.read_exact(&mut buf).await.unwrap(); - } - ChildOutput::Pty(mut stdout) => { - stdout.read_exact(&mut buf).unwrap(); - } - }; - - let mut shutdown_child = child.clone(); - let shutdown = - tokio::spawn( - async move { shutdown_child.shutdown(ShutdownStyle::Graceful(None)).await }, - ); - - tokio::time::sleep(Duration::from_millis(200)).await; - assert!( - !shutdown.is_finished(), - "graceful shutdown should keep waiting until explicitly forced" - ); - - assert_eq!(child.kill().await, Some(ChildExit::Killed)); - assert_eq!(shutdown.await.unwrap(), Some(ChildExit::Killed)); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_multistop(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - let child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - let mut stops = FuturesUnordered::new(); - for _ in 1..10 { - let mut child = child.clone(); - stops.push(async move { - child.stop().await; - }); - } - - while tokio::time::timeout(Duration::from_secs(5), stops.next()) - .await - .expect("timed out") - .is_some() - {} - } - - // Regression tests for https://github.com/vercel/turborepo/issues/11808 - // - // On Windows, portable-pty 0.9.0 added PSEUDOCONSOLE_INHERIT_CURSOR to - // ConPTY creation, which requires the host to handle DSR (Device Status - // Report) escape sequences. Turborepo doesn't, causing ConPTY to hang. - // - // Additionally, an unconditional `drop(stdin)` in the PTY path of - // wait_with_piped_outputs would kill ConPTY children on Windows because - // closing ConPTY stdin terminates the session. - // - // These tests verify the fixes: PTY children start, produce output, and - // exit normally without hanging or being killed by stdin closure. - - /// Verifies that a PTY-spawned short-lived process produces output and - /// exits cleanly via wait_with_piped_outputs. Uses a timeout to catch - /// the ConPTY hang that occurred with PSEUDOCONSOLE_INHERIT_CURSOR. - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_pty_child_does_not_hang(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - let mut out = Vec::new(); - - let result = tokio::time::timeout( - Duration::from_secs(10), - child.wait_with_piped_outputs(&mut out), - ) - .await; - - let exit = result - .expect("PTY child hung — likely PSEUDOCONSOLE_INHERIT_CURSOR regression") - .unwrap(); - - let output = String::from_utf8(out).unwrap(); - let trimmed = output.trim().strip_prefix(EOT).unwrap_or(output.trim()); - assert_eq!(trimmed, "hello world"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - /// Simulates the persistent-task flow: stdin is taken by the caller - /// (as the TUI does for interactive tasks) BEFORE wait_with_piped_outputs - /// is called. The child should still produce output and exit normally - /// without wait_with_piped_outputs interfering with stdin. - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_pty_stdin_taken_before_piped_outputs(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - // Take stdin before piping outputs, simulating TUI taking ownership. - // For PTY children, this returns Some; for non-PTY, stdin() returns None - // (Std variant is filtered out), but stdin_inner still removes it. - let _stdin_guard = child.stdin(); - - // Verify stdin_inner is now empty (already taken). - assert!( - child.stdin_inner().is_none(), - "stdin should already be taken" - ); - - let mut out = Vec::new(); - - let result = tokio::time::timeout( - Duration::from_secs(10), - child.wait_with_piped_outputs(&mut out), - ) - .await; - - let exit = result - .expect("child hung — wait_with_piped_outputs likely interfered with taken stdin") - .unwrap(); - - let output = String::from_utf8(out).unwrap(); - let trimmed = output.trim().strip_prefix(EOT).unwrap_or(output.trim()); - assert_eq!(trimmed, "hello world"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - /// Verifies that a PTY-spawned process with open stdin that has NOT been - /// taken by the caller still completes normally. This is the non-persistent - /// task path where exec.rs does not take stdin before - /// wait_with_piped_outputs. - /// - /// Before the fix, on Windows the unconditional stdin drop inside - /// wait_with_piped_outputs would kill the ConPTY child immediately. - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_pty_untaken_stdin_does_not_kill_child(use_pty: bool) { - let script = find_script_dir().join_component("hello_world.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - // Do NOT take stdin — this simulates a non-persistent task where - // exec.rs skips stdin handling on Windows (closing_stdin_ends_process). - // wait_with_piped_outputs should still work without killing the child. - let mut out = Vec::new(); - - let result = tokio::time::timeout( - Duration::from_secs(10), - child.wait_with_piped_outputs(&mut out), - ) - .await; - - let exit = result - .expect("child process hung or was killed by premature stdin closure") - .unwrap(); - - let output = String::from_utf8(out).unwrap(); - let trimmed = output.trim().strip_prefix(EOT).unwrap_or(output.trim()); - assert_eq!(trimmed, "hello world"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - /// Regression test for #12393: proves that dropping stdin causes a - /// persistent-style child (one that exits on stdin EOF) to terminate. - /// - /// This documents the mechanism behind the bug: when the task executor - /// took stdin and passed it to `TaskOutput::set_stdin()` in stream mode, - /// the stdin was dropped immediately, sending EOF to the child. - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_dropping_stdin_terminates_persistent_child(use_pty: bool) { - let script = find_script_dir().join_component("persistent_server.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - // Take stdin and immediately drop it — simulates the bug where - // TaskOutput::stream().set_stdin() dropped stdin in stream mode. - { - let _dropped = child.stdin(); - } - - // The child should exit because it received EOF on stdin. - let mut out = Vec::new(); - let result = tokio::time::timeout( - Duration::from_secs(5), - child.wait_with_piped_outputs(&mut out), - ) - .await; - - let exit = result - .expect("child should have exited after stdin was dropped") - .unwrap(); - - let output = String::from_utf8(out).unwrap(); - assert!( - output.contains("server ready"), - "expected 'server ready' in output, got: {output:?}" - ); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - /// Regression test for #12393: proves that holding stdin in a guard - /// keeps a persistent-style child alive. - /// - /// This is the correct behavior after the fix: in stream mode, stdin - /// is held by `_stdin_guard` instead of being passed to - /// `TaskOutput::set_stdin()` which would drop it. - /// - /// This covers the writable PTY path used for interactive input. - #[tokio::test] - async fn test_held_stdin_keeps_persistent_child_alive() { - if !TEST_PTY { - return; - } - let script = find_script_dir().join_component("persistent_server.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = Child::spawn(cmd, ShutdownStyle::Kill, Some(PtySize::default())).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - // Hold stdin in a guard — simulates the correct persistent task flow. - let _stdin_guard = child.stdin(); - assert!( - _stdin_guard.is_some(), - "PTY child should return Some from stdin()" - ); - - // The child should NOT exit while we hold stdin. Give it a moment - // and verify it's still alive by checking that wait times out. - let result = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; - assert!( - result.is_err(), - "child should still be alive while stdin is held" - ); - - // Now drop the guard — child should exit. - drop(_stdin_guard); - - let exit = tokio::time::timeout(Duration::from_secs(5), child.wait()) - .await - .expect("child should exit after stdin guard is dropped"); - - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - #[tokio::test] - async fn test_non_pty_stdin_guard_keeps_persistent_child_alive() { - let script = find_script_dir().join_component("persistent_server.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let stdin_guard = child.take_stdin(); - assert!( - matches!(stdin_guard, Some(ChildStdin::Guard(_))), - "non-PTY child should return a stdin guard" - ); - - let result = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; - assert!( - result.is_err(), - "child should still be alive while stdin is held" - ); - - drop(stdin_guard); - - let exit = tokio::time::timeout(Duration::from_secs(5), child.wait()) - .await - .expect("child should exit after stdin guard is dropped"); - assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); - } - - /// Verifies that stopping a parent process also kills its child processes. - /// - /// On Unix this works via process groups (setpgid + kill(-pgid)). - /// On Windows this works via Job Objects - /// (JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE). - /// - /// The test spawns a Node.js script that itself spawns a long-running child - /// process, captures the grandchild's PID from stdout, stops the parent, - /// and then checks that the grandchild is no longer alive. - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - #[traced_test] - async fn test_process_tree_cleanup(use_pty: bool) { - let script = find_script_dir().join_component("spawn_child_sleep.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(500))), - use_pty.then(PtySize::default), - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - // Read stdout to get the grandchild PID - let grandchild_pid = { - let mut out = Vec::new(); - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => { - let mut buf = vec![0u8; 256]; - let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) - .await - .expect("timed out reading grandchild PID") - .expect("failed to read stdout"); - out.extend_from_slice(&buf[..n]); - } - ChildOutput::Pty(mut reader) => { - let mut buf = vec![0u8; 256]; - let n = reader.read(&mut buf).expect("failed to read pty output"); - out.extend_from_slice(&buf[..n]); - } - }; - let output = String::from_utf8(out).unwrap(); - let pid_line = output - .lines() - .find(|line| line.contains("CHILD_PID=")) - .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); - pid_line - .split('=') - .nth(1) - .unwrap() - .trim() - .parse::() - .unwrap() - }; - - // Verify grandchild is alive before we stop - assert!( - is_process_alive(grandchild_pid), - "grandchild process {grandchild_pid} should be alive before stop" - ); - - // Stop the parent process - child.stop().await; - - // Give the OS a moment to clean up - tokio::time::sleep(Duration::from_millis(500)).await; - - // Verify grandchild is dead - assert!( - !is_process_alive(grandchild_pid), - "grandchild process {grandchild_pid} should have been killed" - ); - } - - #[test_case(false)] - #[test_case(TEST_PTY)] - #[tokio::test] - async fn test_force_kill_process_tree_cleanup(use_pty: bool) { - let script = find_script_dir().join_component("spawn_child_sleep.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = - Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let grandchild_pid = { - let mut out = Vec::new(); - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => { - let mut buf = vec![0u8; 256]; - let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) - .await - .expect("timed out reading grandchild PID") - .expect("failed to read stdout"); - out.extend_from_slice(&buf[..n]); - } - ChildOutput::Pty(mut reader) => { - let mut buf = vec![0u8; 256]; - let n = reader.read(&mut buf).expect("failed to read pty output"); - out.extend_from_slice(&buf[..n]); - } - }; - let output = String::from_utf8(out).unwrap(); - let pid_line = output - .lines() - .find(|line| line.contains("CHILD_PID=")) - .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); - pid_line - .split('=') - .nth(1) - .unwrap() - .trim() - .parse::() - .unwrap() - }; - - assert!( - is_process_alive(grandchild_pid), - "grandchild process {grandchild_pid} should be alive before force kill" - ); - - assert_eq!(child.kill().await, Some(ChildExit::Killed)); - tokio::time::sleep(Duration::from_millis(200)).await; - - assert!( - !is_process_alive(grandchild_pid), - "grandchild process {grandchild_pid} should have been force killed" - ); - } - - // Regression tests for the pre_exec/setsid -> process_group(0) migration. - // - // We replaced an unsafe pre_exec callback that called setsid() with tokio's - // safe process_group(0) API. These tests verify the critical invariants: - // - // 1. The child gets its own process group (PGID == child PID, not parent's) - // 2. Grandchildren inherit the child's process group - // 3. kill(-pgid, SIGINT) reaches both child and grandchild - // 4. The child is NOT a session leader (regression guard against setsid) - - #[cfg(unix)] - #[tokio::test] - async fn test_child_has_own_process_group() { - let script = find_script_dir().join_component("sleep_5_interruptable.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(500))), - None, - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let child_pid = child.pid().expect("child should have a pid") as libc::pid_t; - let child_pgid = unsafe { libc::getpgid(child_pid) }; - let parent_pgid = unsafe { libc::getpgid(0) }; - - // process_group(0) should make the child's PGID equal its own PID - assert_eq!( - child_pgid, child_pid, - "child PGID ({child_pgid}) should equal child PID ({child_pid})" - ); - - // The child's process group must differ from the parent's - assert_ne!( - child_pgid, parent_pgid, - "child PGID ({child_pgid}) must differ from parent PGID ({parent_pgid})" - ); - - child.stop().await; - } - - #[cfg(unix)] - #[tokio::test] - async fn test_grandchild_inherits_child_process_group() { - let script = find_script_dir().join_component("spawn_child_sleep.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(500))), - None, - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let child_pid = child.pid().expect("child should have a pid") as libc::pid_t; - - // Read the grandchild PID from stdout - let grandchild_pid = { - let mut out = Vec::new(); - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => { - let mut buf = vec![0u8; 256]; - let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) - .await - .expect("timed out reading grandchild PID") - .expect("failed to read stdout"); - out.extend_from_slice(&buf[..n]); - } - ChildOutput::Pty(_) => unreachable!("test uses non-PTY mode"), - }; - let output = String::from_utf8(out).unwrap(); - let pid_line = output - .lines() - .find(|line| line.contains("CHILD_PID=")) - .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); - pid_line - .split('=') - .nth(1) - .unwrap() - .trim() - .parse::() - .unwrap() - }; - - let child_pgid = unsafe { libc::getpgid(child_pid) }; - let grandchild_pgid = unsafe { libc::getpgid(grandchild_pid) }; - - // Grandchild should be in the same process group as the child - assert_eq!( - grandchild_pgid, child_pgid, - "grandchild PGID ({grandchild_pgid}) should match child PGID ({child_pgid})" - ); - - // Both should use child_pid as the group ID - assert_eq!( - child_pgid, child_pid, - "process group ID ({child_pgid}) should equal child PID ({child_pid})" - ); - - child.stop().await; - // Give OS time to clean up - tokio::time::sleep(Duration::from_millis(200)).await; - } - - #[cfg(unix)] - #[tokio::test] - async fn test_pty_graceful_shutdown_signals_process_group() { - let marker_file = std::env::temp_dir().join(format!( - "turbo-pty-process-group-sigint-{}", - std::process::id() - )); - let ready_file = std::env::temp_dir().join(format!( - "turbo-pty-process-group-ready-{}", - std::process::id() - )); - let _ = fs::remove_file(&marker_file); - let _ = fs::remove_file(&ready_file); - let marker_file = marker_file.to_string_lossy().into_owned(); - let ready_file = ready_file.to_string_lossy().into_owned(); - - let script = r#" -const { spawn } = require("child_process"); -process.on("SIGINT", () => {}); -const child = spawn(process.execPath, [ - "-e", - "process.on('SIGINT', () => { require('fs').writeFileSync(process.argv[1], 'interrupted'); process.exit(0); }); require('fs').writeFileSync(process.argv[2], 'ready'); setInterval(() => {}, 1000);", - process.argv[1], - process.argv[2], -], { stdio: "inherit" }); -child.on("exit", () => process.exit(0)); -"#; - let mut cmd = Command::new("node"); - cmd.args(["-e", script, marker_file.as_str(), ready_file.as_str()]); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_secs(2))), - Some(PtySize::default()), - ) - .unwrap(); - - for _ in 0..50 { - if fs::metadata(&ready_file).is_ok() { - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - assert!( - fs::metadata(&ready_file).is_ok(), - "node child should become ready before shutdown" - ); - - let exit = child.stop().await; - - assert_eq!(exit, Some(ChildExit::Interrupted)); - assert!( - fs::metadata(&marker_file).is_ok(), - "node child should receive SIGINT from PTY process-group shutdown" - ); - - let _ = fs::remove_file(marker_file); - let _ = fs::remove_file(ready_file); - } - - #[cfg(unix)] - #[tokio::test] - async fn test_sigint_to_process_group_reaches_grandchild() { - let script = find_script_dir().join_component("spawn_child_sleep.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - cmd.open_stdin(); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(2000))), - None, - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let child_pid = child.pid().expect("child should have a pid"); - - // Read the grandchild PID - let grandchild_pid = { - let mut out = Vec::new(); - match child.outputs().unwrap() { - ChildOutput::Std { mut stdout, .. } => { - let mut buf = vec![0u8; 256]; - let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) - .await - .expect("timed out reading grandchild PID") - .expect("failed to read stdout"); - out.extend_from_slice(&buf[..n]); - } - ChildOutput::Pty(_) => unreachable!("test uses non-PTY mode"), - }; - let output = String::from_utf8(out).unwrap(); - let pid_line = output - .lines() - .find(|line| line.contains("CHILD_PID=")) - .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); - pid_line - .split('=') - .nth(1) - .unwrap() - .trim() - .parse::() - .unwrap() - }; - - assert!( - is_process_alive(grandchild_pid), - "grandchild should be alive before signal" - ); - - // Send SIGINT to the process group (negative PID), exactly as - // ShutdownStyle::Graceful does in production code - let pgid = -(child_pid as i32); - unsafe { - libc::kill(pgid, libc::SIGINT); - } - - // Wait for processes to die - tokio::time::sleep(Duration::from_millis(500)).await; - - assert!( - !is_process_alive(grandchild_pid), - "grandchild should be dead after SIGINT to process group" - ); - - // Consume the exit - child.wait().await; - } - - // Guard against accidentally reverting to setsid(). With process_group(0), - // the child calls setpgid(0, 0) which creates a new process group but does - // NOT create a new session. If someone reintroduces setsid(), the child's - // SID would equal its PID. With setpgid, the SID is inherited from the - // parent. - #[cfg(unix)] - #[tokio::test] - async fn test_child_is_not_session_leader() { - let script = find_script_dir().join_component("sleep_5_interruptable.js"); - let mut cmd = Command::new("node"); - cmd.args([script.as_std_path()]); - let mut child = Child::spawn( - cmd, - ShutdownStyle::Graceful(Some(Duration::from_millis(500))), - None, - ) - .unwrap(); - - tokio::time::sleep(STARTUP_DELAY).await; - - let child_pid = child.pid().expect("child should have a pid") as libc::pid_t; - let child_sid = unsafe { libc::getsid(child_pid) }; - let parent_sid = unsafe { libc::getsid(0) }; - - // With process_group(0), the child inherits the parent's session. - // If setsid() were used instead, child_sid would equal child_pid. - assert_ne!( - child_sid, child_pid, - "child SID ({child_sid}) should NOT equal child PID ({child_pid}) — that would mean \ - setsid() was called" - ); - assert_eq!( - child_sid, parent_sid, - "child SID ({child_sid}) should equal parent SID ({parent_sid})" - ); - - child.stop().await; - } - - fn is_process_alive(pid: u32) -> bool { - #[cfg(unix)] - { - // kill(pid, 0) checks if process exists without sending a signal - unsafe { libc::kill(pid as i32, 0) == 0 } - } - #[cfg(windows)] - { - use windows_sys::Win32::{ - Foundation::CloseHandle, - System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}, - }; - unsafe { - let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); - if handle.is_null() { - return false; - } - // Process handle opened — check if it's actually still running - let mut exit_code: u32 = 0; - let result = windows_sys::Win32::System::Threading::GetExitCodeProcess( - handle, - &mut exit_code, - ); - CloseHandle(handle); - // STILL_ACTIVE (259) means the process is still running - result != 0 && exit_code == 259 - } - } - } -} diff --git a/crates/turborepo-process/src/child/handle.rs b/crates/turborepo-process/src/child/handle.rs new file mode 100644 index 0000000000000..5510c4d4ce7df --- /dev/null +++ b/crates/turborepo-process/src/child/handle.rs @@ -0,0 +1,648 @@ +use std::{io, time::Duration}; + +use portable_pty::{Child as PtyChild, MasterPty as PtyController, native_pty_system}; +use tokio::{process::Command as TokioCommand, sync::mpsc}; +use tracing::debug; + +use super::{ChildCommand, ChildExit, ChildIO, ChildInput, ChildOutput}; +use crate::{Command, PtySize}; + +const CHILD_POLL_INTERVAL: Duration = Duration::from_micros(50); +#[cfg(any(unix, windows))] +const PROCESS_TREE_DRAIN_POLL_INTERVAL: Duration = Duration::from_millis(10); +#[cfg(windows)] +const WINDOWS_DESCENDANT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + +pub(super) struct ChildHandle { + pid: Option, + imp: ChildHandleImpl, + #[cfg(unix)] + shutdown_semantics: ShutdownSemantics, + #[cfg(unix)] + pub(super) target_identity: Option, + #[cfg(unix)] + pty_controller_fd: Option, + #[cfg(windows)] + _job: Option, +} + +enum ChildHandleImpl { + Tokio(Option), + Pty(Box), +} + +#[cfg(unix)] +#[derive(Debug, Clone, Copy)] +enum GracefulInterruptTarget { + DirectChildWithProcessGroupFallback, + ProcessGroup, +} + +#[cfg(unix)] +#[derive(Debug, Clone, Copy)] +struct ShutdownSemantics { + // Who should receive the first graceful interrupt. + graceful_interrupt_target: GracefulInterruptTarget, + // Whether we should keep waiting on the process group after the direct child exits. + wait_for_process_group_after_child_exit: bool, +} + +#[cfg(unix)] +impl ShutdownSemantics { + fn process_group() -> Self { + Self { + graceful_interrupt_target: GracefulInterruptTarget::ProcessGroup, + wait_for_process_group_after_child_exit: true, + } + } + + fn direct_child_with_process_group_fallback() -> Self { + Self { + graceful_interrupt_target: GracefulInterruptTarget::DirectChildWithProcessGroupFallback, + wait_for_process_group_after_child_exit: true, + } + } +} + +#[cfg(unix)] +#[derive(Debug, Clone, Copy)] +pub(super) struct TargetIdentity { + pub(super) process_group_id: libc::pid_t, + pub(super) session_id: libc::pid_t, +} + +#[cfg(unix)] +fn target_identity(target_pid: libc::pid_t) -> io::Result { + let process_group_id = unsafe { libc::getpgid(target_pid) }; + if process_group_id == -1 { + return Err(io::Error::last_os_error()); + } + + let session_id = unsafe { libc::getsid(target_pid) }; + if session_id == -1 { + return Err(io::Error::last_os_error()); + } + + Ok(TargetIdentity { + process_group_id, + session_id, + }) +} + +#[cfg(unix)] +pub(super) fn process_group_matches_identity( + target_pid: libc::pid_t, + identity: TargetIdentity, +) -> bool { + let process_group_id = unsafe { libc::getpgid(target_pid) }; + if process_group_id != -1 { + if process_group_id != identity.process_group_id { + return false; + } + + let session_id = unsafe { libc::getsid(target_pid) }; + return session_id != -1 && session_id == identity.session_id; + } + + let result = unsafe { libc::kill(-identity.process_group_id, 0) }; + result == 0 || io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +#[cfg(unix)] +pub(super) fn signal_process_group(process_group_id: libc::pid_t, signal: libc::c_int) { + let _ = unsafe { libc::kill(-process_group_id, signal) }; +} + +#[cfg(unix)] +fn capture_target_identity(pid: Option) -> Option { + pid.and_then(|pid| match target_identity(pid as libc::pid_t) { + Ok(identity) => Some(identity), + Err(err) => { + debug!("failed to capture target identity for process {pid}: {err}"); + None + } + }) +} + +impl ChildHandle { + #[tracing::instrument(skip(command))] + pub(super) fn spawn_normal(command: Command) -> io::Result { + #[cfg(windows)] + let command_for_fallback = command.clone(); + + let mut command = TokioCommand::from(command); + + // Create a new process group so we can send signals (e.g. SIGINT) to + // the child and all of its descendants via kill(-pgid, sig). + #[cfg(unix)] + command.process_group(0); + + #[cfg(windows)] + let job = match crate::job_object::JobObject::new() { + Ok(job) => Some(job), + Err(err) => { + debug!("failed to create Windows JobObject: {err}"); + None + } + }; + + #[cfg(windows)] + if job.is_some() { + command.creation_flags( + windows_sys::Win32::System::Threading::CREATE_SUSPENDED + | windows_sys::Win32::System::Threading::CREATE_BREAKAWAY_FROM_JOB, + ); + } + + #[cfg(not(windows))] + let mut child = command.spawn()?; + + #[cfg(windows)] + let mut child = match command.spawn() { + Ok(child) => child, + Err(err) if job.is_some() => { + debug!("failed to spawn child with job breakaway: {err}"); + let mut fallback_command = TokioCommand::from(command_for_fallback); + fallback_command + .creation_flags(windows_sys::Win32::System::Threading::CREATE_SUSPENDED); + fallback_command.spawn()? + } + Err(err) => return Err(err), + }; + let pid = child.id(); + + #[cfg(unix)] + let target_identity = capture_target_identity(pid); + + #[cfg(windows)] + let job = job.and_then(|job| match child.raw_handle() { + Some(handle) => match job.assign_suspended_process(handle) { + Ok(true) => Some(job), + Ok(false) => None, + Err(err) => { + debug!("failed to resume suspended process after job assignment: {err}"); + child.start_kill().ok(); + None + } + }, + None => { + debug!("failed to get child process handle for job assignment"); + child.start_kill().ok(); + None + } + }); + + let stdin = child.stdin.take().map(ChildInput::Std); + let stdout = child + .stdout + .take() + .ok_or_else(|| io::Error::other("child process must be started with piped stdout"))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| io::Error::other("child process must be started with piped stderr"))?; + + Ok(SpawnResult { + handle: Self { + pid, + imp: ChildHandleImpl::Tokio(Some(child)), + #[cfg(unix)] + shutdown_semantics: ShutdownSemantics::process_group(), + #[cfg(unix)] + target_identity, + #[cfg(unix)] + pty_controller_fd: None, + #[cfg(windows)] + _job: job, + }, + io: ChildIO { + stdin, + output: Some(ChildOutput::Std { stdout, stderr }), + }, + controller: None, + }) + } + + #[tracing::instrument(skip(command))] + pub(super) fn spawn_pty(command: Command, size: PtySize) -> io::Result { + let keep_stdin_open = command.will_open_stdin(); + + let command = portable_pty::CommandBuilder::from(command); + let pty_system = native_pty_system(); + let size = portable_pty::PtySize { + rows: size.rows, + cols: size.cols, + pixel_width: 0, + pixel_height: 0, + }; + let pair = pty_system + .openpty(size) + .map_err(|err| match err.downcast() { + Ok(err) => err, + Err(err) => io::Error::other(err), + })?; + + let controller = pair.master; + let receiver = pair.slave; + + #[cfg(unix)] + { + use nix::sys::termios; + if let Some((file_desc, mut termios)) = controller + .as_raw_fd() + .and_then(|fd| Some(fd).zip(termios::tcgetattr(fd).ok())) + { + // We unset ECHOCTL to disable rendering of the closing of stdin + // as ^D + termios.local_flags &= !nix::sys::termios::LocalFlags::ECHOCTL; + if let Err(e) = nix::sys::termios::tcsetattr( + file_desc, + nix::sys::termios::SetArg::TCSANOW, + &termios, + ) { + debug!("unable to unset ECHOCTL: {e}"); + } + } + } + + let child = receiver + .spawn_command(command) + .map_err(|err| match err.downcast() { + Ok(err) => err, + Err(err) => io::Error::other(err), + })?; + + let pid = child.process_id(); + + #[cfg(unix)] + let target_identity = capture_target_identity(pid); + + #[cfg(windows)] + let job = pid.and_then(|pid| { + crate::job_object::JobObject::new() + .and_then(|job| job.assign_pid(pid).map(|_| job)) + .map_err(|e| debug!("failed to set up job object for PTY process {pid}: {e}")) + .ok() + }); + + #[cfg(unix)] + let pty_controller_fd = controller.as_raw_fd(); + + let mut stdin = controller.take_writer().ok(); + let output = controller.try_clone_reader().ok().map(ChildOutput::Pty); + + // portable-pty 0.9.0 creates ConPTY with PSEUDOCONSOLE_INHERIT_CURSOR, + // which sends a Device Status Report (DSR) cursor position request + // (\x1b[6n) on the output pipe during initialization. ConPTY blocks + // until the host responds with a Cursor Position Report on stdin. + // Without this response the PTY hangs indefinitely. + // See https://github.com/vercel/turborepo/issues/11808 + #[cfg(windows)] + if let Some(ref mut writer) = stdin { + // Respond with cursor at position (1,1). The actual position + // doesn't matter — ConPTY just needs a valid CPR to unblock. + if let Err(e) = writer.write_all(b"\x1b[1;1R") { + debug!("failed to write ConPTY cursor position response: {e}"); + } + } + + // If we don't want to keep stdin open we take it here and it is immediately + // dropped resulting in a EOF being sent to the child process. + if !keep_stdin_open { + stdin.take(); + } + + Ok(SpawnResult { + handle: Self { + pid, + imp: ChildHandleImpl::Pty(child), + #[cfg(unix)] + shutdown_semantics: ShutdownSemantics::direct_child_with_process_group_fallback(), + #[cfg(unix)] + target_identity, + #[cfg(unix)] + pty_controller_fd, + #[cfg(windows)] + _job: job, + }, + io: ChildIO { + stdin: stdin.map(ChildInput::Pty), + output, + }, + controller: Some(controller), + }) + } + + pub(super) fn pid(&self) -> Option { + self.pid + } + + #[cfg(unix)] + fn process_group_id(&self) -> Option { + self.target_identity + .map(|identity| identity.process_group_id) + .or(self.pid.map(|pid| pid as libc::pid_t)) + } + + #[cfg(unix)] + fn graceful_process_group_id(&self) -> Option { + self.pty_controller_fd + .and_then(|fd| match unsafe { libc::tcgetpgrp(fd) } { + process_group_id if process_group_id > 0 => Some(process_group_id), + _ => None, + }) + .or_else(|| self.process_group_id()) + } + + #[cfg(unix)] + fn send_signal_to_process_group(&self, pid: libc::pid_t, signal: libc::c_int) { + let Some(process_group_id) = self.graceful_process_group_id() else { + debug!("missing process group id for child {pid}"); + return; + }; + + debug!("sending signal {signal} to process group -{process_group_id}"); + signal_process_group(process_group_id, signal); + } + + #[cfg(unix)] + pub(super) fn send_graceful_interrupt(&self, pid: libc::pid_t) -> bool { + match self.shutdown_semantics.graceful_interrupt_target { + GracefulInterruptTarget::DirectChildWithProcessGroupFallback => { + debug!("sending SIGINT to child {pid}"); + if unsafe { libc::kill(pid, libc::SIGINT) } == -1 { + debug!("failed to send SIGINT to {pid}"); + } + false + } + GracefulInterruptTarget::ProcessGroup => { + self.send_signal_to_process_group(pid, libc::SIGINT); + true + } + } + } + + #[cfg(unix)] + pub(super) fn send_fallback_graceful_interrupt(&self, pid: libc::pid_t) { + self.send_signal_to_process_group(pid, libc::SIGINT); + } + + #[cfg(unix)] + fn should_wait_for_process_group_after_child_exit(&self) -> bool { + self.shutdown_semantics + .wait_for_process_group_after_child_exit + } + + #[cfg(unix)] + fn has_running_process_group(&self, pid: libc::pid_t) -> bool { + if let Some(identity) = self.target_identity { + return process_group_matches_identity(pid, identity); + } + + let process_group_id = self.process_group_id().unwrap_or(pid); + + let result = unsafe { libc::kill(-process_group_id, 0) }; + result == 0 || io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) + } + + #[cfg(unix)] + fn kill_process_group(&self, pid: libc::pid_t) { + let process_group_id = self.process_group_id().unwrap_or(pid); + + debug!("killing process group {}", process_group_id); + signal_process_group(process_group_id, libc::SIGKILL); + } + + #[cfg(unix)] + pub(super) async fn wait_for_process_group_exit( + &mut self, + pid: libc::pid_t, + deadline: Option, + command_rx: &mut mpsc::Receiver, + command_rx_open: &mut bool, + ) -> ChildExit { + if !self.should_wait_for_process_group_after_child_exit() { + return ChildExit::Interrupted; + } + + while self.has_running_process_group(pid) { + match deadline { + Some(deadline) => { + tokio::select! { + command = command_rx.recv(), if *command_rx_open => { + match command { + Some(ChildCommand::Kill) => { + debug!("graceful shutdown interrupted, killing process group"); + self.kill_process_group(pid); + return ChildExit::Killed; + } + Some(ChildCommand::Shutdown(_)) => {} + None => *command_rx_open = false, + } + } + _ = tokio::time::sleep_until(deadline) => { + debug!("graceful shutdown timed out, killing process group"); + self.kill_process_group(pid); + return ChildExit::Killed; + } + _ = tokio::time::sleep(PROCESS_TREE_DRAIN_POLL_INTERVAL) => {} + } + } + None => { + tokio::select! { + command = command_rx.recv(), if *command_rx_open => { + match command { + Some(ChildCommand::Kill) => { + debug!("graceful shutdown interrupted, killing process group"); + self.kill_process_group(pid); + return ChildExit::Killed; + } + Some(ChildCommand::Shutdown(_)) => {} + None => *command_rx_open = false, + } + } + _ = tokio::time::sleep(PROCESS_TREE_DRAIN_POLL_INTERVAL) => {} + } + } + } + } + + ChildExit::Interrupted + } + + #[cfg(windows)] + fn has_active_windows_job(&self) -> bool { + self._job + .as_ref() + .is_some_and(|job| match job.active_processes() { + Ok(active_processes) => active_processes > 0, + Err(err) => { + debug!("failed to query job object: {err}"); + false + } + }) + } + + #[cfg(windows)] + fn has_running_windows_descendants(&self) -> bool { + match self.pid { + Some(pid) => match crate::job_object::has_descendant_processes(pid) { + Ok(has_descendants) => has_descendants, + Err(err) => { + debug!("failed to query descendant processes: {err}"); + false + } + }, + None => false, + } + } + + #[cfg(windows)] + fn terminate_windows_process_tree(&self) { + if let Some(job) = &self._job + && let Err(err) = job.terminate() + { + debug!("failed to terminate job object: {err}"); + } + + if let Some(pid) = self.pid + && let Err(err) = crate::job_object::terminate_descendant_processes(pid) + { + debug!("failed to terminate descendant process tree: {err}"); + } + } + + #[cfg(windows)] + pub(super) async fn wait_for_job_exit( + &mut self, + deadline: Option, + command_rx: &mut mpsc::Receiver, + command_rx_open: &mut bool, + ) -> ChildExit { + // PID snapshots are only a fallback for runners where Job Object + // assignment fails. After the parent exits they can match unrelated + // reused PIDs, so never let that path wait forever. + let descendant_drain_deadline = self + ._job + .is_none() + .then(|| tokio::time::Instant::now() + WINDOWS_DESCENDANT_DRAIN_TIMEOUT); + + loop { + let has_active_job = self.has_active_windows_job(); + let has_descendants = self._job.is_none() && self.has_running_windows_descendants(); + + if !has_active_job && !has_descendants { + break; + } + + tokio::select! { + command = command_rx.recv(), if *command_rx_open => { + match command { + Some(ChildCommand::Kill) => { + debug!("process tree drain interrupted, terminating job object"); + self.terminate_windows_process_tree(); + return ChildExit::Killed; + } + Some(ChildCommand::Shutdown(_)) => {} + None => *command_rx_open = false, + } + } + _ = async { + if let Some(deadline) = deadline { + tokio::time::sleep_until(deadline).await; + } + }, if deadline.is_some() => { + debug!("graceful shutdown timed out, terminating Windows process tree"); + self.terminate_windows_process_tree(); + return ChildExit::Killed; + } + _ = async { + if let Some(deadline) = descendant_drain_deadline { + tokio::time::sleep_until(deadline).await; + } + }, if has_descendants && descendant_drain_deadline.is_some() => { + debug!("timed out waiting for Windows descendant process tree after direct child exit"); + break; + } + _ = tokio::time::sleep(PROCESS_TREE_DRAIN_POLL_INTERVAL) => {} + } + } + + ChildExit::Interrupted + } + + /// Perform a `wait` syscall on the child until it exits + pub(super) async fn wait(&mut self) -> io::Result> { + match &mut self.imp { + ChildHandleImpl::Tokio(child) => { + let result = match child { + Some(child) => child.wait().await.map(|status| status.code()), + None => Ok(None), + }; + + #[cfg(windows)] + if result.is_ok() { + // Drop the process handle before querying the Job Object so + // the exited direct child is not counted during tree drain. + child.take(); + } + + result + } + ChildHandleImpl::Pty(child) => { + // TODO: we currently poll the child to see if it has finished yet which is less + // than ideal + loop { + match child.try_wait() { + Ok(Some(status)) => { + // portable_pty maps the status of being killed by a signal to a 1 exit + // code. The only way to tell if the task + // exited normally with exit code 1 or got killed by a signal is to + // display it as the signal will be included + // in the message. + let exit_code = if status.exit_code() == 1 + && status.to_string().contains("Terminated by") + { + None + } else { + // This is safe as the portable_pty::ExitStatus's exit code is just + // converted from a i32 to an u32 before we get it + Some(status.exit_code() as i32) + }; + return Ok(exit_code); + } + Ok(None) => { + // child hasn't finished, we sleep for a short time + tokio::time::sleep(CHILD_POLL_INTERVAL).await; + } + Err(err) => return Err(err), + } + } + } + } + } + + pub(super) async fn kill(&mut self) -> io::Result<()> { + #[cfg(unix)] + if let Some(process_group_id) = self.process_group_id() { + signal_process_group(process_group_id, libc::SIGKILL); + } + + match &mut self.imp { + ChildHandleImpl::Tokio(Some(child)) => child.kill().await, + ChildHandleImpl::Tokio(None) => Ok(()), + ChildHandleImpl::Pty(child) => { + let mut killer = child.clone_killer(); + tokio::task::spawn_blocking(move || killer.kill()) + .await + .map_err(|err| io::Error::other(format!("pty kill task failed: {err}")))? + } + } + } +} + +pub(super) struct SpawnResult { + pub(super) handle: ChildHandle, + pub(super) io: ChildIO, + pub(super) controller: Option>, +} diff --git a/crates/turborepo-process/src/child/io.rs b/crates/turborepo-process/src/child/io.rs new file mode 100644 index 0000000000000..913f8cdfb37e4 --- /dev/null +++ b/crates/turborepo-process/src/child/io.rs @@ -0,0 +1,315 @@ +use std::{ + fmt, + io::{self, BufRead, Read, Write}, +}; + +use tokio::{ + io::{AsyncBufRead, AsyncBufReadExt, BufReader}, + sync::mpsc, +}; +use tracing::{debug, trace}; + +use super::{Child, ChildExit}; + +const POST_EXIT_OUTPUT_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(100); + +pub(super) struct ChildIO { + pub(super) stdin: Option, + pub(super) output: Option, +} + +pub(super) enum ChildInput { + Std(tokio::process::ChildStdin), + Pty(Box), +} + +#[derive(Debug)] +pub struct ChildStdinGuard { + _stdin: ChildInput, +} + +pub enum ChildStdin { + Writable(Box), + Guard(ChildStdinGuard), +} + +impl fmt::Debug for ChildStdin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Writable(_) => f.debug_tuple("Writable").finish(), + Self::Guard(guard) => f.debug_tuple("Guard").field(guard).finish(), + } + } +} + +pub(super) enum ChildOutput { + Std { + stdout: tokio::process::ChildStdout, + stderr: tokio::process::ChildStderr, + }, + Pty(Box), +} + +impl fmt::Debug for ChildInput { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Std(arg0) => f.debug_tuple("Std").field(arg0).finish(), + Self::Pty(_) => f.debug_tuple("Pty").finish(), + } + } +} + +impl fmt::Debug for ChildOutput { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Std { stdout, stderr } => f + .debug_struct("Std") + .field("stdout", stdout) + .field("stderr", stderr) + .finish(), + Self::Pty(_) => f.debug_tuple("Pty").finish(), + } + } +} + +impl Child { + pub(super) fn stdin_inner(&mut self) -> Option { + self.stdin + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take() + } + + pub(super) fn outputs(&self) -> Option { + self.output + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take() + } + + pub fn stdin(&mut self) -> Option> { + let stdin = self.stdin_inner()?; + match stdin { + ChildInput::Std(_) => None, + ChildInput::Pty(stdin) => Some(stdin), + } + } + + pub fn take_stdin(&mut self) -> Option { + let stdin = self.stdin_inner()?; + match stdin { + ChildInput::Std(stdin) => Some(ChildStdin::Guard(ChildStdinGuard { + _stdin: ChildInput::Std(stdin), + })), + ChildInput::Pty(stdin) => Some(ChildStdin::Writable(stdin)), + } + } + + /// Wait for the `Child` to exit and pipe any stdout and stderr to the + /// provided writer. + #[tracing::instrument(skip_all)] + pub async fn wait_with_piped_outputs( + &mut self, + stdout_pipe: W, + ) -> Result, std::io::Error> { + match self.outputs() { + Some(ChildOutput::Std { stdout, stderr }) => { + self.wait_with_piped_async_outputs( + stdout_pipe, + Some(BufReader::new(stdout)), + Some(BufReader::new(stderr)), + ) + .await + } + Some(ChildOutput::Pty(output)) => { + // On Unix, drop stdin before reading so the master PTY writer + // sends EOT and releases its fd, allowing the reader to reach + // EOF once the controller is dropped after the child exits. + // + // On Windows, do NOT drop stdin here: ConPTY treats a closed + // stdin pipe as the session ending and immediately terminates + // the child process. + if !cfg!(windows) { + drop(self.stdin_inner()); + } + self.wait_with_piped_sync_output(stdout_pipe, std::io::BufReader::new(output)) + .await + } + None => Ok(self.wait().await), + } + } + + #[tracing::instrument(skip_all)] + async fn wait_with_piped_sync_output( + &mut self, + mut stdout_pipe: impl Write, + mut stdout_lines: R, + ) -> Result, std::io::Error> { + // TODO: in order to not impose that a stdout_pipe is Send we send the bytes + // across a channel + let (byte_tx, mut byte_rx) = mpsc::channel(48); + tokio::task::spawn_blocking(move || { + let mut buffer = [0; 1024]; + let mut last_byte = None; + loop { + match stdout_lines.read(&mut buffer) { + Ok(0) => { + if !matches!(last_byte, Some(b'\n')) { + // Ignore if this fails as we already are shutting down + byte_tx.blocking_send(vec![b'\n']).ok(); + } + break; + } + Ok(n) => { + let mut bytes = Vec::with_capacity(n); + bytes.extend_from_slice(&buffer[..n]); + last_byte = bytes.last().copied(); + if byte_tx.blocking_send(bytes).is_err() { + // A dropped receiver indicates that there was an issue writing to the + // pipe. We can stop reading output. + break; + } + } + Err(e) => return Err(e), + } + } + Ok(()) + }); + + let writer_fut = async { + let mut result = Ok(()); + while let Some(bytes) = byte_rx.recv().await { + if let Err(err) = stdout_pipe.write_all(&bytes) { + result = Err(err); + break; + } + } + result + }; + + let (status, write_result) = tokio::join!(self.wait(), writer_fut); + write_result?; + self.cleanup_if_successful(status); + + Ok(status) + } + + #[tracing::instrument(skip_all)] + async fn wait_with_piped_async_outputs( + &mut self, + mut stdout_pipe: impl Write, + mut stdout_lines: Option, + mut stderr_lines: Option, + ) -> Result, std::io::Error> { + async fn next_line( + stream: &mut Option, + buffer: &mut Vec, + ) -> Option> { + match stream { + Some(stream) => match stream.read_until(b'\n', buffer).await { + Ok(0) => { + trace!("reached EOF"); + None + } + Ok(_) => Some(Ok(())), + Err(e) => Some(Err(e)), + }, + None => None, + } + } + + let mut stdout_buffer = Vec::new(); + let mut stderr_buffer = Vec::new(); + + let mut is_exited = false; + let mut exit_status = None; + let mut draining_after_exit = false; + let mut drain_deadline = tokio::time::Instant::now() + POST_EXIT_OUTPUT_DRAIN_TIMEOUT; + loop { + tokio::select! { + Some(result) = next_line(&mut stdout_lines, &mut stdout_buffer) => { + trace!("processing stdout line"); + result?; + add_trailing_newline(&mut stdout_buffer); + stdout_pipe.write_all(&stdout_buffer)?; + stdout_buffer.clear(); + } + Some(result) = next_line(&mut stderr_lines, &mut stderr_buffer) => { + trace!("processing stderr line"); + result?; + add_trailing_newline(&mut stderr_buffer); + stdout_pipe.write_all(&stderr_buffer)?; + stderr_buffer.clear(); + } + status = self.wait(), if !is_exited => { + trace!("child process exited: {}", self.label()); + is_exited = true; + exit_status = status; + // We don't abort in the cases of a zero exit code as we could be + // caching this task and should read all the logs it produces. + if status == Some(ChildExit::Finished(Some(0))) { + continue; + } + + if self.is_closing() { + // During Turbo-initiated shutdown, give the pipe readers a + // short grace window to pull the child's final log lines. + draining_after_exit = true; + drain_deadline = tokio::time::Instant::now() + POST_EXIT_OUTPUT_DRAIN_TIMEOUT; + } else { + debug!("child process failed, skipping reading stdout/stderr"); + return Ok(status); + } + } + _ = tokio::time::sleep_until(drain_deadline), if draining_after_exit => { + trace!("post-exit output drain timed out"); + if !stdout_buffer.is_empty() { + add_trailing_newline(&mut stdout_buffer); + stdout_pipe.write_all(&stdout_buffer)?; + stdout_buffer.clear(); + } + if !stderr_buffer.is_empty() { + add_trailing_newline(&mut stderr_buffer); + stdout_pipe.write_all(&stderr_buffer)?; + stderr_buffer.clear(); + } + return Ok(exit_status); + } + else => { + trace!("flushing child stdout/stderr buffers"); + // In the case that both futures read a complete line + // the future not chosen in the select will return None if it's at EOF + // as the number of bytes read will be 0. + // We check and flush the buffers to avoid missing the last line of output. + if !stdout_buffer.is_empty() { + add_trailing_newline(&mut stdout_buffer); + stdout_pipe.write_all(&stdout_buffer)?; + stdout_buffer.clear(); + } + if !stderr_buffer.is_empty() { + add_trailing_newline(&mut stderr_buffer); + stdout_pipe.write_all(&stderr_buffer)?; + stderr_buffer.clear(); + } + break; + } + } + } + debug_assert!(stdout_buffer.is_empty(), "buffer should be empty"); + debug_assert!(stderr_buffer.is_empty(), "buffer should be empty"); + + let status = exit_status.or(self.wait().await); + self.cleanup_if_successful(status); + Ok(status) + } +} + +// Adds a trailing newline if necessary to the buffer +fn add_trailing_newline(buffer: &mut Vec) { + // If the line doesn't end with a newline, that indicates we hit a EOF. + // We add a newline so output from other tasks doesn't get written to the same + // line. + if buffer.last() != Some(&b'\n') { + buffer.push(b'\n'); + } +} diff --git a/crates/turborepo-process/src/child/shutdown.rs b/crates/turborepo-process/src/child/shutdown.rs new file mode 100644 index 0000000000000..5c872766964b6 --- /dev/null +++ b/crates/turborepo-process/src/child/shutdown.rs @@ -0,0 +1,241 @@ +use std::time::Duration; + +use tokio::sync::mpsc; +use tracing::debug; + +use super::{ChildCommand, ChildHandle}; + +#[cfg(unix)] +const PTY_PROCESS_GROUP_SIGINT_DELAY: Duration = Duration::from_secs(1); + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum ChildExit { + Finished(Option), + /// The child process exited during graceful shutdown. + Interrupted, + /// The child process was killed, it could either be explicitly killed or it + /// did not respond to an interrupt and was killed as a result + Killed, + /// The child process was killed by someone else. Note that on + /// windows, it is not possible to distinguish between whether + /// the process exited normally or was killed + KilledExternal, + Failed, +} + +#[derive(Debug, Clone, Copy)] +pub enum ShutdownStyle { + /// On Unix this sends SIGINT to the process group. On Windows, Turbo cannot + /// send a signal directly, so it waits for an externally delivered console + /// event or an explicit kill. + /// + /// `Graceful(Some(timeout))` escalates to `Kill` after `timeout` elapses. + /// `Graceful(None)` waits indefinitely until an explicit `Kill` command + /// arrives. + Graceful(Option), + + Kill, +} + +/// Child process stopped. +#[allow(dead_code)] +#[derive(Debug)] +pub struct ShutdownFailed; + +impl From for ShutdownFailed { + fn from(_: std::io::Error) -> Self { + ShutdownFailed + } +} + +impl ShutdownStyle { + /// Process the shutdown style for the given child process. + /// + /// If an exit channel is provided, the exit code will be sent to the + /// channel when the child process exits. + pub(super) async fn process( + &self, + child: &mut ChildHandle, + command_rx: &mut mpsc::Receiver, + ) -> ChildExit { + match self { + #[allow(unused)] + ShutdownStyle::Graceful(timeout) => { + // try ro run the command for the given timeout + #[cfg(unix)] + { + let Some(pid) = child.pid() else { + return ChildExit::Interrupted; + }; + + let pid = pid as libc::pid_t; + let mut process_group_interrupt_sent = child.send_graceful_interrupt(pid); + let process_group_interrupt_deadline = + tokio::time::Instant::now() + PTY_PROCESS_GROUP_SIGINT_DELAY; + debug!("waiting for child {}", pid); + + let deadline = timeout.map(|timeout| tokio::time::Instant::now() + timeout); + let mut command_rx_open = true; + + let exit = loop { + match deadline { + Some(deadline) => { + tokio::select! { + result = child.wait() => { + break match result { + Ok(_exit_code) => ChildExit::Interrupted, + Err(_) => ChildExit::Failed, + }; + } + _ = tokio::time::sleep_until(process_group_interrupt_deadline), if !process_group_interrupt_sent => { + child.send_fallback_graceful_interrupt(pid); + process_group_interrupt_sent = true; + } + command = command_rx.recv(), if command_rx_open => { + match command { + Some(ChildCommand::Kill) => { + debug!("graceful shutdown interrupted, killing child"); + break match child.kill().await { + Ok(_) => ChildExit::Killed, + Err(_) => ChildExit::Failed, + }; + } + Some(ChildCommand::Shutdown(_)) => {} + None => command_rx_open = false, + } + } + _ = tokio::time::sleep_until(deadline) => { + debug!("graceful shutdown timed out, killing child"); + break match child.kill().await { + Ok(_) => ChildExit::Killed, + Err(_) => ChildExit::Failed, + }; + } + } + } + None => { + tokio::select! { + result = child.wait() => { + break match result { + Ok(_exit_code) => ChildExit::Interrupted, + Err(_) => ChildExit::Failed, + }; + } + _ = tokio::time::sleep_until(process_group_interrupt_deadline), if !process_group_interrupt_sent => { + child.send_fallback_graceful_interrupt(pid); + process_group_interrupt_sent = true; + } + command = command_rx.recv(), if command_rx_open => { + match command { + Some(ChildCommand::Kill) => { + debug!("graceful shutdown interrupted, killing child"); + break match child.kill().await { + Ok(_) => ChildExit::Killed, + Err(_) => ChildExit::Failed, + }; + } + Some(ChildCommand::Shutdown(_)) => {} + None => command_rx_open = false, + } + } + } + } + } + }; + + if exit == ChildExit::Interrupted { + child + .wait_for_process_group_exit( + pid, + deadline, + command_rx, + &mut command_rx_open, + ) + .await + } else { + exit + } + } + + #[cfg(windows)] + { + // Windows consoles deliver Ctrl+C to attached child processes. + // Turbo can't send a targeted signal, so graceful shutdown + // waits for that external event when no timeout is provided. + let deadline = timeout.map(|timeout| tokio::time::Instant::now() + timeout); + let mut command_rx_open = true; + + let exit = loop { + match deadline { + Some(deadline) => { + tokio::select! { + result = child.wait() => { + break match result { + Ok(_exit_code) => ChildExit::Interrupted, + Err(_) => ChildExit::Failed, + }; + } + command = command_rx.recv(), if command_rx_open => { + match command { + Some(ChildCommand::Kill) => { + debug!("graceful shutdown interrupted, killing child"); + break match child.kill().await { + Ok(_) => ChildExit::Killed, + Err(_) => ChildExit::Failed, + }; + } + Some(ChildCommand::Shutdown(_)) => {} + None => command_rx_open = false, + } + } + _ = tokio::time::sleep_until(deadline) => { + debug!("graceful shutdown timed out, killing child"); + break match child.kill().await { + Ok(_) => ChildExit::Killed, + Err(_) => ChildExit::Failed, + }; + } + } + } + None => { + tokio::select! { + result = child.wait() => { + break match result { + Ok(_exit_code) => ChildExit::Interrupted, + Err(_) => ChildExit::Failed, + }; + } + command = command_rx.recv(), if command_rx_open => { + match command { + Some(ChildCommand::Kill) => { + debug!("graceful shutdown interrupted, killing child"); + break match child.kill().await { + Ok(_) => ChildExit::Killed, + Err(_) => ChildExit::Failed, + }; + } + Some(ChildCommand::Shutdown(_)) => {} + None => command_rx_open = false, + } + } + } + } + } + }; + + if exit == ChildExit::Interrupted { + child + .wait_for_job_exit(deadline, command_rx, &mut command_rx_open) + .await + } else { + exit + } + } + } + ShutdownStyle::Kill => match child.kill().await { + Ok(_) => ChildExit::Killed, + Err(_) => ChildExit::Failed, + }, + } + } +} diff --git a/crates/turborepo-process/src/child/state.rs b/crates/turborepo-process/src/child/state.rs new file mode 100644 index 0000000000000..1ace5f94d59ce --- /dev/null +++ b/crates/turborepo-process/src/child/state.rs @@ -0,0 +1,92 @@ +use std::io; + +use portable_pty::MasterPty as PtyController; +use tokio::sync::{mpsc, watch}; +use tracing::{debug, trace}; + +use super::{ChildExit, ChildHandle, ShutdownStyle}; + +/// The structure that holds logic regarding interacting with the underlying +/// child process +#[derive(Debug)] +pub(super) struct ChildStateManager { + pub(super) shutdown_style: ShutdownStyle, + pub(super) exit_tx: watch::Sender>, + pub(super) shutdown_initiated: bool, +} + +#[derive(Clone, Debug)] +pub(super) struct ChildCommandChannel(pub(super) mpsc::Sender); + +impl ChildCommandChannel { + pub(super) fn new() -> (Self, mpsc::Receiver) { + let (tx, rx) = mpsc::channel(1); + (ChildCommandChannel(tx), rx) + } + + pub async fn shutdown( + &self, + shutdown_style: ShutdownStyle, + ) -> Result<(), mpsc::error::SendError> { + self.0.send(ChildCommand::Shutdown(shutdown_style)).await + } + + pub async fn kill(&self) -> Result<(), mpsc::error::SendError> { + self.0.send(ChildCommand::Kill).await + } +} + +pub(super) enum ChildCommand { + Shutdown(ShutdownStyle), + Kill, +} + +impl ChildStateManager { + pub(super) async fn handle_child_command( + &self, + command: Option, + command_rx: &mut mpsc::Receiver, + child: &mut ChildHandle, + controller: Option>, + ) { + let exit = match command.unwrap_or(ChildCommand::Shutdown(self.shutdown_style)) { + ChildCommand::Shutdown(shutdown_style) => { + debug!("stopping child process"); + shutdown_style.process(child, command_rx).await + } + ChildCommand::Kill => { + debug!("killing child process"); + ShutdownStyle::Kill.process(child, command_rx).await + } + }; + // ignore the send error, failure means the channel is dropped + trace!("sending child exit after shutdown"); + self.exit_tx.send(Some(exit)).ok(); + drop(controller); + } + + pub(super) async fn handle_child_exit(&self, status: io::Result>) { + // If a shutdown was initiated we defer to the exit returned by + // `ShutdownStyle::process` as that will have information if the child + // responded to a SIGINT or a SIGKILL. The `wait` response this function + // gets in that scenario would make it appear that the child was killed by an + // external process. + if self.shutdown_initiated { + return; + } + + debug!("child process exited normally"); + // the child process exited + let child_exit = match status { + Ok(Some(c)) => ChildExit::Finished(Some(c)), + // if we hit this case, it means that the child process was killed + // by someone else, and we should report that it was killed + Ok(None) => ChildExit::KilledExternal, + Err(_e) => ChildExit::Failed, + }; + + // ignore the send error, the channel is dropped anyways + trace!("sending child exit"); + self.exit_tx.send(Some(child_exit)).ok(); + } +} diff --git a/crates/turborepo-process/src/child/test.rs b/crates/turborepo-process/src/child/test.rs new file mode 100644 index 0000000000000..f975aff8fc76a --- /dev/null +++ b/crates/turborepo-process/src/child/test.rs @@ -0,0 +1,1413 @@ +use std::{ + assert_matches, fs, io, + sync::{Arc, Mutex}, + time::Duration, +}; + +use futures::{StreamExt, stream::FuturesUnordered}; +use test_case::test_case; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + sync::oneshot, +}; +use tracing_test::traced_test; +use turbopath::AbsoluteSystemPathBuf; + +use super::{Child, ChildInput, ChildOutput, ChildStdin, Command}; +use crate::{ + PtySize, + child::{ChildExit, ShutdownStyle}, +}; + +const STARTUP_DELAY: Duration = Duration::from_millis(500); +// We skip testing PTY usage on Windows +const TEST_PTY: bool = !cfg!(windows); + +struct ObservedOutput { + buffer: Arc>>, + ready_tx: Option>, +} + +impl ObservedOutput { + fn new() -> (Self, Arc>>, oneshot::Receiver<()>) { + let buffer = Arc::new(Mutex::new(Vec::new())); + let (ready_tx, ready_rx) = oneshot::channel(); + ( + Self { + buffer: buffer.clone(), + ready_tx: Some(ready_tx), + }, + buffer, + ready_rx, + ) + } +} + +impl io::Write for ObservedOutput { + fn write(&mut self, buf: &[u8]) -> io::Result { + let saw_ready = { + let mut buffer = self.buffer.lock().unwrap(); + buffer.extend_from_slice(buf); + String::from_utf8_lossy(&buffer).contains("ready") + }; + + if saw_ready && let Some(ready_tx) = self.ready_tx.take() { + ready_tx.send(()).ok(); + } + + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +const EOT: char = '\u{4}'; + +fn find_script_dir() -> AbsoluteSystemPathBuf { + let cwd = AbsoluteSystemPathBuf::cwd().unwrap(); + let mut root = cwd; + while !root.join_component(".git").exists() { + root = root.parent().unwrap().to_owned(); + } + root.join_components(&["crates", "turborepo-process", "test", "scripts"]) +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_pid(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + assert_matches!(child.pid(), Some(_)); + child.stop().await; + + let exit = child.wait().await; + assert_matches!(exit, Some(ChildExit::Killed)); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tracing_test::traced_test] +#[tokio::test] +async fn test_wait(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + let exit1 = child.wait().await; + let exit2 = child.wait().await; + assert_matches!(exit1, Some(ChildExit::Finished(Some(0)))); + assert_matches!(exit2, Some(ChildExit::Finished(Some(0)))); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_spawn(use_pty: bool) { + let cmd = { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd + }; + + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + assert!(child.is_running()); + + let code = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("child wait should not hang after process exit"); + assert_eq!(code, Some(ChildExit::Finished(Some(0)))); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_stdout(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + { + let mut output = Vec::new(); + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => { + stdout + .read_to_end(&mut output) + .await + .expect("Failed to read stdout"); + } + ChildOutput::Pty(mut outputs) => { + outputs + .read_to_end(&mut output) + .expect("failed to read stdout"); + } + }; + + let output_str = String::from_utf8(output).expect("Failed to parse stdout"); + let trimmed_output = output_str.trim(); + let trimmed_output = trimmed_output.strip_prefix(EOT).unwrap_or(trimmed_output); + + assert_eq!(trimmed_output, "hello world"); + } + + let exit = child.wait().await; + + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_stdio(use_pty: bool) { + let script = find_script_dir().join_component("stdin_stdout.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let input = "hello world"; + // drop stdin to close the pipe + { + match child.stdin_inner().unwrap() { + ChildInput::Std(mut stdin) => stdin.write_all(input.as_bytes()).await.unwrap(), + ChildInput::Pty(mut stdin) => stdin.write_all(input.as_bytes()).unwrap(), + } + } + + let mut output = Vec::new(); + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => stdout.read_to_end(&mut output).await.unwrap(), + ChildOutput::Pty(mut stdout) => stdout.read_to_end(&mut output).unwrap(), + }; + + let output_str = String::from_utf8(output).expect("Failed to parse stdout"); + let trimmed_out = output_str.trim(); + let trimmed_out = trimmed_out.strip_prefix(EOT).unwrap_or(trimmed_out); + + assert!(trimmed_out.contains(input), "got: {trimmed_out}"); + + let exit = child.wait().await; + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +/// Regression test for #7834: proves that a child process can block +/// before producing any output if stdin is an open pipe instead of EOF. +/// +/// This models the v1.13 regression on Windows stream mode: +/// `tsx watch` received an open piped stdin and never started executing. +#[tokio::test] +async fn test_std_open_stdin_blocks_startup_until_eof() { + let script = find_script_dir().join_component("startup_after_stdin_eof.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let ChildOutput::Std { mut stdout, .. } = child.outputs().unwrap() else { + panic!("expected stdio child"); + }; + + let mut output = Vec::new(); + let result = + tokio::time::timeout(Duration::from_secs(1), stdout.read_to_end(&mut output)).await; + assert!( + result.is_err(), + "child should stay blocked while stdin is held open" + ); + assert!( + output.is_empty(), + "child should not produce output before stdin reaches EOF" + ); + + // Closing the parent's stdin pipe should unblock the child immediately. + drop(child.stdin_inner()); + + tokio::time::timeout(Duration::from_secs(5), stdout.read_to_end(&mut output)) + .await + .expect("child should finish reading after stdin is closed") + .expect("failed to read child output"); + + let exit = tokio::time::timeout(Duration::from_secs(5), child.wait()) + .await + .expect("child should exit after stdin is closed"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); + + let output = String::from_utf8(output).unwrap().replace("\r\n", "\n"); + assert_eq!(output, "stdin bytes=0\nstarted\n"); +} + +/// Regression test for #7834: verifies the pre-v1.13 behavior where tasks +/// that do not need input start immediately when stdin is already at EOF. +#[tokio::test] +async fn test_std_null_stdin_allows_startup() { + let script = find_script_dir().join_component("startup_after_stdin_eof.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); + + let mut output = Vec::new(); + let exit = tokio::time::timeout( + Duration::from_secs(5), + child.wait_with_piped_outputs(&mut output), + ) + .await + .expect("child should not block when stdin is null") + .expect("failed to wait for child output"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); + + let output = String::from_utf8(output).unwrap().replace("\r\n", "\n"); + assert_eq!(output, "stdin bytes=0\nstarted\n"); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_graceful_shutdown_timeout(use_pty: bool) { + let cmd = { + let script = find_script_dir().join_component("sleep_5_ignore.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd + }; + + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), + use_pty.then(PtySize::default), + ) + .unwrap(); + + let mut buf = vec![0; 4]; + // wait for the process to print "here" + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => { + stdout.read_exact(&mut buf).await.unwrap(); + } + ChildOutput::Pty(mut stdout) => { + stdout.read_exact(&mut buf).unwrap(); + } + }; + child.stop().await; + + let exit = child.wait().await; + // this should time out and be killed + assert_matches!(exit, Some(ChildExit::Killed)); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_graceful_shutdown(use_pty: bool) { + let cmd = { + let script = find_script_dir().join_component("sleep_5_interruptable.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd + }; + + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), + use_pty.then(PtySize::default), + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + // We need to read the child output otherwise the child will be unable to + // cleanly shut down as it waits for the receiving end of the PTY to read + // the output before exiting. + let mut output_child = child.clone(); + tokio::task::spawn(async move { + let mut output = Vec::new(); + output_child.wait_with_piped_outputs(&mut output).await.ok(); + }); + + child.stop().await; + let exit = child.wait().await; + + if cfg!(windows) { + assert_matches!(exit, Some(ChildExit::Killed)); + } else { + assert_matches!(exit, Some(ChildExit::Interrupted)); + } +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_graceful_shutdown_drains_final_output(use_pty: bool) { + let script = find_script_dir().join_component("graceful_sigint_output.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), + use_pty.then(PtySize::default), + ) + .unwrap(); + + let mut output_child = child.clone(); + let (mut observer, output, ready_rx) = ObservedOutput::new(); + let output_task = tokio::spawn(async move { + output_child + .wait_with_piped_outputs(&mut observer) + .await + .unwrap() + }); + + tokio::time::timeout(Duration::from_secs(2), ready_rx) + .await + .expect("timed out waiting for startup output") + .expect("ready notification channel closed unexpectedly"); + child.set_closing(); + child.stop().await; + let exit = output_task.await.unwrap(); + let output = String::from_utf8(output.lock().unwrap().clone()).unwrap(); + + assert!(output.contains("ready"), "missing startup output: {output}"); + + if cfg!(windows) { + assert_matches!(exit, Some(ChildExit::Killed)); + } else { + assert!( + output.contains("received SIGINT"), + "missing SIGINT receipt log: {output}" + ); + assert!( + output.contains("exiting after SIGINT"), + "missing SIGINT exit log: {output}" + ); + assert_matches!(exit, Some(ChildExit::Interrupted)); + } +} + +// Regression test: a wrapper process (simulating npm/pnpm) forwards SIGINT +// to its child. When turbo sends SIGINT to the process group, the child +// gets it twice — once from the group signal, once from the wrapper. +// For PTY children we now signal only the direct PID to avoid this. +#[cfg(unix)] +#[tokio::test] +#[traced_test] +async fn test_pty_child_receives_single_sigint() { + let script = find_script_dir().join_component("wrapper_count_sigints.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(2000))), + Some(PtySize::default()), + ) + .unwrap(); + + let mut output_child = child.clone(); + let (mut observer, output, ready_rx) = ObservedOutput::new(); + let output_task = tokio::spawn(async move { + output_child + .wait_with_piped_outputs(&mut observer) + .await + .unwrap() + }); + + tokio::time::timeout(Duration::from_secs(5), ready_rx) + .await + .expect("timed out waiting for ready") + .expect("ready channel closed"); + + child.set_closing(); + child.stop().await; + output_task.await.unwrap(); + + let output = String::from_utf8(output.lock().unwrap().clone()).unwrap(); + assert!( + output.contains("SIGINT_COUNT=1"), + "expected exactly one SIGINT, got output: {output}" + ); + assert!( + !output.contains("SIGINT_COUNT=2"), + "child received SIGINT twice: {output}" + ); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_detect_killed_someone_else(use_pty: bool) { + let cmd = { + let script = find_script_dir().join_component("sleep_5_interruptable.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd + }; + + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), + use_pty.then(PtySize::default), + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + #[cfg(unix)] + if let Some(pid) = child.pid() { + unsafe { + libc::kill(pid as i32, libc::SIGINT); + } + } + #[cfg(windows)] + if let Some(pid) = child.pid() { + unsafe { + println!("killing"); + windows_sys::Win32::System::Threading::TerminateProcess( + windows_sys::Win32::System::Threading::OpenProcess( + windows_sys::Win32::System::Threading::PROCESS_TERMINATE, + 0, + pid, + ), + 3, + ); + } + } + + let exit = child.wait().await; + + #[cfg(unix)] + assert_matches!(exit, Some(ChildExit::KilledExternal)); + #[cfg(not(unix))] + assert_matches!(exit, Some(ChildExit::Finished(Some(3)))); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_wait_with_output(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + let mut out = Vec::new(); + + let exit = child.wait_with_piped_outputs(&mut out).await.unwrap(); + + let out = String::from_utf8(out).unwrap(); + let trimmed_out = out.trim(); + let trimmed_out = trimmed_out.strip_prefix(EOT).unwrap_or(trimmed_out); + + assert_eq!(trimmed_out, "hello world"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_wait_with_single_output(use_pty: bool) { + let script = find_script_dir().join_component("hello_world_hello_moon.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + let mut buffer = Vec::new(); + + let exit = child.wait_with_piped_outputs(&mut buffer).await.unwrap(); + + let output = String::from_utf8(buffer).unwrap(); + + // There are no ordering guarantees so we just check that both logs made it + let expected_stdout = "hello world"; + let expected_stderr = "hello moon"; + assert!(output.contains(expected_stdout), "got: {output}"); + assert!(output.contains(expected_stderr), "got: {output}"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_wait_with_with_non_utf8_output(use_pty: bool) { + let script = find_script_dir().join_component("hello_non_utf8.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + let mut out = Vec::new(); + + let exit = child.wait_with_piped_outputs(&mut out).await.unwrap(); + + let expected = &[0, 159, 146, 150]; + let trimmed_out = out.trim_ascii(); + let trimmed_out = trimmed_out.strip_prefix(&[4]).unwrap_or(trimmed_out); + assert_eq!(trimmed_out, expected); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_no_newline(use_pty: bool) { + let script = find_script_dir().join_component("hello_no_line.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + let mut out = Vec::new(); + + let exit = child.wait_with_piped_outputs(&mut out).await.unwrap(); + + let output = String::from_utf8(out).unwrap(); + let trimmed_out = output.trim(); + let trimmed_out = trimmed_out.strip_prefix(EOT).unwrap_or(trimmed_out); + assert!( + output.ends_with('\n'), + "expected newline to be added: {output}" + ); + assert_eq!(trimmed_out, "look ma, no newline!"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +#[cfg(unix)] +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_kill_process_group(use_pty: bool) { + let mut cmd = Command::new("sh"); + cmd.args(["-c", "while true; do sleep 0.2; done"]); + let mut child = Child::spawn( + cmd, + // Bumping this to give ample time for the process to respond to the SIGINT to reduce + // flakiness inherent with sending and receiving signals. + ShutdownStyle::Graceful(Some(Duration::from_millis(1000))), + use_pty.then(PtySize::default), + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + // We need to read the child output otherwise the child will be unable to + // cleanly shut down as it waits for the receiving end of the PTY to read + // the output before exiting. + let mut output_child = child.clone(); + tokio::task::spawn(async move { + let mut output = Vec::new(); + output_child.wait_with_piped_outputs(&mut output).await.ok(); + }); + + let exit = child.stop().await; + + // On Unix, shell scripts may not respond to SIGINT and will timeout, + // resulting in being killed rather than interrupted. + if cfg!(unix) { + assert_matches!(exit, Some(ChildExit::Killed) | Some(ChildExit::Interrupted)); + } else { + assert_matches!(exit, Some(ChildExit::Interrupted)); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn test_orphan_process() { + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo hello; exec sleep 120"]); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let child_pid = child.pid().unwrap() as i32; + // We don't kill the process group to simulate what an external program might do + unsafe { + libc::kill(child_pid, libc::SIGKILL); + } + + let exit = child.wait().await; + assert_matches!(exit, Some(ChildExit::KilledExternal)); + + let mut output = Vec::new(); + match tokio::time::timeout( + Duration::from_millis(500), + child.wait_with_piped_outputs(&mut output), + ) + .await + { + Ok(exit_status) => { + assert_matches!(exit_status, Ok(Some(ChildExit::KilledExternal))); + } + Err(_) => panic!("expected wait_with_piped_outputs to exit after it was killed"), + } +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_graceful_shutdown_waits_for_force_kill(use_pty: bool) { + let script = find_script_dir().join_component("sleep_5_ignore.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_secs(5))), + use_pty.then(PtySize::default), + ) + .unwrap(); + + let mut buf = vec![0; 4]; + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => { + stdout.read_exact(&mut buf).await.unwrap(); + } + ChildOutput::Pty(mut stdout) => { + stdout.read_exact(&mut buf).unwrap(); + } + }; + + let mut shutdown_child = child.clone(); + let shutdown = + tokio::spawn(async move { shutdown_child.shutdown(ShutdownStyle::Graceful(None)).await }); + + tokio::time::sleep(Duration::from_millis(200)).await; + assert!( + !shutdown.is_finished(), + "graceful shutdown should keep waiting until explicitly forced" + ); + + assert_eq!(child.kill().await, Some(ChildExit::Killed)); + assert_eq!(shutdown.await.unwrap(), Some(ChildExit::Killed)); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_multistop(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + let child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + let mut stops = FuturesUnordered::new(); + for _ in 1..10 { + let mut child = child.clone(); + stops.push(async move { + child.stop().await; + }); + } + + while tokio::time::timeout(Duration::from_secs(5), stops.next()) + .await + .expect("timed out") + .is_some() + {} +} + +// Regression tests for https://github.com/vercel/turborepo/issues/11808 +// +// On Windows, portable-pty 0.9.0 added PSEUDOCONSOLE_INHERIT_CURSOR to +// ConPTY creation, which requires the host to handle DSR (Device Status +// Report) escape sequences. Turborepo doesn't, causing ConPTY to hang. +// +// Additionally, an unconditional `drop(stdin)` in the PTY path of +// wait_with_piped_outputs would kill ConPTY children on Windows because +// closing ConPTY stdin terminates the session. +// +// These tests verify the fixes: PTY children start, produce output, and +// exit normally without hanging or being killed by stdin closure. + +/// Verifies that a PTY-spawned short-lived process produces output and +/// exits cleanly via wait_with_piped_outputs. Uses a timeout to catch +/// the ConPTY hang that occurred with PSEUDOCONSOLE_INHERIT_CURSOR. +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_pty_child_does_not_hang(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + let mut out = Vec::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + child.wait_with_piped_outputs(&mut out), + ) + .await; + + let exit = result + .expect("PTY child hung — likely PSEUDOCONSOLE_INHERIT_CURSOR regression") + .unwrap(); + + let output = String::from_utf8(out).unwrap(); + let trimmed = output.trim().strip_prefix(EOT).unwrap_or(output.trim()); + assert_eq!(trimmed, "hello world"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +/// Simulates the persistent-task flow: stdin is taken by the caller +/// (as the TUI does for interactive tasks) BEFORE wait_with_piped_outputs +/// is called. The child should still produce output and exit normally +/// without wait_with_piped_outputs interfering with stdin. +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_pty_stdin_taken_before_piped_outputs(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + // Take stdin before piping outputs, simulating TUI taking ownership. + // For PTY children, this returns Some; for non-PTY, stdin() returns None + // (Std variant is filtered out), but stdin_inner still removes it. + let _stdin_guard = child.stdin(); + + // Verify stdin_inner is now empty (already taken). + assert!( + child.stdin_inner().is_none(), + "stdin should already be taken" + ); + + let mut out = Vec::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + child.wait_with_piped_outputs(&mut out), + ) + .await; + + let exit = result + .expect("child hung — wait_with_piped_outputs likely interfered with taken stdin") + .unwrap(); + + let output = String::from_utf8(out).unwrap(); + let trimmed = output.trim().strip_prefix(EOT).unwrap_or(output.trim()); + assert_eq!(trimmed, "hello world"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +/// Verifies that a PTY-spawned process with open stdin that has NOT been +/// taken by the caller still completes normally. This is the non-persistent +/// task path where exec.rs does not take stdin before +/// wait_with_piped_outputs. +/// +/// Before the fix, on Windows the unconditional stdin drop inside +/// wait_with_piped_outputs would kill the ConPTY child immediately. +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_pty_untaken_stdin_does_not_kill_child(use_pty: bool) { + let script = find_script_dir().join_component("hello_world.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + // Do NOT take stdin — this simulates a non-persistent task where + // exec.rs skips stdin handling on Windows (closing_stdin_ends_process). + // wait_with_piped_outputs should still work without killing the child. + let mut out = Vec::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + child.wait_with_piped_outputs(&mut out), + ) + .await; + + let exit = result + .expect("child process hung or was killed by premature stdin closure") + .unwrap(); + + let output = String::from_utf8(out).unwrap(); + let trimmed = output.trim().strip_prefix(EOT).unwrap_or(output.trim()); + assert_eq!(trimmed, "hello world"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +/// Regression test for #12393: proves that dropping stdin causes a +/// persistent-style child (one that exits on stdin EOF) to terminate. +/// +/// This documents the mechanism behind the bug: when the task executor +/// took stdin and passed it to `TaskOutput::set_stdin()` in stream mode, +/// the stdin was dropped immediately, sending EOF to the child. +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_dropping_stdin_terminates_persistent_child(use_pty: bool) { + let script = find_script_dir().join_component("persistent_server.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + // Take stdin and immediately drop it — simulates the bug where + // TaskOutput::stream().set_stdin() dropped stdin in stream mode. + { + let _dropped = child.stdin(); + } + + // The child should exit because it received EOF on stdin. + let mut out = Vec::new(); + let result = tokio::time::timeout( + Duration::from_secs(5), + child.wait_with_piped_outputs(&mut out), + ) + .await; + + let exit = result + .expect("child should have exited after stdin was dropped") + .unwrap(); + + let output = String::from_utf8(out).unwrap(); + assert!( + output.contains("server ready"), + "expected 'server ready' in output, got: {output:?}" + ); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +/// Regression test for #12393: proves that holding stdin in a guard +/// keeps a persistent-style child alive. +/// +/// This is the correct behavior after the fix: in stream mode, stdin +/// is held by `_stdin_guard` instead of being passed to +/// `TaskOutput::set_stdin()` which would drop it. +/// +/// This covers the writable PTY path used for interactive input. +#[tokio::test] +async fn test_held_stdin_keeps_persistent_child_alive() { + if !TEST_PTY { + return; + } + let script = find_script_dir().join_component("persistent_server.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, Some(PtySize::default())).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + // Hold stdin in a guard — simulates the correct persistent task flow. + let _stdin_guard = child.stdin(); + assert!( + _stdin_guard.is_some(), + "PTY child should return Some from stdin()" + ); + + // The child should NOT exit while we hold stdin. Give it a moment + // and verify it's still alive by checking that wait times out. + let result = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + assert!( + result.is_err(), + "child should still be alive while stdin is held" + ); + + // Now drop the guard — child should exit. + drop(_stdin_guard); + + let exit = tokio::time::timeout(Duration::from_secs(5), child.wait()) + .await + .expect("child should exit after stdin guard is dropped"); + + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +#[tokio::test] +async fn test_non_pty_stdin_guard_keeps_persistent_child_alive() { + let script = find_script_dir().join_component("persistent_server.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, None).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let stdin_guard = child.take_stdin(); + assert!( + matches!(stdin_guard, Some(ChildStdin::Guard(_))), + "non-PTY child should return a stdin guard" + ); + + let result = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + assert!( + result.is_err(), + "child should still be alive while stdin is held" + ); + + drop(stdin_guard); + + let exit = tokio::time::timeout(Duration::from_secs(5), child.wait()) + .await + .expect("child should exit after stdin guard is dropped"); + assert_matches!(exit, Some(ChildExit::Finished(Some(0)))); +} + +/// Verifies that stopping a parent process also kills its child processes. +/// +/// On Unix this works via process groups (setpgid + kill(-pgid)). +/// On Windows this works via Job Objects +/// (JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE). +/// +/// The test spawns a Node.js script that itself spawns a long-running child +/// process, captures the grandchild's PID from stdout, stops the parent, +/// and then checks that the grandchild is no longer alive. +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +#[traced_test] +async fn test_process_tree_cleanup(use_pty: bool) { + let script = find_script_dir().join_component("spawn_child_sleep.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(500))), + use_pty.then(PtySize::default), + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + // Read stdout to get the grandchild PID + let grandchild_pid = { + let mut out = Vec::new(); + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => { + let mut buf = vec![0u8; 256]; + let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) + .await + .expect("timed out reading grandchild PID") + .expect("failed to read stdout"); + out.extend_from_slice(&buf[..n]); + } + ChildOutput::Pty(mut reader) => { + let mut buf = vec![0u8; 256]; + let n = reader.read(&mut buf).expect("failed to read pty output"); + out.extend_from_slice(&buf[..n]); + } + }; + let output = String::from_utf8(out).unwrap(); + let pid_line = output + .lines() + .find(|line| line.contains("CHILD_PID=")) + .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); + pid_line + .split('=') + .nth(1) + .unwrap() + .trim() + .parse::() + .unwrap() + }; + + // Verify grandchild is alive before we stop + assert!( + is_process_alive(grandchild_pid), + "grandchild process {grandchild_pid} should be alive before stop" + ); + + // Stop the parent process + child.stop().await; + + // Give the OS a moment to clean up + tokio::time::sleep(Duration::from_millis(500)).await; + + // Verify grandchild is dead + assert!( + !is_process_alive(grandchild_pid), + "grandchild process {grandchild_pid} should have been killed" + ); +} + +#[test_case(false)] +#[test_case(TEST_PTY)] +#[tokio::test] +async fn test_force_kill_process_tree_cleanup(use_pty: bool) { + let script = find_script_dir().join_component("spawn_child_sleep.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn(cmd, ShutdownStyle::Kill, use_pty.then(PtySize::default)).unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let grandchild_pid = { + let mut out = Vec::new(); + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => { + let mut buf = vec![0u8; 256]; + let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) + .await + .expect("timed out reading grandchild PID") + .expect("failed to read stdout"); + out.extend_from_slice(&buf[..n]); + } + ChildOutput::Pty(mut reader) => { + let mut buf = vec![0u8; 256]; + let n = reader.read(&mut buf).expect("failed to read pty output"); + out.extend_from_slice(&buf[..n]); + } + }; + let output = String::from_utf8(out).unwrap(); + let pid_line = output + .lines() + .find(|line| line.contains("CHILD_PID=")) + .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); + pid_line + .split('=') + .nth(1) + .unwrap() + .trim() + .parse::() + .unwrap() + }; + + assert!( + is_process_alive(grandchild_pid), + "grandchild process {grandchild_pid} should be alive before force kill" + ); + + assert_eq!(child.kill().await, Some(ChildExit::Killed)); + tokio::time::sleep(Duration::from_millis(200)).await; + + assert!( + !is_process_alive(grandchild_pid), + "grandchild process {grandchild_pid} should have been force killed" + ); +} + +// Regression tests for the pre_exec/setsid -> process_group(0) migration. +// +// We replaced an unsafe pre_exec callback that called setsid() with tokio's +// safe process_group(0) API. These tests verify the critical invariants: +// +// 1. The child gets its own process group (PGID == child PID, not parent's) +// 2. Grandchildren inherit the child's process group +// 3. kill(-pgid, SIGINT) reaches both child and grandchild +// 4. The child is NOT a session leader (regression guard against setsid) + +#[cfg(unix)] +#[tokio::test] +async fn test_child_has_own_process_group() { + let script = find_script_dir().join_component("sleep_5_interruptable.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(500))), + None, + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let child_pid = child.pid().expect("child should have a pid") as libc::pid_t; + let child_pgid = unsafe { libc::getpgid(child_pid) }; + let parent_pgid = unsafe { libc::getpgid(0) }; + + // process_group(0) should make the child's PGID equal its own PID + assert_eq!( + child_pgid, child_pid, + "child PGID ({child_pgid}) should equal child PID ({child_pid})" + ); + + // The child's process group must differ from the parent's + assert_ne!( + child_pgid, parent_pgid, + "child PGID ({child_pgid}) must differ from parent PGID ({parent_pgid})" + ); + + child.stop().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_grandchild_inherits_child_process_group() { + let script = find_script_dir().join_component("spawn_child_sleep.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(500))), + None, + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let child_pid = child.pid().expect("child should have a pid") as libc::pid_t; + + // Read the grandchild PID from stdout + let grandchild_pid = { + let mut out = Vec::new(); + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => { + let mut buf = vec![0u8; 256]; + let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) + .await + .expect("timed out reading grandchild PID") + .expect("failed to read stdout"); + out.extend_from_slice(&buf[..n]); + } + ChildOutput::Pty(_) => unreachable!("test uses non-PTY mode"), + }; + let output = String::from_utf8(out).unwrap(); + let pid_line = output + .lines() + .find(|line| line.contains("CHILD_PID=")) + .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); + pid_line + .split('=') + .nth(1) + .unwrap() + .trim() + .parse::() + .unwrap() + }; + + let child_pgid = unsafe { libc::getpgid(child_pid) }; + let grandchild_pgid = unsafe { libc::getpgid(grandchild_pid) }; + + // Grandchild should be in the same process group as the child + assert_eq!( + grandchild_pgid, child_pgid, + "grandchild PGID ({grandchild_pgid}) should match child PGID ({child_pgid})" + ); + + // Both should use child_pid as the group ID + assert_eq!( + child_pgid, child_pid, + "process group ID ({child_pgid}) should equal child PID ({child_pid})" + ); + + child.stop().await; + // Give OS time to clean up + tokio::time::sleep(Duration::from_millis(200)).await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_pty_graceful_shutdown_signals_process_group() { + let marker_file = std::env::temp_dir().join(format!( + "turbo-pty-process-group-sigint-{}", + std::process::id() + )); + let ready_file = std::env::temp_dir().join(format!( + "turbo-pty-process-group-ready-{}", + std::process::id() + )); + let _ = fs::remove_file(&marker_file); + let _ = fs::remove_file(&ready_file); + let marker_file = marker_file.to_string_lossy().into_owned(); + let ready_file = ready_file.to_string_lossy().into_owned(); + + let script = r#" +const { spawn } = require("child_process"); +process.on("SIGINT", () => {}); +const child = spawn(process.execPath, [ + "-e", + "process.on('SIGINT', () => { require('fs').writeFileSync(process.argv[1], 'interrupted'); process.exit(0); }); require('fs').writeFileSync(process.argv[2], 'ready'); setInterval(() => {}, 1000);", + process.argv[1], + process.argv[2], +], { stdio: "inherit" }); +child.on("exit", () => process.exit(0)); +"#; + let mut cmd = Command::new("node"); + cmd.args(["-e", script, marker_file.as_str(), ready_file.as_str()]); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_secs(2))), + Some(PtySize::default()), + ) + .unwrap(); + + for _ in 0..50 { + if fs::metadata(&ready_file).is_ok() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!( + fs::metadata(&ready_file).is_ok(), + "node child should become ready before shutdown" + ); + + let exit = child.stop().await; + + assert_eq!(exit, Some(ChildExit::Interrupted)); + assert!( + fs::metadata(&marker_file).is_ok(), + "node child should receive SIGINT from PTY process-group shutdown" + ); + + let _ = fs::remove_file(marker_file); + let _ = fs::remove_file(ready_file); +} + +#[cfg(unix)] +#[tokio::test] +async fn test_sigint_to_process_group_reaches_grandchild() { + let script = find_script_dir().join_component("spawn_child_sleep.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + cmd.open_stdin(); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(2000))), + None, + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let child_pid = child.pid().expect("child should have a pid"); + + // Read the grandchild PID + let grandchild_pid = { + let mut out = Vec::new(); + match child.outputs().unwrap() { + ChildOutput::Std { mut stdout, .. } => { + let mut buf = vec![0u8; 256]; + let n = tokio::time::timeout(Duration::from_secs(5), stdout.read(&mut buf)) + .await + .expect("timed out reading grandchild PID") + .expect("failed to read stdout"); + out.extend_from_slice(&buf[..n]); + } + ChildOutput::Pty(_) => unreachable!("test uses non-PTY mode"), + }; + let output = String::from_utf8(out).unwrap(); + let pid_line = output + .lines() + .find(|line| line.contains("CHILD_PID=")) + .unwrap_or_else(|| panic!("CHILD_PID not found in output: {output}")); + pid_line + .split('=') + .nth(1) + .unwrap() + .trim() + .parse::() + .unwrap() + }; + + assert!( + is_process_alive(grandchild_pid), + "grandchild should be alive before signal" + ); + + // Send SIGINT to the process group (negative PID), exactly as + // ShutdownStyle::Graceful does in production code + let pgid = -(child_pid as i32); + unsafe { + libc::kill(pgid, libc::SIGINT); + } + + // Wait for processes to die + tokio::time::sleep(Duration::from_millis(500)).await; + + assert!( + !is_process_alive(grandchild_pid), + "grandchild should be dead after SIGINT to process group" + ); + + // Consume the exit + child.wait().await; +} + +// Guard against accidentally reverting to setsid(). With process_group(0), +// the child calls setpgid(0, 0) which creates a new process group but does +// NOT create a new session. If someone reintroduces setsid(), the child's +// SID would equal its PID. With setpgid, the SID is inherited from the +// parent. +#[cfg(unix)] +#[tokio::test] +async fn test_child_is_not_session_leader() { + let script = find_script_dir().join_component("sleep_5_interruptable.js"); + let mut cmd = Command::new("node"); + cmd.args([script.as_std_path()]); + let mut child = Child::spawn( + cmd, + ShutdownStyle::Graceful(Some(Duration::from_millis(500))), + None, + ) + .unwrap(); + + tokio::time::sleep(STARTUP_DELAY).await; + + let child_pid = child.pid().expect("child should have a pid") as libc::pid_t; + let child_sid = unsafe { libc::getsid(child_pid) }; + let parent_sid = unsafe { libc::getsid(0) }; + + // With process_group(0), the child inherits the parent's session. + // If setsid() were used instead, child_sid would equal child_pid. + assert_ne!( + child_sid, child_pid, + "child SID ({child_sid}) should NOT equal child PID ({child_pid}) — that would mean \ + setsid() was called" + ); + assert_eq!( + child_sid, parent_sid, + "child SID ({child_sid}) should equal parent SID ({parent_sid})" + ); + + child.stop().await; +} + +fn is_process_alive(pid: u32) -> bool { + #[cfg(unix)] + { + // kill(pid, 0) checks if process exists without sending a signal + unsafe { libc::kill(pid as i32, 0) == 0 } + } + #[cfg(windows)] + { + use windows_sys::Win32::{ + Foundation::CloseHandle, + System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}, + }; + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return false; + } + // Process handle opened — check if it's actually still running + let mut exit_code: u32 = 0; + let result = + windows_sys::Win32::System::Threading::GetExitCodeProcess(handle, &mut exit_code); + CloseHandle(handle); + // STILL_ACTIVE (259) means the process is still running + result != 0 && exit_code == 259 + } + } +} diff --git a/crates/turborepo-process/src/child/test_guard.rs b/crates/turborepo-process/src/child/test_guard.rs new file mode 100644 index 0000000000000..37135792a7aaf --- /dev/null +++ b/crates/turborepo-process/src/child/test_guard.rs @@ -0,0 +1,80 @@ +use std::{ + io, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +// The atomic covers `cargo test`'s in-process parallelism; flock covers +// nextest's process-per-test parallelism. +#[cfg(test)] +static PTY_TEST_LOCK: AtomicBool = AtomicBool::new(false); + +#[cfg(test)] +#[derive(Debug)] +pub(super) struct PtyTestGuard { + #[cfg(unix)] + file: std::fs::File, +} + +#[cfg(test)] +impl PtyTestGuard { + pub(super) fn acquire() -> Self { + while PTY_TEST_LOCK + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + std::thread::sleep(Duration::from_millis(10)); + } + + #[cfg(unix)] + { + use std::{fs::OpenOptions, os::fd::AsRawFd}; + + let path = std::env::temp_dir().join("turborepo-process-pty.lock"); + let file = match OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(path) + { + Ok(file) => file, + Err(err) => { + PTY_TEST_LOCK.store(false, Ordering::Release); + panic!("failed to open PTY test lock: {err}"); + } + }; + let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) }; + if result != 0 { + PTY_TEST_LOCK.store(false, Ordering::Release); + panic!( + "failed to lock PTY test lock: {}", + io::Error::last_os_error() + ); + } + Self { file } + } + + #[cfg(not(unix))] + { + Self {} + } + } +} + +#[cfg(all(test, unix))] +impl Drop for PtyTestGuard { + fn drop(&mut self) { + use std::os::fd::AsRawFd; + + let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; + PTY_TEST_LOCK.store(false, Ordering::Release); + } +} + +#[cfg(all(test, not(unix)))] +impl Drop for PtyTestGuard { + fn drop(&mut self) { + PTY_TEST_LOCK.store(false, Ordering::Release); + } +}