From b8a0eb77aac18a19e25db304e611dd89a002a797 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 11 Jun 2026 00:58:41 -0400 Subject: [PATCH 1/4] fix: debug output capturing for TUI / panics --- crates/mesh-llm-events/src/lib.rs | 9 ++ .../mesh-llm-host-runtime/src/runtime/mod.rs | 115 +++++++++++++++--- crates/mesh-llm-tui/src/output/mod.rs | 107 ++++++++++++++-- crates/mesh-llm/src/lib.rs | 31 ++++- 4 files changed, 235 insertions(+), 27 deletions(-) diff --git a/crates/mesh-llm-events/src/lib.rs b/crates/mesh-llm-events/src/lib.rs index b3a3502c5c..a9056665ee 100644 --- a/crates/mesh-llm-events/src/lib.rs +++ b/crates/mesh-llm-events/src/lib.rs @@ -364,6 +364,7 @@ pub enum OutputLevel { Info, Warn, Error, + Fatal, } impl OutputLevel { @@ -373,6 +374,7 @@ impl OutputLevel { OutputLevel::Info => "info", OutputLevel::Warn => "warn", OutputLevel::Error => "error", + OutputLevel::Fatal => "fatal", } } } @@ -559,6 +561,10 @@ pub enum OutputEvent { message: String, context: Option, }, + Fatal { + message: String, + context: Option, + }, ShutdownRequested { signal: &'static str, }, @@ -611,6 +617,7 @@ impl OutputEvent { OutputEvent::RequestRouted { .. } => "request_routed", OutputEvent::Warning { .. } => "warning", OutputEvent::Error { .. } => "error", + OutputEvent::Fatal { .. } => "fatal", OutputEvent::ShutdownRequested { signal } => signal, OutputEvent::Shutdown { .. } => "shutdown", OutputEvent::LlamaNativeLog { category, .. } => category, @@ -625,6 +632,7 @@ impl OutputEvent { OutputEvent::LlamaNativeLog { .. } => OutputLevel::Debug, OutputEvent::Warning { .. } => OutputLevel::Warn, OutputEvent::Error { .. } => OutputLevel::Error, + OutputEvent::Fatal { .. } => OutputLevel::Fatal, _ => OutputLevel::Info, } } @@ -780,6 +788,7 @@ impl OutputEvent { } OutputEvent::Warning { message, .. } => message.clone(), OutputEvent::Error { message, .. } => message.clone(), + OutputEvent::Fatal { message, .. } => message.clone(), OutputEvent::ShutdownRequested { signal } => format!("shutdown requested ({signal})"), OutputEvent::Shutdown { reason } => reason .clone() diff --git a/crates/mesh-llm-host-runtime/src/runtime/mod.rs b/crates/mesh-llm-host-runtime/src/runtime/mod.rs index 3dbea27134..a9bbcba929 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/mod.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/mod.rs @@ -286,26 +286,85 @@ impl MeshTracingStderrWriter { } routing.set(true); - let event = match self.level { - tracing::Level::ERROR => OutputEvent::Error { - message: message.clone(), - context: Some("stderr".to_string()), - }, - tracing::Level::WARN => OutputEvent::Warning { - message: message.clone(), - context: Some("stderr".to_string()), - }, - _ => OutputEvent::Info { - message: message.clone(), - context: Some("stderr".to_string()), - }, - }; + let dashboard_message = strip_ansi_escape_sequences(&message); + let event = self.dashboard_event_for_message(&dashboard_message); let result = mesh_llm_events::emit_event(event).or_else(|_| write_stderr_line(&message)); routing.set(false); result }) } + + fn dashboard_event_for_message(&self, message: &str) -> OutputEvent { + let (message, context) = normalize_tracing_message(&self.target, message); + match self.level { + tracing::Level::ERROR => OutputEvent::Error { message, context }, + tracing::Level::WARN => OutputEvent::Warning { message, context }, + _ => OutputEvent::Info { message, context }, + } + } +} + +fn normalize_tracing_message(target: &str, message: &str) -> (String, Option) { + let message = message.trim().to_string(); + if target.starts_with("noq_proto") { + return ( + normalize_noq_proto_message(target, &message), + Some("transport".to_string()), + ); + } + + (message, Some("stderr".to_string())) +} + +fn normalize_noq_proto_message(target: &str, message: &str) -> String { + let without_prefix = message + .find(target) + .and_then(|target_index| { + message[target_index + target.len()..] + .find(':') + .map(|colon_index| message[target_index + target.len() + colon_index + 1..].trim()) + }) + .unwrap_or(message) + .trim(); + format_noq_proto_fields(without_prefix) +} + +fn format_noq_proto_fields(message: &str) -> String { + let Some(rest) = message.strip_prefix("err=") else { + return message.to_string(); + }; + let Some((err, detail)) = rest.split_once(' ') else { + return message.to_string(); + }; + if detail.trim().is_empty() { + message.to_string() + } else { + format!("{} (err={err})", detail.trim()) + } +} + +fn strip_ansi_escape_sequences(input: &str) -> String { + let mut output = String::with_capacity(input.len()); + let mut chars = input.chars().peekable(); + + while let Some(ch) = chars.next() { + if ch != '\u{1b}' { + output.push(ch); + continue; + } + + if matches!(chars.peek(), Some('[')) { + chars.next(); + for code in chars.by_ref() { + if ('@'..='~').contains(&code) { + break; + } + } + } + } + + output } impl Write for MeshTracingStderrWriter { @@ -9057,6 +9116,34 @@ mod tests { } } + #[test] + fn noq_proto_tracing_messages_use_transport_context() { + let message = "2026-06-11T03:49:18.033043Z WARN noq_proto::connection: err=LastOpenPath failed closing path"; + + let (message, context) = normalize_tracing_message("noq_proto::connection", message); + + assert_eq!(message, "failed closing path (err=LastOpenPath)"); + assert_eq!(context.as_deref(), Some("transport")); + } + + #[test] + fn routed_tracing_messages_strip_ansi_sequences() { + let formatted = "\u{1b}[2m2026-06-11T03:49:18.033043Z\u{1b}[0m \u{1b}[33m WARN\u{1b}[0m"; + + assert_eq!( + strip_ansi_escape_sequences(formatted), + "2026-06-11T03:49:18.033043Z WARN" + ); + } + + #[test] + fn non_proto_tracing_messages_keep_stderr_context() { + let (message, context) = normalize_tracing_message("mesh_llm::runtime", "runtime warning"); + + assert_eq!(message, "runtime warning"); + assert_eq!(context.as_deref(), Some("stderr")); + } + fn reconciliation_target_with_required_bytes( required_bytes: Option, ) -> api::status::ModelTargetPayload { diff --git a/crates/mesh-llm-tui/src/output/mod.rs b/crates/mesh-llm-tui/src/output/mod.rs index adcfbb262f..c990a793c8 100644 --- a/crates/mesh-llm-tui/src/output/mod.rs +++ b/crates/mesh-llm-tui/src/output/mod.rs @@ -4,7 +4,7 @@ use chrono::{Local, SecondsFormat, Utc}; use crossterm::{ cursor::{Hide, MoveTo, Show}, execute, - terminal::{Clear, ClearType, EnterAlternateScreen, LeaveAlternateScreen}, + terminal::{Clear, ClearType, EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode}, }; pub use mesh_llm_events::{ ConsoleSessionMode, DashboardAcceptedRequestBucket, DashboardEndpointRow, DashboardLaunchPlan, @@ -354,7 +354,7 @@ enum TuiEventListRenderer { Scrollbar, } -const PRETTY_TUI_EVENT_LEVEL_WIDTH: usize = 4; +const PRETTY_TUI_EVENT_LEVEL_WIDTH: usize = 5; const _: TuiEventListRenderer = TuiEventListRenderer::Legacy; @@ -531,7 +531,9 @@ impl OutputEventPresentation for OutputEvent { *total_bytes, status, ), - OutputEvent::Error { context, message } | OutputEvent::Warning { message, context } => { + OutputEvent::Error { context, message } + | OutputEvent::Warning { message, context } + | OutputEvent::Fatal { message, context } => { Self::contextual_summary(context.as_deref(), message) } OutputEvent::LlamaNativeLog { message, .. } => message.clone(), @@ -789,11 +791,10 @@ impl OutputEventPresentation for OutputEvent { json!({ "warning": message, "context": context }) } OutputEvent::Error { message, context } => { - json!({ - "error": message, - "context": context, - "error_type": classify_error_type(message, context.as_deref()), - }) + classified_error_json("error", message, context.as_deref()) + } + OutputEvent::Fatal { message, context } => { + classified_error_json("fatal", message, context.as_deref()) } OutputEvent::ShutdownRequested { signal } => json!({ "signal": signal }), OutputEvent::Shutdown { reason } => json!({ "reason": reason }), @@ -813,6 +814,14 @@ impl OutputEventPresentation for OutputEvent { } } +fn classified_error_json(field: &str, message: &str, context: Option<&str>) -> Value { + json!({ + field: message, + "context": context, + "error_type": classify_error_type(message, context), + }) +} + fn format_message_with_params(message: &str, params: &[(String, Value)]) -> String { if params.is_empty() { return message.to_string(); @@ -2090,7 +2099,7 @@ impl DashboardState { self.startup_lifecycle .finalize_for_runtime_ready(api_url, console_url.as_deref()); } - OutputEvent::Error { message, context } => { + OutputEvent::Error { message, context } | OutputEvent::Fatal { message, context } => { if self.runtime_ready || self.shutdown_in_progress { return; } @@ -2470,7 +2479,7 @@ impl DashboardState { OutputEvent::ShutdownRequested { .. } | OutputEvent::Shutdown { .. } => { self.mark_runtime_shutting_down(); } - OutputEvent::Error { .. } => { + OutputEvent::Error { .. } | OutputEvent::Fatal { .. } => { if let Some(model) = self.running_models.last_mut() { model.status = RuntimeStatus::Error; } @@ -7033,6 +7042,7 @@ fn startup_history_summary(event: &OutputEvent) -> Option { | OutputEvent::ApiReady { .. } | OutputEvent::RuntimeReady { .. } | OutputEvent::Error { .. } + | OutputEvent::Fatal { .. } | OutputEvent::Warning { .. } => Some(event.summary_line()), OutputEvent::ModelDownloadProgress { status, .. } => { if matches!(status, ModelProgressStatus::Ready) { @@ -7318,7 +7328,14 @@ fn empty_panel_message(state: &DashboardState, panel: DashboardPanel) -> &'stati fn event_severity_badge(event: &MeshEventState) -> (&'static str, Style) { let theme = tui_theme(); let summary_lower = event.summary.to_lowercase(); - if matches!(event.level, OutputLevel::Error) + if matches!(event.level, OutputLevel::Fatal) { + ( + "FATAL", + Style::default() + .fg(theme.error) + .add_modifier(Modifier::BOLD), + ) + } else if matches!(event.level, OutputLevel::Error) || summary_lower.contains("err") || summary_lower.contains("failed") { @@ -8371,7 +8388,7 @@ fn build_fatal_error_event(err: &AnyhowError) -> OutputEvent { .skip(1) .map(ToString::to_string) .collect::>(); - OutputEvent::Error { + OutputEvent::Fatal { message, context: (!context.is_empty()).then(|| context.join(": ")), } @@ -8381,6 +8398,30 @@ pub fn emit_fatal_error(err: &AnyhowError) -> io::Result<()> { emit_event(build_fatal_error_event(err)) } +pub fn emit_fatal_panic(message: impl Into, context: Option) -> io::Result<()> { + let event = OutputEvent::Fatal { + message: message.into(), + context, + }; + write_emergency_event(&event) +} + +fn write_emergency_event(event: &OutputEvent) -> io::Result<()> { + let mode = GLOBAL_OUTPUT_MANAGER + .get() + .map(OutputManager::mode) + .unwrap_or(LogFormat::Pretty); + let rendered = render_emergency_event(mode, event)?; + write_rendered_output(mode, &rendered) +} + +fn render_emergency_event(mode: LogFormat, event: &OutputEvent) -> io::Result { + match mode { + LogFormat::Pretty => PrettyFormatter.format(event), + LogFormat::Json => JsonFormatter.format(event), + } +} + pub fn json_mode_enabled() -> bool { GLOBAL_OUTPUT_MANAGER .get() @@ -8443,6 +8484,11 @@ pub fn force_restore_tui_terminal() -> io::Result<()> { write_tui_exit() } +pub fn force_restore_tui_after_panic() { + let _ = force_restore_tui_terminal(); + let _ = disable_raw_mode(); +} + fn write_tui_enter_to_writer(writer: &mut W) -> io::Result<()> { execute!( writer, @@ -9192,6 +9238,10 @@ mod tests { message: "❌ llama-server exited".to_string(), context: Some("model=Qwen3-32B port=9337".to_string()), }, + OutputEvent::Fatal { + message: "panic occurred".to_string(), + context: Some("panic at crates/mesh-llm/src/lib.rs:42".to_string()), + }, OutputEvent::Shutdown { reason: Some("user requested shutdown".to_string()), }, @@ -14955,6 +15005,39 @@ tail line" assert_eq!(value["context"], "model=Qwen3-32B mode=dense port=9337"); } + #[test] + fn json_formatter_includes_fatal_level_and_context() { + let mut formatter = JsonFormatter; + let rendered = formatter + .format(&OutputEvent::Fatal { + message: "panic occurred".to_string(), + context: Some("panic at crates/mesh-llm/src/lib.rs:42".to_string()), + }) + .expect("fatal render should succeed"); + let value: Value = serde_json::from_str(rendered.trim_end()).expect("line should parse"); + + assert_eq!(value["event"], "fatal"); + assert_eq!(value["level"], "fatal"); + assert_eq!(value["fatal"], "panic occurred"); + assert_eq!(value["context"], "panic at crates/mesh-llm/src/lib.rs:42"); + } + + #[test] + fn emergency_fatal_event_renders_without_dashboard_worker() { + let event = OutputEvent::Fatal { + message: "panic occurred".to_string(), + context: Some("panic at crates/mesh-llm/src/lib.rs:42".to_string()), + }; + + let rendered = render_emergency_event(LogFormat::Pretty, &event) + .expect("emergency fatal render should succeed"); + + assert_eq!( + rendered, + "panic at crates/mesh-llm/src/lib.rs:42: panic occurred\n" + ); + } + #[test] fn json_formatter_includes_info_context() { let mut formatter = JsonFormatter; diff --git a/crates/mesh-llm/src/lib.rs b/crates/mesh-llm/src/lib.rs index 218847ebc9..2326271169 100644 --- a/crates/mesh-llm/src/lib.rs +++ b/crates/mesh-llm/src/lib.rs @@ -1,6 +1,6 @@ #![recursion_limit = "256"] -use std::time::Duration; +use std::{panic::PanicHookInfo, sync::Once, time::Duration}; use clap::{CommandFactory, Parser}; @@ -8,6 +8,8 @@ mod commands; pub use mesh_llm_host_runtime::*; +static PANIC_HOOK: Once = Once::new(); + pub async fn run_main() -> i32 { match run_cli_entrypoint().await { Ok(()) => 0, @@ -39,6 +41,7 @@ async fn run_cli_entrypoint() -> anyhow::Result<()> { cli.log_format, mesh_llm_host_runtime::console_session_mode_for_runtime_surface(explicit_surface), ); + install_terminal_panic_hook(); mesh_llm_host_runtime::run_runtime_initialized( runtime_options_from_cli(cli), @@ -48,6 +51,32 @@ async fn run_cli_entrypoint() -> anyhow::Result<()> { .await } +fn install_terminal_panic_hook() { + PANIC_HOOK.call_once(|| { + let previous_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + mesh_llm_tui::force_restore_tui_after_panic(); + let _ = mesh_llm_tui::emit_fatal_panic(panic_message(info), panic_context(info)); + previous_hook(info); + })); + }); +} + +fn panic_message(info: &PanicHookInfo<'_>) -> String { + if let Some(message) = info.payload().downcast_ref::<&str>() { + (*message).to_string() + } else if let Some(message) = info.payload().downcast_ref::() { + message.clone() + } else { + "panic occurred".to_string() + } +} + +fn panic_context(info: &PanicHookInfo<'_>) -> Option { + info.location() + .map(|location| format!("panic at {}:{}", location.file(), location.line())) +} + fn maybe_print_binary_help_and_exit() { let args: Vec<_> = std::env::args_os().collect(); if args.len() == 1 { From da492bb3a5131bd7ee973268bd57dd654f1f7e4d Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 11 Jun 2026 01:56:40 -0400 Subject: [PATCH 2/4] address comments --- crates/mesh-llm-tui/src/lib.rs | 84 +++++++++++++++++++ crates/mesh-llm-tui/src/output/mod.rs | 113 +++++++++++++++++++++++--- crates/mesh-llm/src/lib.rs | 32 +------- 3 files changed, 189 insertions(+), 40 deletions(-) diff --git a/crates/mesh-llm-tui/src/lib.rs b/crates/mesh-llm-tui/src/lib.rs index f71776d3c7..b239e69a0a 100644 --- a/crates/mesh-llm-tui/src/lib.rs +++ b/crates/mesh-llm-tui/src/lib.rs @@ -1,6 +1,90 @@ #![forbid(unsafe_code)] +use std::{panic::PanicHookInfo, sync::Once}; + pub mod output; pub mod terminal_progress; pub use output::*; + +static PANIC_HOOK: Once = Once::new(); + +pub fn install_terminal_panic_hook() { + PANIC_HOOK.call_once(|| { + let previous_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + force_restore_tui_after_panic(); + let _ = emit_fatal_panic(panic_message(info), panic_context(info)); + previous_hook(info); + })); + }); +} + +fn panic_message(info: &PanicHookInfo<'_>) -> String { + if let Some(message) = info.payload().downcast_ref::<&str>() { + (*message).to_string() + } else if let Some(message) = info.payload().downcast_ref::() { + message.clone() + } else { + "panic occurred".to_string() + } +} + +fn panic_context(info: &PanicHookInfo<'_>) -> Option { + info.location() + .map(|location| format!("panic at {}:{}", location.file(), location.line())) +} + +#[cfg(test)] +mod tests { + use super::install_terminal_panic_hook; + use std::{ + panic::{self, AssertUnwindSafe}, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + }; + + #[test] + fn install_terminal_panic_hook_chains_previous_hook() { + let previous_hook_calls = Arc::new(AtomicUsize::new(0)); + let previous_payload = Arc::new(Mutex::new(None)); + let previous_location = Arc::new(Mutex::new(None)); + + let hook_calls = Arc::clone(&previous_hook_calls); + let hook_payload = Arc::clone(&previous_payload); + let hook_location = Arc::clone(&previous_location); + panic::set_hook(Box::new(move |info| { + hook_calls.fetch_add(1, Ordering::SeqCst); + let payload = info + .payload() + .downcast_ref::<&str>() + .map(|message| (*message).to_string()) + .or_else(|| info.payload().downcast_ref::().cloned()); + *hook_payload.lock().expect("payload lock") = payload; + *hook_location.lock().expect("location lock") = + info.location().map(|location| location.file().to_string()); + })); + + install_terminal_panic_hook(); + + let result = panic::catch_unwind(AssertUnwindSafe(|| { + panic!("panic hook smoke test"); + })); + + assert!(result.is_err()); + assert_eq!(previous_hook_calls.load(Ordering::SeqCst), 1); + assert_eq!( + previous_payload.lock().expect("payload lock").as_deref(), + Some("panic hook smoke test") + ); + assert_eq!( + previous_location + .lock() + .expect("location lock") + .as_deref(), + Some(file!()) + ); + } +} diff --git a/crates/mesh-llm-tui/src/output/mod.rs b/crates/mesh-llm-tui/src/output/mod.rs index c990a793c8..d457883ee9 100644 --- a/crates/mesh-llm-tui/src/output/mod.rs +++ b/crates/mesh-llm-tui/src/output/mod.rs @@ -3883,7 +3883,7 @@ fn render_mesh_events(state: &DashboardState) -> Vec { .map(|event| { let (badge_text, _) = event_severity_badge(event); format!( - "{} {: Option { | OutputEvent::ApiReady { .. } | OutputEvent::RuntimeReady { .. } | OutputEvent::Error { .. } - | OutputEvent::Fatal { .. } | OutputEvent::Warning { .. } => Some(event.summary_line()), OutputEvent::ModelDownloadProgress { status, .. } => { if matches!(status, ModelProgressStatus::Ready) { @@ -7403,7 +7402,7 @@ fn event_line(event: &MeshEventState, width: usize) -> Line<'static> { event.timestamp, badge_text ); let prefix_len = prefix.chars().count(); - let remaining = width.saturating_sub(prefix_len + 1); + let remaining = width.saturating_sub(prefix_len); if remaining == 0 { return Line::from(vec![Span::styled( truncate_with_ellipsis(&prefix, width), @@ -7415,7 +7414,6 @@ fn event_line(event: &MeshEventState, width: usize) -> Line<'static> { Span::styled(event.timestamp.clone(), Style::default().fg(theme.dim)), Span::raw(" "), event_severity_badge_span(event), - Span::raw(" "), Span::styled( truncate_with_ellipsis(&message, remaining), Style::default().fg(theme.text), @@ -7431,8 +7429,7 @@ fn wrapped_event_lines(event: &MeshEventState, width: usize) -> Vec Vec, terminal_active: bool, + tui_entered: Arc, dirty: bool, } impl InteractiveDashboardFormatter { + fn with_tui_entered(tui_entered: Arc) -> Self { + Self { + tui_entered, + ..Self::default() + } + } + + #[cfg(test)] + fn tui_entered(&self) -> bool { + self.tui_entered.load(Ordering::Acquire) + } + fn handle_output_event(&mut self, event: &OutputEvent) -> io::Result> { self.state .reduce(DashboardAction::OutputEvent(event.clone())); @@ -7711,6 +7720,7 @@ impl InteractiveDashboardFormatter { // cleanup: the terminal may already be in alternate-screen/raw-input // state even if ratatui terminal construction or cursor hiding fails. self.terminal_active = true; + self.tui_entered.store(true, Ordering::Release); self.dirty = true; } @@ -7723,7 +7733,11 @@ impl InteractiveDashboardFormatter { } self.terminal_active = false; self.dirty = false; - write_tui_exit() + let result = write_tui_exit(); + if result.is_ok() { + self.tui_entered.store(false, Ordering::Release); + } + result } fn render_if_dirty(&mut self) -> io::Result { @@ -7886,14 +7900,27 @@ impl Formatter for FormatterSelection { } } +#[cfg(test)] fn select_formatter( mode: LogFormat, console_session_mode: ConsoleSessionMode, +) -> FormatterSelection { + select_formatter_with_tui_entered( + mode, + console_session_mode, + Arc::new(AtomicBool::new(false)), + ) +} + +fn select_formatter_with_tui_entered( + mode: LogFormat, + console_session_mode: ConsoleSessionMode, + tui_entered: Arc, ) -> FormatterSelection { match mode { LogFormat::Pretty => match console_session_mode { ConsoleSessionMode::InteractiveDashboard => { - FormatterSelection::InteractiveDashboard(InteractiveDashboardFormatter::default()) + FormatterSelection::InteractiveDashboard(InteractiveDashboardFormatter::with_tui_entered(tui_entered)) } ConsoleSessionMode::Fallback => { FormatterSelection::DashboardFallback(DashboardFormatter::default()) @@ -7907,6 +7934,7 @@ fn select_formatter( struct OutputManagerState { tx: tokio::sync::mpsc::UnboundedSender, ready_prompt_active: Arc, + tui_entered: Arc, mode: LogFormat, console_session_mode: Option, dashboard_snapshot_provider: Arc>>>, @@ -8040,12 +8068,18 @@ impl OutputManager { ) -> OutputManagerState { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); let ready_prompt_active = Arc::new(AtomicBool::new(false)); + let tui_entered = Arc::new(AtomicBool::new(false)); let worker_prompt_active = ready_prompt_active.clone(); + let worker_tui_entered = tui_entered.clone(); let dashboard_snapshot_provider: Arc>>> = Arc::new(RwLock::new(None)); let worker_snapshot_provider = dashboard_snapshot_provider.clone(); tokio::spawn(async move { - let mut formatter = select_formatter(mode, console_session_mode); + let mut formatter = select_formatter_with_tui_entered( + mode, + console_session_mode, + worker_tui_entered, + ); let mut redraw_tick = time::interval(PRETTY_TUI_REDRAW_INTERVAL); redraw_tick.set_missed_tick_behavior(MissedTickBehavior::Skip); let mut snapshot_tick = time::interval(PRETTY_TUI_SNAPSHOT_INTERVAL); @@ -8124,6 +8158,7 @@ impl OutputManager { OutputManagerState { tx, ready_prompt_active, + tui_entered, mode, console_session_mode: matches!(mode, LogFormat::Pretty).then_some(console_session_mode), dashboard_snapshot_provider, @@ -8225,6 +8260,13 @@ impl OutputManager { .unwrap_or(None) } + fn tui_entered(&self) -> bool { + self.state + .read() + .map(|state| state.tui_entered.load(Ordering::Acquire)) + .unwrap_or(false) + } + pub fn register_dashboard_snapshot_provider( &self, provider: Arc, @@ -8485,6 +8527,10 @@ pub fn force_restore_tui_terminal() -> io::Result<()> { } pub fn force_restore_tui_after_panic() { + if !GLOBAL_OUTPUT_MANAGER.get().is_some_and(|output_manager| output_manager.tui_entered()) { + return; + } + let _ = force_restore_tui_terminal(); let _ = disable_raw_mode(); } @@ -12429,6 +12475,41 @@ mod tests { assert!(api_start_index < api_ready_index); } + #[test] + fn fatal_events_do_not_consume_startup_history_slots() { + let mut formatter = InteractiveDashboardFormatter::default(); + let fatal = OutputEvent::Fatal { + message: "panic occurred".to_string(), + context: Some("panic at crates/mesh-llm/src/lib.rs:42".to_string()), + }; + + formatter + .handle_output_event(&OutputEvent::Startup { + version: "v0.68.0".to_string(), + message: None, + }) + .expect("startup event should reduce cleanly"); + formatter + .handle_output_event(&fatal) + .expect("fatal event should reduce cleanly"); + + assert_eq!(formatter.state.startup_history.len(), 1); + assert!( + formatter + .state + .startup_history + .iter() + .all(|event| !event.summary.contains("panic occurred")) + ); + assert!( + formatter + .state + .mesh_events + .iter() + .any(|event| event.summary.contains("panic occurred")) + ); + } + #[test] fn startup_failures_surface_in_tui_events_and_status() { let mut formatter = InteractiveDashboardFormatter::default(); @@ -13817,10 +13898,22 @@ tail line" formatter.mark_terminal_escape_written(); assert!(formatter.terminal_active); + assert!(formatter.tui_entered()); assert!(formatter.dirty); assert!(formatter.terminal.is_none()); } + #[test] + fn tui_panic_restore_flag_tracks_terminal_entry() { + let mut formatter = InteractiveDashboardFormatter::default(); + + assert!(!formatter.tui_entered()); + formatter.mark_terminal_escape_written(); + assert!(formatter.tui_entered()); + formatter.exit_terminal().expect("exit should succeed"); + assert!(!formatter.tui_entered()); + } + #[test] fn tui_narrow_terminal_renders_resize_guidance_instead_of_dashboard() { let mut state = DashboardState::default(); diff --git a/crates/mesh-llm/src/lib.rs b/crates/mesh-llm/src/lib.rs index 2326271169..db39808919 100644 --- a/crates/mesh-llm/src/lib.rs +++ b/crates/mesh-llm/src/lib.rs @@ -1,6 +1,6 @@ #![recursion_limit = "256"] -use std::{panic::PanicHookInfo, sync::Once, time::Duration}; +use std::time::Duration; use clap::{CommandFactory, Parser}; @@ -8,8 +8,6 @@ mod commands; pub use mesh_llm_host_runtime::*; -static PANIC_HOOK: Once = Once::new(); - pub async fn run_main() -> i32 { match run_cli_entrypoint().await { Ok(()) => 0, @@ -41,7 +39,7 @@ async fn run_cli_entrypoint() -> anyhow::Result<()> { cli.log_format, mesh_llm_host_runtime::console_session_mode_for_runtime_surface(explicit_surface), ); - install_terminal_panic_hook(); + mesh_llm_tui::install_terminal_panic_hook(); mesh_llm_host_runtime::run_runtime_initialized( runtime_options_from_cli(cli), @@ -51,32 +49,6 @@ async fn run_cli_entrypoint() -> anyhow::Result<()> { .await } -fn install_terminal_panic_hook() { - PANIC_HOOK.call_once(|| { - let previous_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |info| { - mesh_llm_tui::force_restore_tui_after_panic(); - let _ = mesh_llm_tui::emit_fatal_panic(panic_message(info), panic_context(info)); - previous_hook(info); - })); - }); -} - -fn panic_message(info: &PanicHookInfo<'_>) -> String { - if let Some(message) = info.payload().downcast_ref::<&str>() { - (*message).to_string() - } else if let Some(message) = info.payload().downcast_ref::() { - message.clone() - } else { - "panic occurred".to_string() - } -} - -fn panic_context(info: &PanicHookInfo<'_>) -> Option { - info.location() - .map(|location| format!("panic at {}:{}", location.file(), location.line())) -} - fn maybe_print_binary_help_and_exit() { let args: Vec<_> = std::env::args_os().collect(); if args.len() == 1 { From b141e99346217a36508492438a3c18bdb1f7c89b Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 11 Jun 2026 02:50:43 -0400 Subject: [PATCH 3/4] fixup --- crates/mesh-llm-tui/src/lib.rs | 12 +++---- crates/mesh-llm-tui/src/output/mod.rs | 52 +++++++++++++-------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/crates/mesh-llm-tui/src/lib.rs b/crates/mesh-llm-tui/src/lib.rs index b239e69a0a..06461c2372 100644 --- a/crates/mesh-llm-tui/src/lib.rs +++ b/crates/mesh-llm-tui/src/lib.rs @@ -13,8 +13,8 @@ pub fn install_terminal_panic_hook() { PANIC_HOOK.call_once(|| { let previous_hook = std::panic::take_hook(); std::panic::set_hook(Box::new(move |info| { - force_restore_tui_after_panic(); - let _ = emit_fatal_panic(panic_message(info), panic_context(info)); + output::force_restore_tui_after_panic(); + let _ = output::emit_fatal_panic(panic_message(info), panic_context(info)); previous_hook(info); })); }); @@ -48,6 +48,7 @@ mod tests { #[test] fn install_terminal_panic_hook_chains_previous_hook() { + let previous_hook = panic::take_hook(); let previous_hook_calls = Arc::new(AtomicUsize::new(0)); let previous_payload = Arc::new(Mutex::new(None)); let previous_location = Arc::new(Mutex::new(None)); @@ -73,6 +74,8 @@ mod tests { panic!("panic hook smoke test"); })); + panic::set_hook(previous_hook); + assert!(result.is_err()); assert_eq!(previous_hook_calls.load(Ordering::SeqCst), 1); assert_eq!( @@ -80,10 +83,7 @@ mod tests { Some("panic hook smoke test") ); assert_eq!( - previous_location - .lock() - .expect("location lock") - .as_deref(), + previous_location.lock().expect("location lock").as_deref(), Some(file!()) ); } diff --git a/crates/mesh-llm-tui/src/output/mod.rs b/crates/mesh-llm-tui/src/output/mod.rs index d457883ee9..f7df0e1b85 100644 --- a/crates/mesh-llm-tui/src/output/mod.rs +++ b/crates/mesh-llm-tui/src/output/mod.rs @@ -354,7 +354,7 @@ enum TuiEventListRenderer { Scrollbar, } -const PRETTY_TUI_EVENT_LEVEL_WIDTH: usize = 5; +const PRETTY_TUI_EVENT_LEVEL_WIDTH: usize = 6; const _: TuiEventListRenderer = TuiEventListRenderer::Legacy; @@ -1536,9 +1536,9 @@ impl DashboardState { let llama_component = self.startup_component_for_truthful_status(TruthfulStartupStatusKey::LlamaServer); - if let Some(webserver) = &mut self.webserver - && let Some(key) = Self::truthful_startup_key_for_endpoint(&webserver.label) - { + if let Some((webserver, key)) = self.webserver.as_mut().and_then(|webserver| { + Self::truthful_startup_key_for_endpoint(&webserver.label).map(|key| (webserver, key)) + }) { webserver.status = Self::truthful_runtime_status_for_component( match key { TruthfulStartupStatusKey::Console => &console_component, @@ -1548,9 +1548,9 @@ impl DashboardState { &webserver.status, ); } - if let Some(api) = &mut self.api - && let Some(key) = Self::truthful_startup_key_for_endpoint(&api.label) - { + if let Some((api, key)) = self.api.as_mut().and_then(|api| { + Self::truthful_startup_key_for_endpoint(&api.label).map(|key| (api, key)) + }) { api.status = Self::truthful_runtime_status_for_component( match key { TruthfulStartupStatusKey::Console => &console_component, @@ -1862,9 +1862,9 @@ impl DashboardState { return None; } - if let Some(progress) = self.model_progress.as_ref() - && let Some(ratio) = model_download_progress_ratio(progress) - { + if let Some((progress, ratio)) = self.model_progress.as_ref().and_then(|progress| { + model_download_progress_ratio(progress).map(|ratio| (progress, ratio)) + }) { return Some(LoadingProgressState { ratio, detail: loading_progress_detail(model_progress_detail(progress), ratio, None), @@ -2479,7 +2479,7 @@ impl DashboardState { OutputEvent::ShutdownRequested { .. } | OutputEvent::Shutdown { .. } => { self.mark_runtime_shutting_down(); } - OutputEvent::Error { .. } | OutputEvent::Fatal { .. } => { + OutputEvent::Error { .. } => { if let Some(model) = self.running_models.last_mut() { model.status = RuntimeStatus::Error; } @@ -7905,11 +7905,7 @@ fn select_formatter( mode: LogFormat, console_session_mode: ConsoleSessionMode, ) -> FormatterSelection { - select_formatter_with_tui_entered( - mode, - console_session_mode, - Arc::new(AtomicBool::new(false)), - ) + select_formatter_with_tui_entered(mode, console_session_mode, Arc::new(AtomicBool::new(false))) } fn select_formatter_with_tui_entered( @@ -7919,9 +7915,9 @@ fn select_formatter_with_tui_entered( ) -> FormatterSelection { match mode { LogFormat::Pretty => match console_session_mode { - ConsoleSessionMode::InteractiveDashboard => { - FormatterSelection::InteractiveDashboard(InteractiveDashboardFormatter::with_tui_entered(tui_entered)) - } + ConsoleSessionMode::InteractiveDashboard => FormatterSelection::InteractiveDashboard( + InteractiveDashboardFormatter::with_tui_entered(tui_entered), + ), ConsoleSessionMode::Fallback => { FormatterSelection::DashboardFallback(DashboardFormatter::default()) } @@ -8075,11 +8071,8 @@ impl OutputManager { Arc::new(RwLock::new(None)); let worker_snapshot_provider = dashboard_snapshot_provider.clone(); tokio::spawn(async move { - let mut formatter = select_formatter_with_tui_entered( - mode, - console_session_mode, - worker_tui_entered, - ); + let mut formatter = + select_formatter_with_tui_entered(mode, console_session_mode, worker_tui_entered); let mut redraw_tick = time::interval(PRETTY_TUI_REDRAW_INTERVAL); redraw_tick.set_missed_tick_behavior(MissedTickBehavior::Skip); let mut snapshot_tick = time::interval(PRETTY_TUI_SNAPSHOT_INTERVAL); @@ -8527,7 +8520,10 @@ pub fn force_restore_tui_terminal() -> io::Result<()> { } pub fn force_restore_tui_after_panic() { - if !GLOBAL_OUTPUT_MANAGER.get().is_some_and(|output_manager| output_manager.tui_entered()) { + if !GLOBAL_OUTPUT_MANAGER + .get() + .is_some_and(|output_manager| output_manager.tui_entered()) + { return; } @@ -9752,7 +9748,7 @@ mod tests { assert_eq!( spans_plain_text(&line.spans), - "12:34:56 OK joined mesh poker-night" + "12:34:56 OK joined mesh poker-night" ); } @@ -10318,7 +10314,7 @@ mod tests { assert!( rendered_lines .iter() - .any(|line| line.contains("INFO plain operati")), + .any(|line| line.contains("INFO plain operati")), "expected INFO badge row to remain visible: {rendered_lines:?}" ); assert!( @@ -10354,7 +10350,7 @@ mod tests { .expect("expected timestamp token"); assert_hh_mm_ss(timestamp); assert!( - event_line.contains(" OK joined mesh poker-night"), + event_line.contains(" OK joined mesh poker-night"), "expected compact log row in {event_line}" ); assert!(event_line.contains("joined mesh poker-night")); From 668f98848dd5e0e869f11f58b3c86f7cbddd1081 Mon Sep 17 00:00:00 2001 From: Nick DiZazzo Date: Thu, 11 Jun 2026 04:00:14 -0400 Subject: [PATCH 4/4] final fixes --- crates/mesh-llm-tui/src/output/mod.rs | 152 +++++++++++++++++++++++--- 1 file changed, 137 insertions(+), 15 deletions(-) diff --git a/crates/mesh-llm-tui/src/output/mod.rs b/crates/mesh-llm-tui/src/output/mod.rs index f7df0e1b85..710da1971a 100644 --- a/crates/mesh-llm-tui/src/output/mod.rs +++ b/crates/mesh-llm-tui/src/output/mod.rs @@ -2479,11 +2479,7 @@ impl DashboardState { OutputEvent::ShutdownRequested { .. } | OutputEvent::Shutdown { .. } => { self.mark_runtime_shutting_down(); } - OutputEvent::Error { .. } => { - if let Some(model) = self.running_models.last_mut() { - model.status = RuntimeStatus::Error; - } - } + OutputEvent::Error { .. } => {} OutputEvent::InviteToken { token, mesh_id, @@ -7659,13 +7655,15 @@ pub struct InteractiveDashboardFormatter { terminal: Option, terminal_active: bool, tui_entered: Arc, + panic_restored: Arc, dirty: bool, } impl InteractiveDashboardFormatter { - fn with_tui_entered(tui_entered: Arc) -> Self { + fn with_tui_state(tui_entered: Arc, panic_restored: Arc) -> Self { Self { tui_entered, + panic_restored, ..Self::default() } } @@ -7675,7 +7673,22 @@ impl InteractiveDashboardFormatter { self.tui_entered.load(Ordering::Acquire) } + fn panic_restored(&self) -> bool { + self.panic_restored.load(Ordering::Acquire) + } + + fn mark_panic_restored(&mut self) { + self.terminal = None; + self.terminal_active = false; + self.dirty = false; + self.tui_entered.store(false, Ordering::Release); + self.panic_restored.store(true, Ordering::Release); + } + fn handle_output_event(&mut self, event: &OutputEvent) -> io::Result> { + if self.panic_restored() { + return Ok(None); + } self.state .reduce(DashboardAction::OutputEvent(event.clone())); if self.terminal_active { @@ -7687,6 +7700,9 @@ impl InteractiveDashboardFormatter { } fn handle_snapshot(&mut self, snapshot: DashboardSnapshot) { + if self.panic_restored() { + return; + } self.state .reduce(DashboardAction::SnapshotUpdated(snapshot)); if self.terminal_active { @@ -7695,6 +7711,9 @@ impl InteractiveDashboardFormatter { } fn handle_tui_event(&mut self, event: TuiEvent) -> TuiControlFlow { + if self.panic_restored() { + return TuiControlFlow::Continue; + } let control = self.state.apply_tui_event(event); if self.terminal_active { self.dirty = true; @@ -7703,6 +7722,9 @@ impl InteractiveDashboardFormatter { } fn enter_terminal(&mut self) -> io::Result<()> { + if self.panic_restored() { + return Ok(()); + } if self.terminal_active { return Ok(()); } @@ -7741,6 +7763,9 @@ impl InteractiveDashboardFormatter { } fn render_if_dirty(&mut self) -> io::Result { + if self.panic_restored() { + return Ok(false); + } if self .state .clear_expired_join_token_copy_status(Instant::now()) @@ -7877,6 +7902,12 @@ impl FormatterSelection { } } + fn mark_panic_restored(&mut self) { + if let Self::InteractiveDashboard(formatter) = self { + formatter.mark_panic_restored(); + } + } + fn render_interactive_if_dirty(&mut self) -> io::Result { match self { Self::InteractiveDashboard(formatter) => formatter.render_if_dirty(), @@ -7905,18 +7936,24 @@ fn select_formatter( mode: LogFormat, console_session_mode: ConsoleSessionMode, ) -> FormatterSelection { - select_formatter_with_tui_entered(mode, console_session_mode, Arc::new(AtomicBool::new(false))) + select_formatter_with_tui_state( + mode, + console_session_mode, + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + ) } -fn select_formatter_with_tui_entered( +fn select_formatter_with_tui_state( mode: LogFormat, console_session_mode: ConsoleSessionMode, tui_entered: Arc, + panic_restored: Arc, ) -> FormatterSelection { match mode { LogFormat::Pretty => match console_session_mode { ConsoleSessionMode::InteractiveDashboard => FormatterSelection::InteractiveDashboard( - InteractiveDashboardFormatter::with_tui_entered(tui_entered), + InteractiveDashboardFormatter::with_tui_state(tui_entered, panic_restored), ), ConsoleSessionMode::Fallback => { FormatterSelection::DashboardFallback(DashboardFormatter::default()) @@ -7931,6 +7968,7 @@ struct OutputManagerState { tx: tokio::sync::mpsc::UnboundedSender, ready_prompt_active: Arc, tui_entered: Arc, + panic_restored: Arc, mode: LogFormat, console_session_mode: Option, dashboard_snapshot_provider: Arc>>>, @@ -8016,6 +8054,7 @@ enum OutputCommand { response: tokio::sync::oneshot::Sender>, }, RenderTui(tokio::sync::oneshot::Sender>), + PanicRestored, } static GLOBAL_OUTPUT_MANAGER: OnceLock = OnceLock::new(); @@ -8065,14 +8104,20 @@ impl OutputManager { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); let ready_prompt_active = Arc::new(AtomicBool::new(false)); let tui_entered = Arc::new(AtomicBool::new(false)); + let panic_restored = Arc::new(AtomicBool::new(false)); let worker_prompt_active = ready_prompt_active.clone(); let worker_tui_entered = tui_entered.clone(); + let worker_panic_restored = panic_restored.clone(); let dashboard_snapshot_provider: Arc>>> = Arc::new(RwLock::new(None)); let worker_snapshot_provider = dashboard_snapshot_provider.clone(); tokio::spawn(async move { - let mut formatter = - select_formatter_with_tui_entered(mode, console_session_mode, worker_tui_entered); + let mut formatter = select_formatter_with_tui_state( + mode, + console_session_mode, + worker_tui_entered, + worker_panic_restored, + ); let mut redraw_tick = time::interval(PRETTY_TUI_REDRAW_INTERVAL); redraw_tick.set_missed_tick_behavior(MissedTickBehavior::Skip); let mut snapshot_tick = time::interval(PRETTY_TUI_SNAPSHOT_INTERVAL); @@ -8125,6 +8170,9 @@ impl OutputManager { OutputCommand::RenderTui(response) => { let _ = response.send(formatter.render_interactive_if_dirty()); } + OutputCommand::PanicRestored => { + formatter.mark_panic_restored(); + } } } _ = redraw_tick.tick(), if formatter.is_interactive_dashboard() => { @@ -8152,6 +8200,7 @@ impl OutputManager { tx, ready_prompt_active, tui_entered, + panic_restored, mode, console_session_mode: matches!(mode, LogFormat::Pretty).then_some(console_session_mode), dashboard_snapshot_provider, @@ -8260,6 +8309,21 @@ impl OutputManager { .unwrap_or(false) } + fn mark_panic_restored(&self) { + let tx = match self.state.read() { + Ok(state) => { + state.panic_restored.store(true, Ordering::Release); + state.tui_entered.store(false, Ordering::Release); + state.tx.clone() + } + Err(err) => { + tracing::warn!("output manager state lock poisoned during panic restore: {err}"); + return; + } + }; + let _ = tx.send(OutputCommand::PanicRestored); + } + pub fn register_dashboard_snapshot_provider( &self, provider: Arc, @@ -8520,13 +8584,14 @@ pub fn force_restore_tui_terminal() -> io::Result<()> { } pub fn force_restore_tui_after_panic() { - if !GLOBAL_OUTPUT_MANAGER - .get() - .is_some_and(|output_manager| output_manager.tui_entered()) - { + let Some(output_manager) = GLOBAL_OUTPUT_MANAGER.get() else { + return; + }; + if !output_manager.tui_entered() { return; } + output_manager.mark_panic_restored(); let _ = force_restore_tui_terminal(); let _ = disable_raw_mode(); } @@ -12610,6 +12675,30 @@ mod tests { )); } + #[test] + fn generic_error_does_not_guess_last_running_model() { + let mut state = DashboardState::default(); + state.reduce(DashboardAction::OutputEvent(OutputEvent::ModelReady { + model: "Qwen3-32B".to_string(), + internal_port: Some(9338), + role: Some("host".to_string()), + })); + + state.reduce(DashboardAction::OutputEvent(OutputEvent::Error { + message: "transport stderr surfaced".to_string(), + context: Some("stderr".to_string()), + })); + + assert!(matches!( + state + .running_models + .iter() + .find(|model| model.model == "Qwen3-32B") + .map(|model| &model.status), + Some(RuntimeStatus::Ready) + )); + } + #[test] fn discovery_and_join_failures_mark_startup_mesh_component_failed() { let mut discovery_failed = DashboardState::default(); @@ -13910,6 +13999,39 @@ tail line" assert!(!formatter.tui_entered()); } + #[test] + fn tui_panic_restore_disables_interactive_redraws() { + let tui_entered = Arc::new(AtomicBool::new(false)); + let panic_restored = Arc::new(AtomicBool::new(false)); + let mut formatter = InteractiveDashboardFormatter::with_tui_state( + tui_entered.clone(), + panic_restored.clone(), + ); + formatter.mark_terminal_escape_written(); + + formatter.mark_panic_restored(); + + assert!(!formatter.terminal_active); + assert!(!formatter.dirty); + assert!(!tui_entered.load(Ordering::Acquire)); + assert!(panic_restored.load(Ordering::Acquire)); + assert_eq!( + formatter + .handle_output_event(&OutputEvent::Shutdown { reason: None }) + .expect("panic-restored formatter should ignore output events"), + None + ); + assert_eq!( + formatter.handle_tui_event(TuiEvent::Key(TuiKeyEvent::Char('q'))), + TuiControlFlow::Continue + ); + assert!( + !formatter + .render_if_dirty() + .expect("panic-restored formatter should skip redraws") + ); + } + #[test] fn tui_narrow_terminal_renders_resize_guidance_instead_of_dashboard() { let mut state = DashboardState::default();