linux: Implement GPU device loss recovery for wgpu

crates/gpui_linux/src/linux/wayland/window.rs

@@ -10,6 +10,7 @@ use collections::{FxHashSet, HashMap};
 use futures::channel::oneshot::Receiver;
 
 use raw_window_handle as rwh;
+use util::ResultExt;
 use wayland_backend::client::ObjectId;
 use wayland_client::WEnum;
 use wayland_client::{Proxy, protocol::wl_surface};
@@ -1306,7 +1307,7 @@ impl PlatformWindow for WaylandWindow {
 
     fn draw(&self, scene: &Scene) {
         let mut state = self.borrow_mut();
-        state.renderer.draw(scene);
+        state.renderer.draw(scene).log_err();
     }
 
     fn completed_frame(&self) {

crates/gpui_linux/src/linux/x11/client.rs

@@ -227,26 +227,146 @@ impl X11ClientStatePtr {
         self.0.upgrade().map(X11Client)
     }
 
-    pub fn drop_window(&self, x_window: u32) {
+    /// Updates the GPU context for all windows after recovery
+    pub(crate) fn update_gpu_context(&self, context: crate::platform::wgpu::WgpuContext) {
+        if let Some(client) = self.get_client() {
+            client.0.borrow_mut().gpu_context = context;
+        }
+    }
+
+    /// Orchestrates full GPU device recovery across all windows
+    pub(crate) fn recover_gpu(&self) -> anyhow::Result<()> {
+        use crate::platform::wgpu::{WgpuContext, WgpuRenderer, WgpuSurfaceConfig};
+        use anyhow::anyhow;
+        use std::sync::{Arc, mpsc};
+        use std::time::Duration;
+
+        let Some(client) = self.get_client() else {
+            return Err(anyhow!("Client state unavailable during GPU recovery"));
+        };
+
+        log::info!("Starting GPU recovery...");
+
+        let windows: Vec<_> = {
+            let state = client.0.borrow();
+            state.windows
+                .values()
+                .map(|window_ref| window_ref.window.clone())
+                .collect()
+        };
+
+        log::info!("Found {} windows to recover", windows.len());
+
+        for window in &windows {
+            window.pause_rendering();
+        }
+        log::debug!("Paused rendering on all windows");
+
+        // This prevents the main thread from hanging if GPU init fails
+        let (sender, receiver) = mpsc::channel();
+        std::thread::spawn(move || {
+            log::debug!("Creating new GPU context...");
+            sender.send(WgpuContext::new()).log_err();
+        });
+
+        let new_context = receiver
+            .recv_timeout(Duration::from_secs(10))
+            .map_err(|_| anyhow!("GPU context creation timed out after 10 seconds"))??;
+
+        log::info!(
+            "Created new GPU context with adapter: {:?}",
+            new_context.adapter.get_info().name
+        );
+
+        let atlases: Vec<_> = windows.iter().map(|window| window.get_atlas()).collect();
+        log::debug!("Saved {} atlases", atlases.len());
+
+        for window in &windows {
+            window.prepare_atlas();
+        }
+        log::debug!("Prepared atlases for recovery");
+
+        // In wgpu, surfaces are RAII, but we still mark them as invalid
+        for window in &windows {
+            window.destroy_surface();
+        }
+        log::debug!("Destroyed old surfaces");
+
+        let renderer_params: Vec<_> = windows
+            .iter()
+            .map(|window| window.renderer_params())
+            .collect();
+
+        let (sender, receiver) = mpsc::channel();
+        let device = Arc::clone(&new_context.device);
+        let queue = Arc::clone(&new_context.queue);
+        let instance = new_context.instance.clone();
+        let adapter = new_context.adapter.clone();
+
+        std::thread::spawn(move || {
+            log::debug!("Creating {} new renderers...", renderer_params.len());
+            let result: anyhow::Result<Vec<WgpuRenderer>> = renderer_params
+                .into_iter()
+                .enumerate()
+                .map(|(index, (raw_window, size, transparent))| {
+                    log::trace!("Creating renderer {} (size: {:?})", index, size);
+                    let config = WgpuSurfaceConfig { size, transparent };
+                    WgpuRenderer::new_with_device_queue(
+                        instance.clone(),
+                        adapter.clone(),
+                        Arc::clone(&device),
+                        Arc::clone(&queue),
+                        &raw_window,
+                        config,
+                    )
+                })
+                .collect();
+            sender.send(result).log_err();
+        });
+
+        let new_renderers = receiver
+            .recv_timeout(Duration::from_secs(10))
+            .map_err(|_| anyhow!("Renderer creation timed out after 10 seconds"))??;
+
+        log::info!("Created {} new renderers", new_renderers.len());
+
+        for ((window, renderer), atlas) in windows.iter().zip(new_renderers).zip(atlases) {
+            window.replace_renderer(renderer, &atlas);
+        }
+        log::debug!("Replaced renderers and adopted atlases");
+
+        self.update_gpu_context(new_context);
+        log::debug!("Updated client GPU context");
+
+        for window in &windows {
+            window.resume_rendering();
+        }
+        log::debug!("Resumed rendering on all windows");
+
+        log::info!("GPU recovery successful for {} windows", windows.len());
+        Ok(())
+    }
+
+    pub fn drop_window(&self, window_id: u32) {
         let Some(client) = self.get_client() else {
             return;
         };
         let mut state = client.0.borrow_mut();
 
-        if let Some(window_ref) = state.windows.remove(&x_window)
+        if let Some(window_ref) = state.windows.remove(&window_id)
             && let Some(RefreshState::PeriodicRefresh {
                 event_loop_token, ..
             }) = window_ref.refresh_state
         {
             state.loop_handle.remove(event_loop_token);
         }
-        if state.mouse_focused_window == Some(x_window) {
+        if state.mouse_focused_window == Some(window_id) {
             state.mouse_focused_window = None;
         }
-        if state.keyboard_focused_window == Some(x_window) {
+        if state.keyboard_focused_window == Some(window_id) {
             state.keyboard_focused_window = None;
         }
-        state.cursor_styles.remove(&x_window);
+        state.cursor_styles.remove(&window_id);
     }
 
     pub fn update_ime_position(&self, bounds: Bounds<Pixels>) {
@@ -779,6 +899,14 @@ impl X11Client {
                     drop(state);
                     window.close();
                     state = self.0.borrow_mut();
+                } else if atom == state.atoms._GPUI_FORCE_UPDATE_WINDOW {
+                    window.resume_rendering();
+                    drop(state);
+                    window.refresh(crate::RequestFrameOptions {
+                        force_render: true,
+                        require_presentation: false,
+                    });
+                    return Some(());
                 } else if atom == state.atoms._NET_WM_SYNC_REQUEST {
                     window.state.borrow_mut().last_sync_counter =
                         Some(x11rb::protocol::sync::Int64 {

crates/gpui_linux/src/linux/x11/window.rs

@@ -78,6 +78,7 @@ x11rb::atom_manager! {
         _NET_WM_SYNC,
         _NET_SUPPORTED,
         _MOTIF_WM_HINTS,
+        _GPUI_FORCE_UPDATE_WINDOW,
         _GTK_SHOW_WINDOW_MENU,
         _GTK_FRAME_EXTENTS,
         _GTK_EDGE_CONSTRAINTS,
@@ -225,7 +226,7 @@ fn find_visuals(xcb: &XCBConnection, screen_index: usize) -> VisualSet {
     set
 }
 
-struct RawWindow {
+pub(crate) struct RawWindow {
     connection: *mut c_void,
     screen_id: usize,
     window_id: u32,
@@ -916,9 +917,126 @@ impl X11Window {
         xcb_flush(&self.0.xcb);
         Ok(())
     }
+
+    /// Handles draw failures by attempting GPU recovery
+    fn handle_draw_failure(&self, error: anyhow::Error) {
+        log::error!("Renderer draw failed: {}", error);
+
+        let inner = self.0.state.borrow();
+        let client = inner.client.clone();
+        let force_update_atom = inner.atoms._GPUI_FORCE_UPDATE_WINDOW;
+        drop(inner);
+
+        // Attempt GPU recovery
+        if let Err(recovery_error) = client.recover_gpu() {
+            // Recovery failed - this is unrecoverable, panic to trigger crash reporting
+            panic!(
+                "GPU device lost (recovery failed: {}), original error: {}",
+                recovery_error, error
+            );
+        }
+
+        // Recovery succeeded - send force update to trigger redraw
+        self.send_force_update(force_update_atom);
+    }
+
+    /// Sends a force update event to trigger window redraw after GPU recovery
+    fn send_force_update(&self, force_update_atom: xproto::Atom) {
+        let message = ClientMessageEvent::new(
+            32,
+            self.0.x_window,
+            force_update_atom,
+            [0, 0, 0, 0, 0],
+        );
+
+        check_reply(
+            || "X11 SendEvent for GPU recovery force update failed",
+            self.0.xcb.send_event(
+                false,
+                self.0.x_window,
+                EventMask::default(),
+                message,
+            ),
+        )
+        .log_err();
+
+        xcb_flush(&self.0.xcb);
+    }
 }
 
 impl X11WindowStatePtr {
+    /// Pauses rendering (sets skip_draws flag)
+    pub(crate) fn pause_rendering(&self) {
+        let mut state = self.state.borrow_mut();
+        state.renderer.pause_rendering();
+    }
+
+    /// Resumes rendering (clears skip_draws flag)
+    pub(crate) fn resume_rendering(&self) {
+        let mut state = self.state.borrow_mut();
+        state.renderer.resume_rendering();
+    }
+
+    /// Destroys the surface (marks it as invalid for wgpu)
+    pub(crate) fn destroy_surface(&self) {
+        let mut state = self.state.borrow_mut();
+        state.renderer.destroy_surface();
+    }
+
+    /// Gets renderer parameters needed for recreation
+    pub(crate) fn renderer_params(
+        &self,
+    ) -> (
+        RawWindow,
+        crate::Size<crate::DevicePixels>,
+        bool,
+    ) {
+        let state = self.state.borrow();
+        let screen_index = state.display.id().0 as usize;
+        let visual_set = find_visuals(&self.xcb, screen_index);
+        let visual = visual_set.transparent.unwrap_or(visual_set.inherit);
+
+        let raw_window = RawWindow {
+            connection: self.xcb.get_raw_xcb_connection(),
+            screen_id: screen_index,
+            window_id: self.x_window,
+            visual_id: visual.id,
+        };
+
+        let size = state.bounds.size;
+        let device_size = crate::Size {
+            width: crate::DevicePixels((size.width.0 * state.scale_factor) as i32),
+            height: crate::DevicePixels((size.height.0 * state.scale_factor) as i32),
+        };
+
+        let transparent = state.is_transparent();
+
+        (raw_window, device_size, transparent)
+    }
+
+    /// Gets the current atlas (for preservation during recovery)
+    pub(crate) fn get_atlas(&self) -> std::sync::Arc<crate::platform::wgpu::WgpuAtlas> {
+        let state = self.state.borrow();
+        std::sync::Arc::clone(state.renderer.sprite_atlas())
+    }
+
+    /// Prepares atlas for recovery (clears without destroying)
+    pub(crate) fn prepare_atlas(&self) {
+        let state = self.state.borrow();
+        state.renderer.prepare_atlas();
+    }
+
+    /// Replaces the renderer with a new one and adopts the atlas
+    pub(crate) fn replace_renderer(
+        &self,
+        mut new_renderer: crate::platform::wgpu::WgpuRenderer,
+        atlas: &std::sync::Arc<crate::platform::wgpu::WgpuAtlas>,
+    ) {
+        new_renderer.adopt_atlas(atlas);
+        let mut state = self.state.borrow_mut();
+        state.renderer = new_renderer;
+    }
+
     pub fn should_close(&self) -> bool {
         let mut cb = self.callbacks.borrow_mut();
         if let Some(mut should_close) = cb.should_close.take() {
@@ -1558,7 +1676,10 @@ impl PlatformWindow for X11Window {
 
     fn draw(&self, scene: &Scene) {
         let mut inner = self.0.state.borrow_mut();
-        inner.renderer.draw(scene);
+        if let Err(error) = inner.renderer.draw(scene) {
+            drop(inner);
+            self.handle_draw_failure(error);
+        }
     }
 
     fn sprite_atlas(&self) -> Arc<dyn PlatformAtlas> {