monolith

public

Star0

Fork 0

Watch0

202 commits 1 branch 0 tags 0 releases1.5 MiB

Commit `6dfb54e27e7f`

tarball

atagen <boss@atagen.co>2026-10-19 07:56 UTC

6 files changed+155-31

render-vk: report a lost or hung GPU instead of waiting on it forever

VK_ERROR_DEVICE_LOST became a generic Vulkan error or Interrupted, reap treated a failed fence query as signalled, and fence waits used u64::MAX or an unbounded poll, so a hung GPU froze the main thread. A From<vk::Result> now yields Error::DeviceLost and Error::Timeout, waits give up after three seconds, and device_lost() turns true after a lost device or three timeouts in a row.

Modified file: crates/render-vk/src/error.rs+89 additions−1 deletions@@ -1,12 +1,18 @@
 use ash::vk;
 use smithay::backend::allocator::Fourcc;
+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
+use std::time::Duration;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("failed to load the Vulkan loader: {0}")]
     Loading(#[from] ash::LoadingError),
     #[error("Vulkan call failed: {0}")]
-    Vulkan(#[from] vk::Result),
+    Vulkan(vk::Result),
+    #[error("the GPU was lost")]
+    DeviceLost,
+    #[error("the GPU did not finish in time")]
+    Timeout,
     #[error("GPU memory allocation failed: {0}")]
     Allocation(#[from] gpu_allocator::AllocationError),
     #[error("no Vulkan 1.3 device with dynamic rendering, synchronization2 and push descriptors")]
@@ -30,3 +36,85 @@ pub enum Error {
     #[error("invalid size {0}x{1}")]
     InvalidSize(i32, i32),
 }
+
+/// How long any wait for the GPU may last.
+pub(crate) const WAIT: Duration = Duration::from_secs(3);
+
+/// Consecutive timed out waits after which the GPU counts as hung.
+const HANG_AFTER: u32 = 3;
+
+/// Set once a call reports the device lost, and never cleared. A compositor
+/// has one device, so the state is process wide, which lets every `?` on a
+/// `vk::Result` record it without threading a handle through.
+static LOST: AtomicBool = AtomicBool::new(false);
+static HUNG: AtomicBool = AtomicBool::new(false);
+static TIMEOUTS: AtomicU32 = AtomicU32::new(0);
+
+/// Whether the GPU is gone or stopped answering, so rendering cannot go on.
+pub fn device_lost() -> bool {
+    LOST.load(Ordering::Relaxed) || HUNG.load(Ordering::Relaxed)
+}
+
+/// Whether waiting for the GPU timed out repeatedly. Tearing the device down
+/// then would wait for work that never ends.
+pub(crate) fn hung() -> bool {
+    HUNG.load(Ordering::Relaxed)
+}
+
+/// Records what a failed call says about the device.
+pub(crate) fn observe(result: vk::Result) {
+    match result {
+        vk::Result::ERROR_DEVICE_LOST => LOST.store(true, Ordering::Relaxed),
+        vk::Result::TIMEOUT => {
+            let timeouts = TIMEOUTS.fetch_add(1, Ordering::Relaxed) + 1;
+            HUNG.fetch_or(timeouts >= HANG_AFTER, Ordering::Relaxed);
+        }
+        _ => {}
+    }
+}
+
+/// Records that a wait for the GPU finished.
+pub(crate) fn waited() {
+    TIMEOUTS.store(0, Ordering::Relaxed);
+}
+
+impl From<vk::Result> for Error {
+    fn from(result: vk::Result) -> Self {
+        observe(result);
+        match result {
+            vk::Result::ERROR_DEVICE_LOST => Self::DeviceLost,
+            vk::Result::TIMEOUT => Self::Timeout,
+            other => Self::Vulkan(other),
+        }
+    }
+}
+
+/// What a failed `SyncPoint::wait`, which says nothing itself, meant.
+pub(crate) fn wait_error() -> Error {
+    if device_lost() {
+        Error::DeviceLost
+    } else {
+        Error::Interrupted
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn lost_device_and_repeated_timeouts_are_fatal() {
+        assert!(!device_lost());
+        assert!(matches!(Error::from(vk::Result::TIMEOUT), Error::Timeout));
+        waited();
+        assert!(matches!(Error::from(vk::Result::TIMEOUT), Error::Timeout));
+        assert!(matches!(Error::from(vk::Result::TIMEOUT), Error::Timeout));
+        assert!(!device_lost());
+        assert!(matches!(Error::from(vk::Result::TIMEOUT), Error::Timeout));
+        assert!(hung() && device_lost());
+        assert!(matches!(
+            Error::from(vk::Result::ERROR_DEVICE_LOST),
+            Error::DeviceLost
+        ));
+    }
+}

Modified file: crates/render-vk/src/frame.rs+1−1

@@ -302,7 +302,7 @@ impl Frame for VulkanFrame<'_, '_> {

302302 }

303303

304304 fn wait(&mut self, sync: &SyncPoint) -> Result<(), Error> {

305- sync.wait().map_err(|_| Error::Interrupted)

305+ sync.wait().map_err(|_| crate::error::wait_error())

306306 }

307307

308308 fn finish(mut self) -> Result<SyncPoint, Error> {

Modified file: crates/render-vk/src/lib.rs+1−1

@@ -23,7 +23,7 @@ mod sync;

2323 mod transform;

2424

2525 pub use dmabuf::VulkanAllocator;

26-pub use error::Error;

26+pub use error::{Error, device_lost};

2727 pub use frame::{Clip, Effect, Transition, VulkanFrame};

2828 pub use image::{VulkanImage, VulkanMapping, VulkanTarget, VulkanTexture};

2929 pub use renderer::{DmabufImporter, ShmStager, StagedShm, VulkanRenderer};

Modified file: crates/render-vk/src/renderer/mod.rs+23 additions−13 deletions@@ -209,19 +209,27 @@ impl VulkanRenderer {
         self.samplers[(idx(self.min_filter) << 1) | idx(self.mag_filter)]
     }
 
-    fn reap(&mut self) {
+    /// Recycles finished work. A fence that cannot be queried stays
+    /// pending and fails the call, since its work may never have finished.
+    fn reap(&mut self) -> Result<(), Error> {
         let spare = &mut self.spare_cmds;
-        self.pending.retain(|p| {
-            if !p.fence.signaled() {
-                return true;
+        let mut failed = None;
+        self.pending.retain(|p| match p.fence.status() {
+            Ok(false) => true,
+            Ok(true) => {
+                spare.push(p.cmd);
+                false
+            }
+            Err(e) => {
+                failed.get_or_insert(e);
+                true
             }
-            spare.push(p.cmd);
-            false
         });
         self.dmabuf_textures.retain(|k, _| !k.is_gone());
         self.dmabuf_targets.retain(|k, _| !k.is_gone());
         self.staged.retain(|_, (b, _)| b.is_alive());
         self.staging_pool.trim();
+        failed.map_or(Ok(()), |e| Err(e.into()))
     }
 
     fn begin(&mut self) -> Result<vk::CommandBuffer, Error> {
@@ -414,7 +422,7 @@ impl VulkanRenderer {
         target: &mut VulkanTarget<'_>,
         ops: Vec<Op>,
     ) -> Result<SyncPoint, Error> {
-        self.reap();
+        self.reap()?;
         let image = target.0.clone();
         let mut sampled: Vec<Arc<ImageInner>> = Vec::new();
         for op in &ops {
@@ -605,7 +613,7 @@ impl VulkanRenderer {
         if image.external {
             return Err(Error::ForeignTexture);
         }
-        self.reap();
+        self.reap()?;
         if region.size.w <= 0 || region.size.h <= 0 {
             return Ok(());
         }
@@ -749,7 +757,9 @@ impl Drop for VulkanRenderer {
     fn drop(&mut self) {
         let d = &self.shared.device.device;
         unsafe {
-            let _ = d.device_wait_idle();
+            if !crate::error::hung() {
+                let _ = d.device_wait_idle();
+            }
             for p in self.pending.drain(..) {
                 d.free_command_buffers(self.pool, &[p.cmd]);
             }
@@ -823,7 +833,7 @@ impl Renderer for VulkanRenderer {
         'buffer: 'frame,
     {
         self.own(&framebuffer.0)?;
-        self.reap();
+        self.reap()?;
         Ok(VulkanFrame::new(
             self,
             framebuffer,
@@ -833,16 +843,16 @@ impl Renderer for VulkanRenderer {
     }
 
     fn wait(&mut self, sync: &SyncPoint) -> Result<(), Error> {
-        sync.wait().map_err(|_| Error::Interrupted)
+        sync.wait().map_err(|_| crate::error::wait_error())
     }
 
     fn cleanup_texture_cache(&mut self) -> Result<(), Error> {
-        self.reap();
+        self.reap()?;
         Ok(())
     }
 
     fn invalidate_caches(&mut self) -> Result<(), Error> {
-        self.reap();
+        self.reap()?;
         self.dmabuf_textures.clear();
         self.dmabuf_targets.clear();
         Ok(())

Modified file: crates/render-vk/src/renderer/stall.rs+1−1

@@ -49,7 +49,7 @@ impl VulkanRenderer {

4949 /// right after a flip. NVIDIA stalls for milliseconds on every 1024th

5050 /// submission, see research/nvidia.md.

5151 pub fn absorb_submit_stall(&mut self) -> Result<(), Error> {

52- self.reap();

52+ self.reap()?;

5353 let Some(phase) = self.stall.get().phase else {

5454 return Ok(());

5555 };

Modified file: crates/render-vk/src/sync.rs+40 additions−14 deletions@@ -1,4 +1,5 @@
 use crate::device::Shared;
+use crate::error::{WAIT, observe, waited};
 use ash::vk;
 use rustix::event::{PollFd, PollFlags, poll};
 use smithay::backend::renderer::sync::{Fence, Interrupted};
@@ -55,36 +56,58 @@ impl FenceInner {
         self.fence
     }
 
-    pub fn signaled(&self) -> bool {
+    /// Whether the GPU has signalled, or the error the query failed with.
+    pub fn status(&self) -> Result<bool, vk::Result> {
         match &*self.state.lock().unwrap_or_else(|e| e.into_inner()) {
-            State::Pending => {
-                unsafe { self.shared.device.device.get_fence_status(self.fence) }.unwrap_or(true)
-            }
-            State::Exported(None) => true,
-            State::Exported(Some(fd)) => readable(fd, Some(&Default::default())),
+            State::Pending => unsafe { self.shared.device.device.get_fence_status(self.fence) },
+            State::Exported(None) => Ok(true),
+            State::Exported(Some(fd)) => Ok(readable(fd, Some(&Default::default()))),
         }
     }
 
+    /// Like `status` for callers that cannot report a failure. A failed query
+    /// counts as signalled so nobody waits on a fence that cannot signal.
+    pub fn signaled(&self) -> bool {
+        self.status().unwrap_or_else(|e| {
+            observe(e);
+            true
+        })
+    }
+
     /// Blocks until the GPU signals. A pending fence is first turned into its
     /// sync_file and polled, because waiting on the Vulkan fence itself sleeps
     /// in 10 ms steps on NVIDIA even when the work is done in microseconds.
+    /// Gives up with `TIMEOUT` after [`WAIT`], so a hung GPU cannot freeze the
+    /// caller.
     pub fn wait(&self) -> Result<(), vk::Result> {
         if self.exportable {
             drop(self.export());
         }
-        match &*self.state.lock().unwrap_or_else(|e| e.into_inner()) {
+        let waited_for = match &*self.state.lock().unwrap_or_else(|e| e.into_inner()) {
             State::Pending => unsafe {
-                self.shared
-                    .device
-                    .device
-                    .wait_for_fences(&[self.fence], true, u64::MAX)
+                self.shared.device.device.wait_for_fences(
+                    &[self.fence],
+                    true,
+                    WAIT.as_nanos() as u64,
+                )
             },
             State::Exported(None) => Ok(()),
             State::Exported(Some(fd)) => {
-                readable(fd, None);
-                Ok(())
+                let timeout = rustix::event::Timespec {
+                    tv_sec: WAIT.as_secs() as _,
+                    tv_nsec: 0,
+                };
+                if readable(fd, Some(&timeout)) {
+                    Ok(())
+                } else {
+                    Err(vk::Result::TIMEOUT)
+                }
             }
+        };
+        if waited_for.is_ok() {
+            waited();
         }
+        waited_for
     }
 
     #[profiling::function]
@@ -157,7 +180,10 @@ impl Fence for VkFence {
     }
 
     fn wait(&self) -> Result<(), Interrupted> {
-        self.0.wait().map_err(|_| Interrupted)
+        self.0.wait().map_err(|e| {
+            observe(e);
+            Interrupted
+        })
     }
 
     fn is_exportable(&self) -> bool {

monolith

Commit 6dfb54e27e7f

Commit `6dfb54e27e7f`