e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 1 | use crate::Error; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 2 | use crate::device::Shared; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 3 | use ash::vk; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 4 | use gpu_allocator::MemoryLocation; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 5 | use gpu_allocator::vulkan::{Allocation, AllocationCreateDesc, AllocationScheme}; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 6 | use std::sync::{Arc, Mutex}; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 7 | use std::time::{Duration, Instant}; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 8 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 9 | /// Pooled staging memory kept at most, past which dropped buffers are freed. |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 10 | const POOL_BYTES: usize = 32 * 1024 * 1024; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 11 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 12 | /// How long a pooled buffer may go unused before it is freed. |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 13 | const POOL_IDLE: Duration = Duration::from_secs(2); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 14 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 15 | struct Pooled { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 16 | buffer: vk::Buffer, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 17 | allocation: Allocation, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 18 | size: usize, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 19 | usage: vk::BufferUsageFlags, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 20 | location: MemoryLocation, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 21 | returned: Instant, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 22 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 23 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 24 | /// Staging buffers kept for reuse, see [`Staging`]. |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 25 | pub(crate) struct StagingPool { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 26 | shared: Arc<Shared>, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 27 | free: Mutex<Vec<Pooled>>, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 28 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 29 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 30 | impl StagingPool { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 31 | pub fn new(shared: &Arc<Shared>) -> Arc<Self> { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 32 | Arc::new(Self { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 33 | shared: shared.clone(), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 34 | free: Mutex::new(Vec::new()), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 35 | }) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 36 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 37 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 38 | /// Frees pooled buffers that have sat unused for [`POOL_IDLE`]. |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 39 | pub fn trim(&self) { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 40 | let now = Instant::now(); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 41 | let expired: Vec<Pooled> = self |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 42 | .free |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 43 | .lock() |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 44 | .unwrap_or_else(|e| e.into_inner()) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 45 | .extract_if(.., |p| now - p.returned >= POOL_IDLE) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 46 | .collect(); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 47 | self.destroy(expired.into_iter().map(|p| (p.buffer, p.allocation))); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 48 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 49 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 50 | fn destroy(&self, buffers: impl IntoIterator<Item = (vk::Buffer, Allocation)>) { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 51 | let mut buffers = buffers.into_iter().peekable(); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 52 | if buffers.peek().is_none() { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 53 | return; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 54 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 55 | let mut allocator = self |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 56 | .shared |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 57 | .allocator |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 58 | .lock() |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 59 | .unwrap_or_else(|e| e.into_inner()); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 60 | for (buffer, allocation) in buffers { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 61 | unsafe { self.shared.device.device.destroy_buffer(buffer, None) }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 62 | let _ = allocator.free(allocation); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 63 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 64 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 65 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 66 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 67 | impl Drop for StagingPool { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 68 | fn drop(&mut self) { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 69 | let free = std::mem::take(self.free.get_mut().unwrap_or_else(|e| e.into_inner())); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 70 | self.destroy(free.into_iter().map(|p| (p.buffer, p.allocation))); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 71 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 72 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 73 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 74 | /// A host-visible buffer for uploads and readbacks. Dropped ones go back to |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 75 | /// a pool instead of being freed. Allocating and freeing large host-visible |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 76 | /// memory takes a driver lock on NVIDIA that `vkQueueSubmit` also waits on, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 77 | /// so a staging buffer made per upload on the import worker stalled the main |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 78 | /// thread's submits for 1 to 3 ms. |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 79 | pub(crate) struct Staging { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 80 | pub pool: Arc<StagingPool>, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 81 | pub buffer: vk::Buffer, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 82 | allocation: Option<Allocation>, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 83 | size: usize, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 84 | usage: vk::BufferUsageFlags, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 85 | location: MemoryLocation, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 86 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 87 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 88 | impl Staging { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 89 | pub fn new( |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 90 | pool: &Arc<StagingPool>, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 91 | size: usize, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 92 | usage: vk::BufferUsageFlags, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 93 | location: MemoryLocation, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 94 | ) -> Result<Self, Error> { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 95 | let reused = { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 96 | let mut free = pool.free.lock().unwrap_or_else(|e| e.into_inner()); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 97 | let fits = |p: &Pooled| { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 98 | p.usage == usage && p.location == location && p.size >= size && p.size <= size * 2 |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 99 | }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 100 | free.iter().position(fits).map(|i| free.swap_remove(i)) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 101 | }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 102 | if let Some(p) = reused { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 103 | return Ok(Self { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 104 | pool: pool.clone(), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 105 | buffer: p.buffer, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 106 | allocation: Some(p.allocation), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 107 | size: p.size, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 108 | usage, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 109 | location, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 110 | }); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 111 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 112 | let d = &pool.shared.device.device; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 113 | let buffer = unsafe { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 114 | d.create_buffer( |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 115 | &vk::BufferCreateInfo::default() |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 116 | .size(size as u64) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 117 | .usage(usage), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 118 | None, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 119 | )? |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 120 | }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 121 | let requirements = unsafe { d.get_buffer_memory_requirements(buffer) }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 122 | let allocation = pool |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 123 | .shared |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 124 | .allocator |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 125 | .lock() |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 126 | .unwrap_or_else(|e| e.into_inner()) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 127 | .allocate(&AllocationCreateDesc { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 128 | name: "staging", |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 129 | requirements, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 130 | location, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 131 | linear: true, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 132 | allocation_scheme: AllocationScheme::GpuAllocatorManaged, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 133 | }); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 134 | let allocation = match allocation { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 135 | Ok(a) => a, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 136 | Err(e) => { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 137 | unsafe { d.destroy_buffer(buffer, None) }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 138 | return Err(e.into()); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 139 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 140 | }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 141 | unsafe { d.bind_buffer_memory(buffer, allocation.memory(), allocation.offset())? }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 142 | Ok(Self { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 143 | pool: pool.clone(), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 144 | buffer, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 145 | allocation: Some(allocation), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 146 | size, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 147 | usage, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 148 | location, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 149 | }) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 150 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 151 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 152 | pub fn bytes(&self) -> &[u8] { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 153 | self.allocation |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 154 | .as_ref() |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 155 | .and_then(|a| a.mapped_slice()) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 156 | .unwrap_or(&[]) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 157 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 158 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 159 | pub fn bytes_mut(&mut self) -> &mut [u8] { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 160 | self.allocation |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 161 | .as_mut() |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 162 | .and_then(|a| a.mapped_slice_mut()) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 163 | .unwrap_or(&mut []) |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 164 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 165 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 166 | |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 167 | impl Drop for Staging { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 168 | fn drop(&mut self) { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 169 | let Some(allocation) = self.allocation.take() else { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 170 | return; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 171 | }; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 172 | let mut free = self.pool.free.lock().unwrap_or_else(|e| e.into_inner()); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 173 | let pooled: usize = free.iter().map(|p| p.size).sum(); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 174 | if pooled + self.size <= POOL_BYTES { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 175 | free.push(Pooled { |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 176 | buffer: self.buffer, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 177 | allocation, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 178 | size: self.size, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 179 | usage: self.usage, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 180 | location: self.location, |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 181 | returned: Instant::now(), |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 182 | }); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 183 | return; |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 184 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 185 | drop(free); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 186 | self.pool.destroy([(self.buffer, allocation)]); |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 187 | } |
e0b486d6097f | atagen | 2026-10-19 07:26 UTC | 188 | } |