atagen

monolith

public
Star0
Watch0
43e0e8e33c3e08f47bcca32dfe53708e351b36ea
1
ssh://brewery@pls.lobotomise.me:33/atagen/monolith.git

Blame: crates/render-vk/src/staging.rs

e0b486d6097fatagen2026-10-19 07:26 UTC1
use crate::Error;
e0b486d6097fatagen2026-10-19 07:26 UTC2
use crate::device::Shared;
e0b486d6097fatagen2026-10-19 07:26 UTC3
use ash::vk;
e0b486d6097fatagen2026-10-19 07:26 UTC4
use gpu_allocator::MemoryLocation;
e0b486d6097fatagen2026-10-19 07:26 UTC5
use gpu_allocator::vulkan::{Allocation, AllocationCreateDesc, AllocationScheme};
e0b486d6097fatagen2026-10-19 07:26 UTC6
use std::sync::{Arc, Mutex};
e0b486d6097fatagen2026-10-19 07:26 UTC7
use std::time::{Duration, Instant};
e0b486d6097fatagen2026-10-19 07:26 UTC8
e0b486d6097fatagen2026-10-19 07:26 UTC9
/// Pooled staging memory kept at most, past which dropped buffers are freed.
e0b486d6097fatagen2026-10-19 07:26 UTC10
const POOL_BYTES: usize = 32 * 1024 * 1024;
e0b486d6097fatagen2026-10-19 07:26 UTC11
e0b486d6097fatagen2026-10-19 07:26 UTC12
/// How long a pooled buffer may go unused before it is freed.
e0b486d6097fatagen2026-10-19 07:26 UTC13
const POOL_IDLE: Duration = Duration::from_secs(2);
e0b486d6097fatagen2026-10-19 07:26 UTC14
e0b486d6097fatagen2026-10-19 07:26 UTC15
struct Pooled {
e0b486d6097fatagen2026-10-19 07:26 UTC16
    buffer: vk::Buffer,
e0b486d6097fatagen2026-10-19 07:26 UTC17
    allocation: Allocation,
e0b486d6097fatagen2026-10-19 07:26 UTC18
    size: usize,
e0b486d6097fatagen2026-10-19 07:26 UTC19
    usage: vk::BufferUsageFlags,
e0b486d6097fatagen2026-10-19 07:26 UTC20
    location: MemoryLocation,
e0b486d6097fatagen2026-10-19 07:26 UTC21
    returned: Instant,
e0b486d6097fatagen2026-10-19 07:26 UTC22
}
e0b486d6097fatagen2026-10-19 07:26 UTC23
e0b486d6097fatagen2026-10-19 07:26 UTC24
/// Staging buffers kept for reuse, see [`Staging`].
e0b486d6097fatagen2026-10-19 07:26 UTC25
pub(crate) struct StagingPool {
e0b486d6097fatagen2026-10-19 07:26 UTC26
    shared: Arc<Shared>,
e0b486d6097fatagen2026-10-19 07:26 UTC27
    free: Mutex<Vec<Pooled>>,
e0b486d6097fatagen2026-10-19 07:26 UTC28
}
e0b486d6097fatagen2026-10-19 07:26 UTC29
e0b486d6097fatagen2026-10-19 07:26 UTC30
impl StagingPool {
e0b486d6097fatagen2026-10-19 07:26 UTC31
    pub fn new(shared: &Arc<Shared>) -> Arc<Self> {
e0b486d6097fatagen2026-10-19 07:26 UTC32
        Arc::new(Self {
e0b486d6097fatagen2026-10-19 07:26 UTC33
            shared: shared.clone(),
e0b486d6097fatagen2026-10-19 07:26 UTC34
            free: Mutex::new(Vec::new()),
e0b486d6097fatagen2026-10-19 07:26 UTC35
        })
e0b486d6097fatagen2026-10-19 07:26 UTC36
    }
e0b486d6097fatagen2026-10-19 07:26 UTC37
e0b486d6097fatagen2026-10-19 07:26 UTC38
    /// Frees pooled buffers that have sat unused for [`POOL_IDLE`].
e0b486d6097fatagen2026-10-19 07:26 UTC39
    pub fn trim(&self) {
e0b486d6097fatagen2026-10-19 07:26 UTC40
        let now = Instant::now();
e0b486d6097fatagen2026-10-19 07:26 UTC41
        let expired: Vec<Pooled> = self
e0b486d6097fatagen2026-10-19 07:26 UTC42
            .free
e0b486d6097fatagen2026-10-19 07:26 UTC43
            .lock()
e0b486d6097fatagen2026-10-19 07:26 UTC44
            .unwrap_or_else(|e| e.into_inner())
e0b486d6097fatagen2026-10-19 07:26 UTC45
            .extract_if(.., |p| now - p.returned >= POOL_IDLE)
e0b486d6097fatagen2026-10-19 07:26 UTC46
            .collect();
e0b486d6097fatagen2026-10-19 07:26 UTC47
        self.destroy(expired.into_iter().map(|p| (p.buffer, p.allocation)));
e0b486d6097fatagen2026-10-19 07:26 UTC48
    }
e0b486d6097fatagen2026-10-19 07:26 UTC49
e0b486d6097fatagen2026-10-19 07:26 UTC50
    fn destroy(&self, buffers: impl IntoIterator<Item = (vk::Buffer, Allocation)>) {
e0b486d6097fatagen2026-10-19 07:26 UTC51
        let mut buffers = buffers.into_iter().peekable();
e0b486d6097fatagen2026-10-19 07:26 UTC52
        if buffers.peek().is_none() {
e0b486d6097fatagen2026-10-19 07:26 UTC53
            return;
e0b486d6097fatagen2026-10-19 07:26 UTC54
        }
e0b486d6097fatagen2026-10-19 07:26 UTC55
        let mut allocator = self
e0b486d6097fatagen2026-10-19 07:26 UTC56
            .shared
e0b486d6097fatagen2026-10-19 07:26 UTC57
            .allocator
e0b486d6097fatagen2026-10-19 07:26 UTC58
            .lock()
e0b486d6097fatagen2026-10-19 07:26 UTC59
            .unwrap_or_else(|e| e.into_inner());
e0b486d6097fatagen2026-10-19 07:26 UTC60
        for (buffer, allocation) in buffers {
e0b486d6097fatagen2026-10-19 07:26 UTC61
            unsafe { self.shared.device.device.destroy_buffer(buffer, None) };
e0b486d6097fatagen2026-10-19 07:26 UTC62
            let _ = allocator.free(allocation);
e0b486d6097fatagen2026-10-19 07:26 UTC63
        }
e0b486d6097fatagen2026-10-19 07:26 UTC64
    }
e0b486d6097fatagen2026-10-19 07:26 UTC65
}
e0b486d6097fatagen2026-10-19 07:26 UTC66
e0b486d6097fatagen2026-10-19 07:26 UTC67
impl Drop for StagingPool {
e0b486d6097fatagen2026-10-19 07:26 UTC68
    fn drop(&mut self) {
e0b486d6097fatagen2026-10-19 07:26 UTC69
        let free = std::mem::take(self.free.get_mut().unwrap_or_else(|e| e.into_inner()));
e0b486d6097fatagen2026-10-19 07:26 UTC70
        self.destroy(free.into_iter().map(|p| (p.buffer, p.allocation)));
e0b486d6097fatagen2026-10-19 07:26 UTC71
    }
e0b486d6097fatagen2026-10-19 07:26 UTC72
}
e0b486d6097fatagen2026-10-19 07:26 UTC73
e0b486d6097fatagen2026-10-19 07:26 UTC74
/// A host-visible buffer for uploads and readbacks. Dropped ones go back to
e0b486d6097fatagen2026-10-19 07:26 UTC75
/// a pool instead of being freed. Allocating and freeing large host-visible
e0b486d6097fatagen2026-10-19 07:26 UTC76
/// memory takes a driver lock on NVIDIA that `vkQueueSubmit` also waits on,
e0b486d6097fatagen2026-10-19 07:26 UTC77
/// so a staging buffer made per upload on the import worker stalled the main
e0b486d6097fatagen2026-10-19 07:26 UTC78
/// thread's submits for 1 to 3 ms.
e0b486d6097fatagen2026-10-19 07:26 UTC79
pub(crate) struct Staging {
e0b486d6097fatagen2026-10-19 07:26 UTC80
    pub pool: Arc<StagingPool>,
e0b486d6097fatagen2026-10-19 07:26 UTC81
    pub buffer: vk::Buffer,
e0b486d6097fatagen2026-10-19 07:26 UTC82
    allocation: Option<Allocation>,
e0b486d6097fatagen2026-10-19 07:26 UTC83
    size: usize,
e0b486d6097fatagen2026-10-19 07:26 UTC84
    usage: vk::BufferUsageFlags,
e0b486d6097fatagen2026-10-19 07:26 UTC85
    location: MemoryLocation,
e0b486d6097fatagen2026-10-19 07:26 UTC86
}
e0b486d6097fatagen2026-10-19 07:26 UTC87
e0b486d6097fatagen2026-10-19 07:26 UTC88
impl Staging {
e0b486d6097fatagen2026-10-19 07:26 UTC89
    pub fn new(
e0b486d6097fatagen2026-10-19 07:26 UTC90
        pool: &Arc<StagingPool>,
e0b486d6097fatagen2026-10-19 07:26 UTC91
        size: usize,
e0b486d6097fatagen2026-10-19 07:26 UTC92
        usage: vk::BufferUsageFlags,
e0b486d6097fatagen2026-10-19 07:26 UTC93
        location: MemoryLocation,
e0b486d6097fatagen2026-10-19 07:26 UTC94
    ) -> Result<Self, Error> {
e0b486d6097fatagen2026-10-19 07:26 UTC95
        let reused = {
e0b486d6097fatagen2026-10-19 07:26 UTC96
            let mut free = pool.free.lock().unwrap_or_else(|e| e.into_inner());
e0b486d6097fatagen2026-10-19 07:26 UTC97
            let fits = |p: &Pooled| {
e0b486d6097fatagen2026-10-19 07:26 UTC98
                p.usage == usage && p.location == location && p.size >= size && p.size <= size * 2
e0b486d6097fatagen2026-10-19 07:26 UTC99
            };
e0b486d6097fatagen2026-10-19 07:26 UTC100
            free.iter().position(fits).map(|i| free.swap_remove(i))
e0b486d6097fatagen2026-10-19 07:26 UTC101
        };
e0b486d6097fatagen2026-10-19 07:26 UTC102
        if let Some(p) = reused {
e0b486d6097fatagen2026-10-19 07:26 UTC103
            return Ok(Self {
e0b486d6097fatagen2026-10-19 07:26 UTC104
                pool: pool.clone(),
e0b486d6097fatagen2026-10-19 07:26 UTC105
                buffer: p.buffer,
e0b486d6097fatagen2026-10-19 07:26 UTC106
                allocation: Some(p.allocation),
e0b486d6097fatagen2026-10-19 07:26 UTC107
                size: p.size,
e0b486d6097fatagen2026-10-19 07:26 UTC108
                usage,
e0b486d6097fatagen2026-10-19 07:26 UTC109
                location,
e0b486d6097fatagen2026-10-19 07:26 UTC110
            });
e0b486d6097fatagen2026-10-19 07:26 UTC111
        }
e0b486d6097fatagen2026-10-19 07:26 UTC112
        let d = &pool.shared.device.device;
e0b486d6097fatagen2026-10-19 07:26 UTC113
        let buffer = unsafe {
e0b486d6097fatagen2026-10-19 07:26 UTC114
            d.create_buffer(
e0b486d6097fatagen2026-10-19 07:26 UTC115
                &vk::BufferCreateInfo::default()
e0b486d6097fatagen2026-10-19 07:26 UTC116
                    .size(size as u64)
e0b486d6097fatagen2026-10-19 07:26 UTC117
                    .usage(usage),
e0b486d6097fatagen2026-10-19 07:26 UTC118
                None,
e0b486d6097fatagen2026-10-19 07:26 UTC119
            )?
e0b486d6097fatagen2026-10-19 07:26 UTC120
        };
e0b486d6097fatagen2026-10-19 07:26 UTC121
        let requirements = unsafe { d.get_buffer_memory_requirements(buffer) };
e0b486d6097fatagen2026-10-19 07:26 UTC122
        let allocation = pool
e0b486d6097fatagen2026-10-19 07:26 UTC123
            .shared
e0b486d6097fatagen2026-10-19 07:26 UTC124
            .allocator
e0b486d6097fatagen2026-10-19 07:26 UTC125
            .lock()
e0b486d6097fatagen2026-10-19 07:26 UTC126
            .unwrap_or_else(|e| e.into_inner())
e0b486d6097fatagen2026-10-19 07:26 UTC127
            .allocate(&AllocationCreateDesc {
e0b486d6097fatagen2026-10-19 07:26 UTC128
                name: "staging",
e0b486d6097fatagen2026-10-19 07:26 UTC129
                requirements,
e0b486d6097fatagen2026-10-19 07:26 UTC130
                location,
e0b486d6097fatagen2026-10-19 07:26 UTC131
                linear: true,
e0b486d6097fatagen2026-10-19 07:26 UTC132
                allocation_scheme: AllocationScheme::GpuAllocatorManaged,
e0b486d6097fatagen2026-10-19 07:26 UTC133
            });
e0b486d6097fatagen2026-10-19 07:26 UTC134
        let allocation = match allocation {
e0b486d6097fatagen2026-10-19 07:26 UTC135
            Ok(a) => a,
e0b486d6097fatagen2026-10-19 07:26 UTC136
            Err(e) => {
e0b486d6097fatagen2026-10-19 07:26 UTC137
                unsafe { d.destroy_buffer(buffer, None) };
e0b486d6097fatagen2026-10-19 07:26 UTC138
                return Err(e.into());
e0b486d6097fatagen2026-10-19 07:26 UTC139
            }
e0b486d6097fatagen2026-10-19 07:26 UTC140
        };
e0b486d6097fatagen2026-10-19 07:26 UTC141
        unsafe { d.bind_buffer_memory(buffer, allocation.memory(), allocation.offset())? };
e0b486d6097fatagen2026-10-19 07:26 UTC142
        Ok(Self {
e0b486d6097fatagen2026-10-19 07:26 UTC143
            pool: pool.clone(),
e0b486d6097fatagen2026-10-19 07:26 UTC144
            buffer,
e0b486d6097fatagen2026-10-19 07:26 UTC145
            allocation: Some(allocation),
e0b486d6097fatagen2026-10-19 07:26 UTC146
            size,
e0b486d6097fatagen2026-10-19 07:26 UTC147
            usage,
e0b486d6097fatagen2026-10-19 07:26 UTC148
            location,
e0b486d6097fatagen2026-10-19 07:26 UTC149
        })
e0b486d6097fatagen2026-10-19 07:26 UTC150
    }
e0b486d6097fatagen2026-10-19 07:26 UTC151
acf5e9848529atagen2026-10-19 07:27 UTC152
    pub fn bytes(&self) -> Result<&[u8], Error> {
e0b486d6097fatagen2026-10-19 07:26 UTC153
        self.allocation
e0b486d6097fatagen2026-10-19 07:26 UTC154
            .as_ref()
e0b486d6097fatagen2026-10-19 07:26 UTC155
            .and_then(|a| a.mapped_slice())
acf5e9848529atagen2026-10-19 07:27 UTC156
            .ok_or(Error::StagingUnmapped)
e0b486d6097fatagen2026-10-19 07:26 UTC157
    }
e0b486d6097fatagen2026-10-19 07:26 UTC158
acf5e9848529atagen2026-10-19 07:27 UTC159
    pub fn bytes_mut(&mut self) -> Result<&mut [u8], Error> {
e0b486d6097fatagen2026-10-19 07:26 UTC160
        self.allocation
e0b486d6097fatagen2026-10-19 07:26 UTC161
            .as_mut()
e0b486d6097fatagen2026-10-19 07:26 UTC162
            .and_then(|a| a.mapped_slice_mut())
acf5e9848529atagen2026-10-19 07:27 UTC163
            .ok_or(Error::StagingUnmapped)
e0b486d6097fatagen2026-10-19 07:26 UTC164
    }
e0b486d6097fatagen2026-10-19 07:26 UTC165
}
e0b486d6097fatagen2026-10-19 07:26 UTC166
e0b486d6097fatagen2026-10-19 07:26 UTC167
impl Drop for Staging {
e0b486d6097fatagen2026-10-19 07:26 UTC168
    fn drop(&mut self) {
e0b486d6097fatagen2026-10-19 07:26 UTC169
        let Some(allocation) = self.allocation.take() else {
e0b486d6097fatagen2026-10-19 07:26 UTC170
            return;
e0b486d6097fatagen2026-10-19 07:26 UTC171
        };
e0b486d6097fatagen2026-10-19 07:26 UTC172
        let mut free = self.pool.free.lock().unwrap_or_else(|e| e.into_inner());
e0b486d6097fatagen2026-10-19 07:26 UTC173
        let pooled: usize = free.iter().map(|p| p.size).sum();
e0b486d6097fatagen2026-10-19 07:26 UTC174
        if pooled + self.size <= POOL_BYTES {
e0b486d6097fatagen2026-10-19 07:26 UTC175
            free.push(Pooled {
e0b486d6097fatagen2026-10-19 07:26 UTC176
                buffer: self.buffer,
e0b486d6097fatagen2026-10-19 07:26 UTC177
                allocation,
e0b486d6097fatagen2026-10-19 07:26 UTC178
                size: self.size,
e0b486d6097fatagen2026-10-19 07:26 UTC179
                usage: self.usage,
e0b486d6097fatagen2026-10-19 07:26 UTC180
                location: self.location,
e0b486d6097fatagen2026-10-19 07:26 UTC181
                returned: Instant::now(),
e0b486d6097fatagen2026-10-19 07:26 UTC182
            });
e0b486d6097fatagen2026-10-19 07:26 UTC183
            return;
e0b486d6097fatagen2026-10-19 07:26 UTC184
        }
e0b486d6097fatagen2026-10-19 07:26 UTC185
        drop(free);
e0b486d6097fatagen2026-10-19 07:26 UTC186
        self.pool.destroy([(self.buffer, allocation)]);
e0b486d6097fatagen2026-10-19 07:26 UTC187
    }
e0b486d6097fatagen2026-10-19 07:26 UTC188
}