monolith

public

Star0

Watch0

122 commits 1 branch 0 tags 0 releases1.5 MiB

use crate::Error;
use crate::device::Shared;
use crate::format::{components, vk_format};
use ash::vk;
use gpu_allocator::MemoryLocation;
use gpu_allocator::vulkan::{Allocation, AllocationCreateDesc, AllocationScheme};
use smithay::backend::allocator::Fourcc;
use smithay::backend::renderer::{Texture, TextureMapping};
use std::fmt;
use std::marker::PhantomData;
use std::sync::{Arc, Mutex};

pub(crate) enum Memory {
    Allocated(Allocation),
    /// Memory imported from a dmabuf, owned directly.
    Imported(vk::DeviceMemory),
}

pub(crate) struct ImageInner {
    pub shared: Arc<Shared>,
    pub image: vk::Image,
    pub view: vk::ImageView,
    pub memory: Mutex<Option<Memory>>,
    pub fourcc: Fourcc,
    pub format: vk::Format,
    pub width: u32,
    pub height: u32,
    pub layout: Mutex<vk::ImageLayout>,
    /// Shared with other devices or processes through a dmabuf. Every use
    /// acquires the image from the foreign queue family and releases it after.
    pub external: bool,
}

impl fmt::Debug for ImageInner {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("ImageInner")
            .field("image", &self.image)
            .field("fourcc", &self.fourcc)
            .field("size", &(self.width, self.height))
            .field("external", &self.external)
            .finish()
    }
}

impl Drop for ImageInner {
    fn drop(&mut self) {
        let d = &self.shared.device.device;
        unsafe {
            d.destroy_image_view(self.view, None);
            d.destroy_image(self.image, None);
        }
        match self.memory.lock().unwrap_or_else(|e| e.into_inner()).take() {
            Some(Memory::Allocated(a)) => {
                let _ = self
                    .shared
                    .allocator
                    .lock()
                    .unwrap_or_else(|e| e.into_inner())
                    .free(a);
            }
            Some(Memory::Imported(m)) => unsafe { d.free_memory(m, None) },
            None => {}
        }
    }
}

impl ImageInner {
    pub fn create(
        shared: &Arc<Shared>,
        fourcc: Fourcc,
        width: u32,
        height: u32,
        usage: vk::ImageUsageFlags,
    ) -> Result<Arc<Self>, Error> {
        let fmt = vk_format(fourcc).ok_or(Error::UnsupportedFormat(fourcc))?;
        if width == 0 || height == 0 {
            return Err(Error::InvalidSize(width as i32, height as i32));
        }
        let d = &shared.device.device;
        let info = vk::ImageCreateInfo::default()
            .image_type(vk::ImageType::TYPE_2D)
            .format(fmt.format)
            .extent(vk::Extent3D {
                width,
                height,
                depth: 1,
            })
            .mip_levels(1)
            .array_layers(1)
            .samples(vk::SampleCountFlags::TYPE_1)
            .tiling(vk::ImageTiling::OPTIMAL)
            .usage(usage)
            .initial_layout(vk::ImageLayout::UNDEFINED);
        let image = unsafe { d.create_image(&info, None)? };
        let requirements = unsafe { d.get_image_memory_requirements(image) };
        let allocation = shared
            .allocator
            .lock()
            .unwrap_or_else(|e| e.into_inner())
            .allocate(&AllocationCreateDesc {
                name: "image",
                requirements,
                location: MemoryLocation::GpuOnly,
                linear: false,
                allocation_scheme: AllocationScheme::GpuAllocatorManaged,
            });
        let allocation = match allocation {
            Ok(a) => a,
            Err(e) => {
                unsafe { d.destroy_image(image, None) };
                return Err(e.into());
            }
        };
        let memory = Memory::Allocated(allocation);
        let (mem, offset) = match &memory {
            Memory::Allocated(a) => (unsafe { a.memory() }, a.offset()),
            Memory::Imported(m) => (*m, 0),
        };
        Self::finish(
            shared, image, memory, mem, offset, fourcc, width, height, false,
        )
    }

    /// Binds memory, creates the view and wraps everything up. Takes
    /// ownership of `image` and `memory`, releasing both on failure.
    #[expect(clippy::too_many_arguments)]
    pub fn finish(
        shared: &Arc<Shared>,
        image: vk::Image,
        memory: Memory,
        mem: vk::DeviceMemory,
        offset: u64,
        fourcc: Fourcc,
        width: u32,
        height: u32,
        external: bool,
    ) -> Result<Arc<Self>, Error> {
        let fmt = vk_format(fourcc).ok_or(Error::UnsupportedFormat(fourcc))?;
        let d = &shared.device.device;
        let mut inner = Self {
            shared: shared.clone(),
            image,
            view: vk::ImageView::null(),
            memory: Mutex::new(Some(memory)),
            fourcc,
            format: fmt.format,
            width,
            height,
            layout: Mutex::new(if external {
                vk::ImageLayout::GENERAL
            } else {
                vk::ImageLayout::UNDEFINED
            }),
            external,
        };
        unsafe { d.bind_image_memory(image, mem, offset)? };
        let view_info = vk::ImageViewCreateInfo::default()
            .image(image)
            .view_type(vk::ImageViewType::TYPE_2D)
            .format(fmt.format)
            .components(components(fmt.opaque))
            .subresource_range(color_range());
        inner.view = unsafe { d.create_image_view(&view_info, None)? };
        Ok(Arc::new(inner))
    }

    pub fn layout(&self) -> vk::ImageLayout {
        *self.layout.lock().unwrap_or_else(|e| e.into_inner())
    }

    pub fn set_layout(&self, layout: vk::ImageLayout) {
        *self.layout.lock().unwrap_or_else(|e| e.into_inner()) = layout;
    }
}

pub(crate) fn color_range() -> vk::ImageSubresourceRange {
    vk::ImageSubresourceRange::default()
        .aspect_mask(vk::ImageAspectFlags::COLOR)
        .level_count(1)
        .layer_count(1)
}

/// A host-visible buffer for uploads and readbacks. Dropped ones go back to
/// a pool on `Shared` instead of being freed. Allocating and freeing large
/// host-visible memory takes a driver lock on NVIDIA that `vkQueueSubmit`
/// also waits on, so a staging buffer made per upload on the import worker
/// stalled the main thread's submits for 1 to 3 ms.
pub(crate) struct Staging {
    pub shared: Arc<Shared>,
    pub buffer: vk::Buffer,
    pub allocation: Option<Allocation>,
    size: usize,
    usage: vk::BufferUsageFlags,
    location: MemoryLocation,
}

pub(crate) struct Pooled {
    pub buffer: vk::Buffer,
    pub allocation: Allocation,
    size: usize,
    usage: vk::BufferUsageFlags,
    location: MemoryLocation,
}

/// Pooled staging memory kept at most, past which dropped buffers are freed.
const POOL_BYTES: usize = 96 * 1024 * 1024;

impl Staging {
    pub fn new(
        shared: &Arc<Shared>,
        size: usize,
        usage: vk::BufferUsageFlags,
        location: MemoryLocation,
    ) -> Result<Self, Error> {
        let reused = {
            let mut pool = shared
                .staging_pool
                .lock()
                .unwrap_or_else(|e| e.into_inner());
            let fits = |p: &Pooled| {
                p.usage == usage && p.location == location && p.size >= size && p.size <= size * 2
            };
            pool.iter().position(fits).map(|i| pool.swap_remove(i))
        };
        if let Some(p) = reused {
            return Ok(Self {
                shared: shared.clone(),
                buffer: p.buffer,
                allocation: Some(p.allocation),
                size: p.size,
                usage,
                location,
            });
        }
        let d = &shared.device.device;
        let buffer = unsafe {
            d.create_buffer(
                &vk::BufferCreateInfo::default()
                    .size(size as u64)
                    .usage(usage),
                None,
            )?
        };
        let requirements = unsafe { d.get_buffer_memory_requirements(buffer) };
        let allocation = shared
            .allocator
            .lock()
            .unwrap_or_else(|e| e.into_inner())
            .allocate(&AllocationCreateDesc {
                name: "staging",
                requirements,
                location,
                linear: true,
                allocation_scheme: AllocationScheme::GpuAllocatorManaged,
            });
        let allocation = match allocation {
            Ok(a) => a,
            Err(e) => {
                unsafe { d.destroy_buffer(buffer, None) };
                return Err(e.into());
            }
        };
        unsafe { d.bind_buffer_memory(buffer, allocation.memory(), allocation.offset())? };
        Ok(Self {
            shared: shared.clone(),
            buffer,
            allocation: Some(allocation),
            size,
            usage,
            location,
        })
    }

    pub fn bytes(&self) -> &[u8] {
        self.allocation
            .as_ref()
            .and_then(|a| a.mapped_slice())
            .unwrap_or(&[])
    }

    pub fn bytes_mut(&mut self) -> &mut [u8] {
        self.allocation
            .as_mut()
            .and_then(|a| a.mapped_slice_mut())
            .unwrap_or(&mut [])
    }
}

impl Drop for Staging {
    fn drop(&mut self) {
        let Some(allocation) = self.allocation.take() else {
            return;
        };
        let mut pool = self
            .shared
            .staging_pool
            .lock()
            .unwrap_or_else(|e| e.into_inner());
        let pooled: usize = pool.iter().map(|p| p.size).sum();
        if pooled + self.size <= POOL_BYTES {
            pool.push(Pooled {
                buffer: self.buffer,
                allocation,
                size: self.size,
                usage: self.usage,
                location: self.location,
            });
            return;
        }
        drop(pool);
        unsafe { self.shared.device.device.destroy_buffer(self.buffer, None) };
        let _ = self
            .shared
            .allocator
            .lock()
            .unwrap_or_else(|e| e.into_inner())
            .free(allocation);
    }
}

/// A sampled texture.
#[derive(Clone, Debug)]
pub struct VulkanTexture(pub(crate) Arc<ImageInner>);

/// An image the renderer can draw into, from [`smithay::backend::renderer::Offscreen`].
#[derive(Debug)]
pub struct VulkanImage(pub(crate) Arc<ImageInner>);

impl VulkanImage {
    /// The image's contents as a texture, for drawing it elsewhere.
    pub fn texture(&self) -> VulkanTexture {
        VulkanTexture(self.0.clone())
    }
}

/// A bound render target, borrowing whatever it was bound from.
#[derive(Debug)]
pub struct VulkanTarget<'a>(
    pub(crate) Arc<ImageInner>,
    pub(crate) PhantomData<&'a mut ()>,
);

/// Pixels read back from the GPU.
#[derive(Debug)]
pub struct VulkanMapping {
    pub(crate) data: Vec<u8>,
    pub(crate) width: u32,
    pub(crate) height: u32,
    pub(crate) fourcc: Fourcc,
}

macro_rules! texture_impl {
    ($t:ty, |$s:ident| $inner:expr) => {
        impl Texture for $t {
            fn width(&self) -> u32 {
                let $s = self;
                $inner.width
            }
            fn height(&self) -> u32 {
                let $s = self;
                $inner.height
            }
            fn format(&self) -> Option<Fourcc> {
                let $s = self;
                Some($inner.fourcc)
            }
        }
    };
}

texture_impl!(VulkanTexture, |s| s.0);
texture_impl!(VulkanImage, |s| s.0);
texture_impl!(VulkanTarget<'_>, |s| s.0);
texture_impl!(VulkanMapping, |s| s);

impl TextureMapping for VulkanMapping {
    fn flipped(&self) -> bool {
        false
    }
}

1	use crate::Error;
2	use crate::device::Shared;
3	use crate::format::{components, vk_format};
4	use ash::vk;
5	use gpu_allocator::MemoryLocation;
6	use gpu_allocator::vulkan::{Allocation, AllocationCreateDesc, AllocationScheme};
7	use smithay::backend::allocator::Fourcc;
8	use smithay::backend::renderer::{Texture, TextureMapping};
9	use std::fmt;
10	use std::marker::PhantomData;
11	use std::sync::{Arc, Mutex};
12
13	pub(crate) enum Memory {
14	Allocated(Allocation),
15	/// Memory imported from a dmabuf, owned directly.
16	Imported(vk::DeviceMemory),
17	}
18
19	pub(crate) struct ImageInner {
20	pub shared: Arc<Shared>,
21	pub image: vk::Image,
22	pub view: vk::ImageView,
23	pub memory: Mutex<Option<Memory>>,
24	pub fourcc: Fourcc,
25	pub format: vk::Format,
26	pub width: u32,
27	pub height: u32,
28	pub layout: Mutex<vk::ImageLayout>,
29	/// Shared with other devices or processes through a dmabuf. Every use
30	/// acquires the image from the foreign queue family and releases it after.
31	pub external: bool,
32	}
33
34	impl fmt::Debug for ImageInner {
35	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
36	f.debug_struct("ImageInner")
37	.field("image", &self.image)
38	.field("fourcc", &self.fourcc)
39	.field("size", &(self.width, self.height))
40	.field("external", &self.external)
41	.finish()
42	}
43	}
44
45	impl Drop for ImageInner {
46	fn drop(&mut self) {
47	let d = &self.shared.device.device;
48	unsafe {
49	d.destroy_image_view(self.view, None);
50	d.destroy_image(self.image, None);
51	}
52	match self.memory.lock().unwrap_or_else(\|e\| e.into_inner()).take() {
53	Some(Memory::Allocated(a)) => {
54	let _ = self
55	.shared
56	.allocator
57	.lock()
58	.unwrap_or_else(\|e\| e.into_inner())
59	.free(a);
60	}
61	Some(Memory::Imported(m)) => unsafe { d.free_memory(m, None) },
62	None => {}
63	}
64	}
65	}
66
67	impl ImageInner {
68	pub fn create(
69	shared: &Arc<Shared>,
70	fourcc: Fourcc,
71	width: u32,
72	height: u32,
73	usage: vk::ImageUsageFlags,
74	) -> Result<Arc<Self>, Error> {
75	let fmt = vk_format(fourcc).ok_or(Error::UnsupportedFormat(fourcc))?;
76	if width == 0 \|\| height == 0 {
77	return Err(Error::InvalidSize(width as i32, height as i32));
78	}
79	let d = &shared.device.device;
80	let info = vk::ImageCreateInfo::default()
81	.image_type(vk::ImageType::TYPE_2D)
82	.format(fmt.format)
83	.extent(vk::Extent3D {
84	width,
85	height,
86	depth: 1,
87	})
88	.mip_levels(1)
89	.array_layers(1)
90	.samples(vk::SampleCountFlags::TYPE_1)
91	.tiling(vk::ImageTiling::OPTIMAL)
92	.usage(usage)
93	.initial_layout(vk::ImageLayout::UNDEFINED);
94	let image = unsafe { d.create_image(&info, None)? };
95	let requirements = unsafe { d.get_image_memory_requirements(image) };
96	let allocation = shared
97	.allocator
98	.lock()
99	.unwrap_or_else(\|e\| e.into_inner())
100	.allocate(&AllocationCreateDesc {
101	name: "image",
102	requirements,
103	location: MemoryLocation::GpuOnly,
104	linear: false,
105	allocation_scheme: AllocationScheme::GpuAllocatorManaged,
106	});
107	let allocation = match allocation {
108	Ok(a) => a,
109	Err(e) => {
110	unsafe { d.destroy_image(image, None) };
111	return Err(e.into());
112	}
113	};
114	let memory = Memory::Allocated(allocation);
115	let (mem, offset) = match &memory {
116	Memory::Allocated(a) => (unsafe { a.memory() }, a.offset()),
117	Memory::Imported(m) => (*m, 0),
118	};
119	Self::finish(
120	shared, image, memory, mem, offset, fourcc, width, height, false,
121	)
122	}
123
124	/// Binds memory, creates the view and wraps everything up. Takes
125	/// ownership of `image` and `memory`, releasing both on failure.
126	#[expect(clippy::too_many_arguments)]
127	pub fn finish(
128	shared: &Arc<Shared>,
129	image: vk::Image,
130	memory: Memory,
131	mem: vk::DeviceMemory,
132	offset: u64,
133	fourcc: Fourcc,
134	width: u32,
135	height: u32,
136	external: bool,
137	) -> Result<Arc<Self>, Error> {
138	let fmt = vk_format(fourcc).ok_or(Error::UnsupportedFormat(fourcc))?;
139	let d = &shared.device.device;
140	let mut inner = Self {
141	shared: shared.clone(),
142	image,
143	view: vk::ImageView::null(),
144	memory: Mutex::new(Some(memory)),
145	fourcc,
146	format: fmt.format,
147	width,
148	height,
149	layout: Mutex::new(if external {
150	vk::ImageLayout::GENERAL
151	} else {
152	vk::ImageLayout::UNDEFINED
153	}),
154	external,
155	};
156	unsafe { d.bind_image_memory(image, mem, offset)? };
157	let view_info = vk::ImageViewCreateInfo::default()
158	.image(image)
159	.view_type(vk::ImageViewType::TYPE_2D)
160	.format(fmt.format)
161	.components(components(fmt.opaque))
162	.subresource_range(color_range());
163	inner.view = unsafe { d.create_image_view(&view_info, None)? };
164	Ok(Arc::new(inner))
165	}
166
167	pub fn layout(&self) -> vk::ImageLayout {
168	*self.layout.lock().unwrap_or_else(\|e\| e.into_inner())
169	}
170
171	pub fn set_layout(&self, layout: vk::ImageLayout) {
172	*self.layout.lock().unwrap_or_else(\|e\| e.into_inner()) = layout;
173	}
174	}
175
176	pub(crate) fn color_range() -> vk::ImageSubresourceRange {
177	vk::ImageSubresourceRange::default()
178	.aspect_mask(vk::ImageAspectFlags::COLOR)
179	.level_count(1)
180	.layer_count(1)
181	}
182
183	/// A host-visible buffer for uploads and readbacks. Dropped ones go back to
184	/// a pool on `Shared` instead of being freed. Allocating and freeing large
185	/// host-visible memory takes a driver lock on NVIDIA that `vkQueueSubmit`
186	/// also waits on, so a staging buffer made per upload on the import worker
187	/// stalled the main thread's submits for 1 to 3 ms.
188	pub(crate) struct Staging {
189	pub shared: Arc<Shared>,
190	pub buffer: vk::Buffer,
191	pub allocation: Option<Allocation>,
192	size: usize,
193	usage: vk::BufferUsageFlags,
194	location: MemoryLocation,
195	}
196
197	pub(crate) struct Pooled {
198	pub buffer: vk::Buffer,
199	pub allocation: Allocation,
200	size: usize,
201	usage: vk::BufferUsageFlags,
202	location: MemoryLocation,
203	}
204
205	/// Pooled staging memory kept at most, past which dropped buffers are freed.
206	const POOL_BYTES: usize = 96 * 1024 * 1024;
207
208	impl Staging {
209	pub fn new(
210	shared: &Arc<Shared>,
211	size: usize,
212	usage: vk::BufferUsageFlags,
213	location: MemoryLocation,
214	) -> Result<Self, Error> {
215	let reused = {
216	let mut pool = shared
217	.staging_pool
218	.lock()
219	.unwrap_or_else(\|e\| e.into_inner());
220	let fits = \|p: &Pooled\| {
221	p.usage == usage && p.location == location && p.size >= size && p.size <= size * 2
222	};
223	pool.iter().position(fits).map(\|i\| pool.swap_remove(i))
224	};
225	if let Some(p) = reused {
226	return Ok(Self {
227	shared: shared.clone(),
228	buffer: p.buffer,
229	allocation: Some(p.allocation),
230	size: p.size,
231	usage,
232	location,
233	});
234	}
235	let d = &shared.device.device;
236	let buffer = unsafe {
237	d.create_buffer(
238	&vk::BufferCreateInfo::default()
239	.size(size as u64)
240	.usage(usage),
241	None,
242	)?
243	};
244	let requirements = unsafe { d.get_buffer_memory_requirements(buffer) };
245	let allocation = shared
246	.allocator
247	.lock()
248	.unwrap_or_else(\|e\| e.into_inner())
249	.allocate(&AllocationCreateDesc {
250	name: "staging",
251	requirements,
252	location,
253	linear: true,
254	allocation_scheme: AllocationScheme::GpuAllocatorManaged,
255	});
256	let allocation = match allocation {
257	Ok(a) => a,
258	Err(e) => {
259	unsafe { d.destroy_buffer(buffer, None) };
260	return Err(e.into());
261	}
262	};
263	unsafe { d.bind_buffer_memory(buffer, allocation.memory(), allocation.offset())? };
264	Ok(Self {
265	shared: shared.clone(),
266	buffer,
267	allocation: Some(allocation),
268	size,
269	usage,
270	location,
271	})
272	}
273
274	pub fn bytes(&self) -> &[u8] {
275	self.allocation
276	.as_ref()
277	.and_then(\|a\| a.mapped_slice())
278	.unwrap_or(&[])
279	}
280
281	pub fn bytes_mut(&mut self) -> &mut [u8] {
282	self.allocation
283	.as_mut()
284	.and_then(\|a\| a.mapped_slice_mut())
285	.unwrap_or(&mut [])
286	}
287	}
288
289	impl Drop for Staging {
290	fn drop(&mut self) {
291	let Some(allocation) = self.allocation.take() else {
292	return;
293	};
294	let mut pool = self
295	.shared
296	.staging_pool
297	.lock()
298	.unwrap_or_else(\|e\| e.into_inner());
299	let pooled: usize = pool.iter().map(\|p\| p.size).sum();
300	if pooled + self.size <= POOL_BYTES {
301	pool.push(Pooled {
302	buffer: self.buffer,
303	allocation,
304	size: self.size,
305	usage: self.usage,
306	location: self.location,
307	});
308	return;
309	}
310	drop(pool);
311	unsafe { self.shared.device.device.destroy_buffer(self.buffer, None) };
312	let _ = self
313	.shared
314	.allocator
315	.lock()
316	.unwrap_or_else(\|e\| e.into_inner())
317	.free(allocation);
318	}
319	}
320
321	/// A sampled texture.
322	#[derive(Clone, Debug)]
323	pub struct VulkanTexture(pub(crate) Arc<ImageInner>);
324
325	/// An image the renderer can draw into, from [`smithay::backend::renderer::Offscreen`].
326	#[derive(Debug)]
327	pub struct VulkanImage(pub(crate) Arc<ImageInner>);
328
329	impl VulkanImage {
330	/// The image's contents as a texture, for drawing it elsewhere.
331	pub fn texture(&self) -> VulkanTexture {
332	VulkanTexture(self.0.clone())
333	}
334	}
335
336	/// A bound render target, borrowing whatever it was bound from.
337	#[derive(Debug)]
338	pub struct VulkanTarget<'a>(
339	pub(crate) Arc<ImageInner>,
340	pub(crate) PhantomData<&'a mut ()>,
341	);
342
343	/// Pixels read back from the GPU.
344	#[derive(Debug)]
345	pub struct VulkanMapping {
346	pub(crate) data: Vec<u8>,
347	pub(crate) width: u32,
348	pub(crate) height: u32,
349	pub(crate) fourcc: Fourcc,
350	}
351
352	macro_rules! texture_impl {
353	($t:ty, \|$s:ident\| $inner:expr) => {
354	impl Texture for $t {
355	fn width(&self) -> u32 {
356	let $s = self;
357	$inner.width
358	}
359	fn height(&self) -> u32 {
360	let $s = self;
361	$inner.height
362	}
363	fn format(&self) -> Option<Fourcc> {
364	let $s = self;
365	Some($inner.fourcc)
366	}
367	}
368	};
369	}
370
371	texture_impl!(VulkanTexture, \|s\| s.0);
372	texture_impl!(VulkanImage, \|s\| s.0);
373	texture_impl!(VulkanTarget<'_>, \|s\| s.0);
374	texture_impl!(VulkanMapping, \|s\| s);
375
376	impl TextureMapping for VulkanMapping {
377	fn flipped(&self) -> bool {
378	false
379	}
380	}