atagen

monolith

public
Star0
Watch0
796c963a4b556ef7d31bd8e7eb4700ece1275a9f
1
ssh://brewery@pls.lobotomise.me:33/atagen/monolith.git
crates/render-vk/src/renderer.rsRust1080 lines34.9 KB
1
use crate::Error;
2
use crate::device::Shared;
3
use crate::dmabuf::{self, DmabufCaps, RENDER_USAGE, SAMPLE_USAGE, VulkanAllocator};
4
use crate::format::{BYTES_PER_PIXEL, SUPPORTED};
5
use crate::frame::{Op, VulkanFrame};
6
use crate::image::{
7
    ImageInner, Staging, VulkanImage, VulkanMapping, VulkanTarget, VulkanTexture, color_range,
8
};
9
use crate::pipeline::Pipelines;
10
use crate::sync::{FenceInner, VkFence};
11
use ash::vk;
12
use gpu_allocator::MemoryLocation;
13
use smithay::backend::allocator::Fourcc;
14
use smithay::backend::allocator::dmabuf::{Dmabuf, WeakDmabuf};
15
use smithay::backend::allocator::format::FormatSet;
16
use smithay::backend::renderer::ImportMemWl;
17
use smithay::backend::renderer::sync::SyncPoint;
18
use smithay::backend::renderer::{
19
    Bind, ContextId, DebugFlags, ExportMem, ImportDma, ImportDmaWl, ImportMem, Offscreen, Renderer,
20
    RendererSuper, Texture, TextureFilter,
21
};
22
use smithay::reexports::wayland_server::protocol::wl_buffer::WlBuffer;
23
use smithay::utils::{Buffer, Physical, Rectangle, Size, Transform};
24
use smithay::wayland::compositor::SurfaceData;
25
use smithay::wayland::shm::{shm_format_to_fourcc, with_buffer_contents};
26
use std::cell::Cell;
27
use std::collections::HashMap;
28
use std::fmt;
29
use std::marker::PhantomData;
30
use std::sync::{Arc, Mutex};
31
use std::time::{Duration, Instant};
32
33
struct Pending {
34
    fence: Arc<FenceInner>,
35
    cmd: vk::CommandBuffer,
36
    _keep: Vec<Arc<ImageInner>>,
37
    _staging: Option<Staging>,
38
}
39
40
pub struct VulkanRenderer {
41
    pending: Vec<Pending>,
42
    dmabuf_textures: HashMap<WeakDmabuf, VulkanTexture>,
43
    dmabuf_targets: HashMap<WeakDmabuf, Arc<ImageInner>>,
44
    pipelines: Pipelines,
45
    pool: vk::CommandPool,
46
    /// Command buffers whose work finished, reset and reused by `begin`.
47
    spare_cmds: Vec<vk::CommandBuffer>,
48
    samplers: [vk::Sampler; 4],
49
    min_filter: TextureFilter,
50
    mag_filter: TextureFilter,
51
    debug: DebugFlags,
52
    context: ContextId<VulkanTexture>,
53
    caps: Arc<DmabufCaps>,
54
    shared: Arc<Shared>,
55
    submissions: Cell<u64>,
56
    stall: Cell<StallPhase>,
57
}
58
59
/// Where in every 1024 submissions the driver stalls. A phase is trusted
60
/// once two stalls land on it.
61
#[derive(Clone, Copy, Default)]
62
struct StallPhase {
63
    phase: Option<u64>,
64
    seen: Option<u64>,
65
}
66
67
impl StallPhase {
68
    fn saw(self, at: u64) -> Self {
69
        if self.seen == Some(at) {
70
            if self.phase != Some(at) {
71
                tracing::debug!(phase = at, "queue submissions stall periodically");
72
            }
73
            Self {
74
                phase: Some(at),
75
                seen: Some(at),
76
            }
77
        } else {
78
            Self {
79
                seen: Some(at),
80
                ..self
81
            }
82
        }
83
    }
84
}
85
86
/// A submission this slow is a driver stall rather than work.
87
const STALL: Duration = Duration::from_micros(1500);
88
/// How many submissions early a stall is absorbed. Each empty one costs
89
/// tens of microseconds.
90
const ABSORB_AHEAD: u64 = 8;
91
92
impl fmt::Debug for VulkanRenderer {
93
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
94
        f.debug_struct("VulkanRenderer")
95
            .field("device", &self.shared.device.name)
96
            .finish()
97
    }
98
}
99
100
impl VulkanRenderer {
101
    pub fn new() -> Result<Self, Error> {
102
        Self::with_node(None)
103
    }
104
105
    /// A renderer on the GPU behind DRM device number `node`, so it can
106
    /// render into buffers that GPU scans out.
107
    pub fn for_node(node: u64) -> Result<Self, Error> {
108
        Self::with_node(Some(node))
109
    }
110
111
    fn with_node(node: Option<u64>) -> Result<Self, Error> {
112
        let shared = Arc::new(Shared::new(node)?);
113
        let d = &shared.device.device;
114
        let pool = unsafe {
115
            d.create_command_pool(
116
                &vk::CommandPoolCreateInfo::default()
117
                    .queue_family_index(shared.device.queue_family)
118
                    .flags(
119
                        vk::CommandPoolCreateFlags::TRANSIENT
120
                            | vk::CommandPoolCreateFlags::RESET_COMMAND_BUFFER,
121
                    ),
122
                None,
123
            )?
124
        };
125
        let filter = |f: TextureFilter| match f {
126
            TextureFilter::Linear => vk::Filter::LINEAR,
127
            TextureFilter::Nearest => vk::Filter::NEAREST,
128
        };
129
        let mut samplers = [vk::Sampler::null(); 4];
130
        for (i, s) in samplers.iter_mut().enumerate() {
131
            let (min, mag) = (sampler_filter(i >> 1), sampler_filter(i & 1));
132
            *s = unsafe {
133
                d.create_sampler(
134
                    &vk::SamplerCreateInfo::default()
135
                        .min_filter(filter(min))
136
                        .mag_filter(filter(mag))
137
                        .address_mode_u(vk::SamplerAddressMode::CLAMP_TO_EDGE)
138
                        .address_mode_v(vk::SamplerAddressMode::CLAMP_TO_EDGE)
139
                        .address_mode_w(vk::SamplerAddressMode::CLAMP_TO_EDGE),
140
                    None,
141
                )?
142
            };
143
        }
144
        Ok(Self {
145
            pending: Vec::new(),
146
            dmabuf_textures: HashMap::new(),
147
            dmabuf_targets: HashMap::new(),
148
            pipelines: Pipelines::new(&shared)?,
149
            pool,
150
            spare_cmds: Vec::new(),
151
            submissions: Cell::new(0),
152
            stall: Cell::new(StallPhase {
153
                // Measured on an RTX 3060, see research/nvidia.md.
154
                phase: shared.device.name.contains("NVIDIA").then_some(1015),
155
                seen: None,
156
            }),
157
            samplers,
158
            min_filter: TextureFilter::Linear,
159
            mag_filter: TextureFilter::Linear,
160
            debug: DebugFlags::empty(),
161
            context: ContextId::new(),
162
            caps: Arc::new(DmabufCaps::query(&shared)),
163
            shared,
164
        })
165
    }
166
167
    /// Name of the physical device in use.
168
    pub fn device_name(&self) -> &str {
169
        &self.shared.device.name
170
    }
171
172
    /// Whether dmabufs can be imported, rendered into and allocated.
173
    pub fn supports_dmabuf(&self) -> bool {
174
        self.caps.render.iter().next().is_some()
175
    }
176
177
    /// Formats and modifiers this renderer can render into as dmabufs.
178
    pub fn render_formats(&self) -> FormatSet {
179
        self.caps.render.clone()
180
    }
181
182
    /// An allocator for render-capable dmabufs on this device.
183
    pub fn allocator(&self) -> VulkanAllocator {
184
        VulkanAllocator {
185
            shared: self.shared.clone(),
186
            caps: self.caps.clone(),
187
        }
188
    }
189
190
    pub(crate) fn shared(&self) -> &Arc<Shared> {
191
        &self.shared
192
    }
193
194
    pub(crate) fn context_id_ref(&self) -> &ContextId<VulkanTexture> {
195
        &self.context
196
    }
197
198
    fn sampler(&self) -> vk::Sampler {
199
        let idx = |f: TextureFilter| usize::from(f == TextureFilter::Nearest);
200
        self.samplers[(idx(self.min_filter) << 1) | idx(self.mag_filter)]
201
    }
202
203
    fn reap(&mut self) {
204
        let spare = &mut self.spare_cmds;
205
        self.pending.retain(|p| {
206
            if !p.fence.signaled() {
207
                return true;
208
            }
209
            spare.push(p.cmd);
210
            false
211
        });
212
        self.dmabuf_textures.retain(|k, _| !k.is_gone());
213
        self.dmabuf_targets.retain(|k, _| !k.is_gone());
214
    }
215
216
    fn begin(&mut self) -> Result<vk::CommandBuffer, Error> {
217
        let d = &self.shared.device.device;
218
        let cmd = match self.spare_cmds.pop() {
219
            Some(cmd) => {
220
                unsafe { d.reset_command_buffer(cmd, vk::CommandBufferResetFlags::empty())? };
221
                cmd
222
            }
223
            None => unsafe {
224
                d.allocate_command_buffers(
225
                    &vk::CommandBufferAllocateInfo::default()
226
                        .command_pool(self.pool)
227
                        .level(vk::CommandBufferLevel::PRIMARY)
228
                        .command_buffer_count(1),
229
                )?[0]
230
            },
231
        };
232
        unsafe {
233
            d.begin_command_buffer(
234
                cmd,
235
                &vk::CommandBufferBeginInfo::default()
236
                    .flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT),
237
            )?
238
        };
239
        Ok(cmd)
240
    }
241
242
    #[profiling::function]
243
    fn end_and_submit(&self, cmd: vk::CommandBuffer) -> Result<Arc<FenceInner>, Error> {
244
        let d = &self.shared.device.device;
245
        {
246
            profiling::scope!("end_command_buffer");
247
            unsafe { d.end_command_buffer(cmd)? };
248
        }
249
        let fence = {
250
            profiling::scope!("create_fence");
251
            FenceInner::new(&self.shared)?
252
        };
253
        let cmds = [vk::CommandBufferSubmitInfo::default().command_buffer(cmd)];
254
        let submit = vk::SubmitInfo2::default().command_buffer_infos(&cmds);
255
        let queue = self
256
            .shared
257
            .device
258
            .queue
259
            .lock()
260
            .unwrap_or_else(|e| e.into_inner());
261
        let began = Instant::now();
262
        {
263
            profiling::scope!("queue_submit");
264
            unsafe { d.queue_submit2(*queue, &[submit], fence.fence())? };
265
        }
266
        let n = self.submissions.get();
267
        self.submissions.set(n + 1);
268
        if began.elapsed() > STALL {
269
            self.stall.set(self.stall.get().saw(n % 1024));
270
        }
271
        Ok(fence)
272
    }
273
274
    /// Spends an upcoming stalling submission on empty command buffers, so
275
    /// the stall lands now, in idle time, instead of in a frame. Call it
276
    /// right after a flip. NVIDIA stalls for milliseconds on every 1024th
277
    /// submission, see research/nvidia.md.
278
    pub fn absorb_submit_stall(&mut self) -> Result<(), Error> {
279
        self.reap();
280
        let Some(phase) = self.stall.get().phase else {
281
            return Ok(());
282
        };
283
        let ahead = (phase + 1024 - self.submissions.get() % 1024) % 1024;
284
        if ahead >= ABSORB_AHEAD {
285
            return Ok(());
286
        }
287
        for _ in 0..=ahead {
288
            let cmd = self.begin()?;
289
            let fence = self.end_and_submit(cmd)?;
290
            self.pending.push(Pending {
291
                fence,
292
                cmd,
293
                _keep: Vec::new(),
294
                _staging: None,
295
            });
296
        }
297
        Ok(())
298
    }
299
300
    /// Records, submits and waits for a short command buffer.
301
    fn one_shot(
302
        &mut self,
303
        record: impl FnOnce(&ash::Device, vk::CommandBuffer, u32),
304
    ) -> Result<(), Error> {
305
        let cmd = self.begin()?;
306
        record(
307
            &self.shared.device.device,
308
            cmd,
309
            self.shared.device.queue_family,
310
        );
311
        let fence = self.end_and_submit(cmd)?;
312
        let res = fence.wait();
313
        self.spare_cmds.push(cmd);
314
        Ok(res?)
315
    }
316
317
    #[profiling::function]
318
    pub(crate) fn submit(
319
        &mut self,
320
        target: &mut VulkanTarget<'_>,
321
        ops: Vec<Op>,
322
    ) -> Result<SyncPoint, Error> {
323
        self.reap();
324
        let image = target.0.clone();
325
        let mut sampled: Vec<Arc<ImageInner>> = Vec::new();
326
        for op in &ops {
327
            if let Op::Draw {
328
                texture: Some(t), ..
329
            } = op
330
                && !sampled.iter().any(|s| Arc::ptr_eq(s, &t.0))
331
            {
332
                sampled.push(t.0.clone());
333
            }
334
        }
335
        let mut pipelines = Vec::with_capacity(ops.len());
336
        for op in &ops {
337
            pipelines.push(match op {
338
                Op::Draw { kind, .. } => Some(self.pipelines.get(image.format, *kind)?),
339
                Op::Clear { .. } => None,
340
            });
341
        }
342
        let cmd = self.begin()?;
343
        let d = &self.shared.device.device;
344
        let family = self.shared.device.queue_family;
345
        let push = &self.shared.device.push_descriptor;
346
        let (w, h) = (image.width, image.height);
347
        let sampler = self.sampler();
348
        unsafe {
349
            for t in &sampled {
350
                acquire(d, cmd, family, t, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL);
351
            }
352
            acquire(
353
                d,
354
                cmd,
355
                family,
356
                &image,
357
                vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
358
            );
359
            let attachments = [vk::RenderingAttachmentInfo::default()
360
                .image_view(image.view)
361
                .image_layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)
362
                .load_op(vk::AttachmentLoadOp::LOAD)
363
                .store_op(vk::AttachmentStoreOp::STORE)];
364
            let area = vk::Rect2D {
365
                offset: vk::Offset2D::default(),
366
                extent: vk::Extent2D {
367
                    width: w,
368
                    height: h,
369
                },
370
            };
371
            d.cmd_begin_rendering(
372
                cmd,
373
                &vk::RenderingInfo::default()
374
                    .render_area(area)
375
                    .layer_count(1)
376
                    .color_attachments(&attachments),
377
            );
378
            d.cmd_set_viewport(
379
                cmd,
380
                0,
381
                &[vk::Viewport {
382
                    x: 0.0,
383
                    y: 0.0,
384
                    width: w as f32,
385
                    height: h as f32,
386
                    min_depth: 0.0,
387
                    max_depth: 1.0,
388
                }],
389
            );
390
            let mut bound = vk::Pipeline::null();
391
            for (op, pipeline) in ops.iter().zip(pipelines) {
392
                match op {
393
                    Op::Clear { color, rects } => {
394
                        let clear = [vk::ClearAttachment {
395
                            aspect_mask: vk::ImageAspectFlags::COLOR,
396
                            color_attachment: 0,
397
                            clear_value: vk::ClearValue {
398
                                color: vk::ClearColorValue { float32: *color },
399
                            },
400
                        }];
401
                        let rects: Vec<_> = rects
402
                            .iter()
403
                            .map(|&rect| vk::ClearRect {
404
                                rect,
405
                                base_array_layer: 0,
406
                                layer_count: 1,
407
                            })
408
                            .collect();
409
                        d.cmd_clear_attachments(cmd, &clear, &rects);
410
                    }
411
                    Op::Draw {
412
                        texture,
413
                        push: constants,
414
                        scissors,
415
                        ..
416
                    } => {
417
                        let pipeline = pipeline.unwrap_or_default();
418
                        if pipeline != bound {
419
                            d.cmd_bind_pipeline(cmd, vk::PipelineBindPoint::GRAPHICS, pipeline);
420
                            bound = pipeline;
421
                        }
422
                        if let Some(t) = texture {
423
                            let info = [vk::DescriptorImageInfo::default()
424
                                .image_view(t.0.view)
425
                                .image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)
426
                                .sampler(sampler)];
427
                            let write = vk::WriteDescriptorSet::default()
428
                                .dst_binding(0)
429
                                .descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER)
430
                                .image_info(&info);
431
                            push.cmd_push_descriptor_set(
432
                                cmd,
433
                                vk::PipelineBindPoint::GRAPHICS,
434
                                self.pipelines.layout,
435
                                0,
436
                                &[write],
437
                            );
438
                        }
439
                        d.cmd_push_constants(
440
                            cmd,
441
                            self.pipelines.layout,
442
                            vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::FRAGMENT,
443
                            0,
444
                            constants.bytes(),
445
                        );
446
                        for s in scissors {
447
                            d.cmd_set_scissor(cmd, 0, &[*s]);
448
                            d.cmd_draw(cmd, 4, 1, 0, 0);
449
                        }
450
                    }
451
                }
452
            }
453
            d.cmd_end_rendering(cmd);
454
            release(
455
                d,
456
                cmd,
457
                family,
458
                &image,
459
                vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL,
460
            );
461
            for t in &sampled {
462
                release(d, cmd, family, t, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL);
463
            }
464
        }
465
        let fence = self.end_and_submit(cmd)?;
466
        sampled.push(image);
467
        self.pending.push(Pending {
468
            fence: fence.clone(),
469
            cmd,
470
            _keep: sampled,
471
            _staging: None,
472
        });
473
        Ok(SyncPoint::from(VkFence(fence)))
474
    }
475
476
    /// Copies `region` of a full-image `data` into `image`, rows top-down or
477
    /// flipped.
478
    fn upload(
479
        &mut self,
480
        image: &Arc<ImageInner>,
481
        data: &[u8],
482
        region: Rectangle<i32, Buffer>,
483
        flipped: bool,
484
    ) -> Result<(), Error> {
485
        let width = image.width as usize;
486
        let rows = image.height as usize;
487
        let needed = width * rows * BYTES_PER_PIXEL;
488
        if data.len() < needed {
489
            return Err(Error::BufferTooSmall {
490
                needed,
491
                got: data.len(),
492
            });
493
        }
494
        let stride = width * BYTES_PER_PIXEL;
495
        self.upload_with(image, region, |y| {
496
            let from = if flipped { rows - 1 - y } else { y };
497
            let start = from * stride + region.loc.x as usize * BYTES_PER_PIXEL;
498
            &data[start..start + region.size.w as usize * BYTES_PER_PIXEL]
499
        })
500
    }
501
502
    /// Uploads `region` of `image`, taking each image row `y` in the region
503
    /// from `row(y)`. The CPU does not wait: the copy is ordered before later
504
    /// frames by its barrier, and its staging memory lives until its fence.
505
    #[profiling::function]
506
    fn upload_with<'a>(
507
        &mut self,
508
        image: &Arc<ImageInner>,
509
        region: Rectangle<i32, Buffer>,
510
        row: impl Fn(usize) -> &'a [u8],
511
    ) -> Result<(), Error> {
512
        if image.external {
513
            return Err(Error::ForeignTexture);
514
        }
515
        self.reap();
516
        let (w, h) = (region.size.w as usize, region.size.h as usize);
517
        if w == 0 || h == 0 {
518
            return Ok(());
519
        }
520
        let line = w * BYTES_PER_PIXEL;
521
        let mut staging = Staging::new(
522
            &self.shared,
523
            line * h,
524
            vk::BufferUsageFlags::TRANSFER_SRC,
525
            MemoryLocation::CpuToGpu,
526
        )?;
527
        let dst = staging.bytes_mut();
528
        for i in 0..h {
529
            dst[i * line..(i + 1) * line].copy_from_slice(row(region.loc.y as usize + i));
530
        }
531
        let copy = vk::BufferImageCopy::default()
532
            .buffer_row_length(w as u32)
533
            .image_subresource(
534
                vk::ImageSubresourceLayers::default()
535
                    .aspect_mask(vk::ImageAspectFlags::COLOR)
536
                    .layer_count(1),
537
            )
538
            .image_offset(vk::Offset3D {
539
                x: region.loc.x,
540
                y: region.loc.y,
541
                z: 0,
542
            })
543
            .image_extent(vk::Extent3D {
544
                width: w as u32,
545
                height: h as u32,
546
                depth: 1,
547
            });
548
        let cmd = self.begin()?;
549
        let d = &self.shared.device.device;
550
        let family = self.shared.device.queue_family;
551
        unsafe {
552
            acquire(d, cmd, family, image, vk::ImageLayout::TRANSFER_DST_OPTIMAL);
553
            d.cmd_copy_buffer_to_image(
554
                cmd,
555
                staging.buffer,
556
                image.image,
557
                vk::ImageLayout::TRANSFER_DST_OPTIMAL,
558
                &[copy],
559
            );
560
            acquire(
561
                d,
562
                cmd,
563
                family,
564
                image,
565
                vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL,
566
            );
567
        }
568
        let fence = self.end_and_submit(cmd)?;
569
        self.pending.push(Pending {
570
            fence,
571
            cmd,
572
            _keep: vec![image.clone()],
573
            _staging: Some(staging),
574
        });
575
        Ok(())
576
    }
577
578
    fn read_back(
579
        &mut self,
580
        image: &Arc<ImageInner>,
581
        region: Rectangle<i32, Buffer>,
582
        fourcc: Fourcc,
583
    ) -> Result<VulkanMapping, Error> {
584
        let region = region
585
            .intersection(Rectangle::from_size(
586
                (image.width as i32, image.height as i32).into(),
587
            ))
588
            .unwrap_or_default();
589
        let (w, h) = (region.size.w.max(0) as u32, region.size.h.max(0) as u32);
590
        let swap = needs_swizzle(image.fourcc, fourcc)?;
591
        let len = (w as usize * h as usize * BYTES_PER_PIXEL).max(BYTES_PER_PIXEL);
592
        let staging = Staging::new(
593
            &self.shared,
594
            len,
595
            vk::BufferUsageFlags::TRANSFER_DST,
596
            MemoryLocation::GpuToCpu,
597
        )?;
598
        if w > 0 && h > 0 {
599
            let copy = vk::BufferImageCopy::default()
600
                .image_subresource(
601
                    vk::ImageSubresourceLayers::default()
602
                        .aspect_mask(vk::ImageAspectFlags::COLOR)
603
                        .layer_count(1),
604
                )
605
                .image_offset(vk::Offset3D {
606
                    x: region.loc.x,
607
                    y: region.loc.y,
608
                    z: 0,
609
                })
610
                .image_extent(vk::Extent3D {
611
                    width: w,
612
                    height: h,
613
                    depth: 1,
614
                });
615
            let buffer = staging.buffer;
616
            let source = image.clone();
617
            self.one_shot(|d, cmd, family| unsafe {
618
                acquire(
619
                    d,
620
                    cmd,
621
                    family,
622
                    &source,
623
                    vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
624
                );
625
                d.cmd_copy_image_to_buffer(
626
                    cmd,
627
                    source.image,
628
                    vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
629
                    buffer,
630
                    &[copy],
631
                );
632
                release(
633
                    d,
634
                    cmd,
635
                    family,
636
                    &source,
637
                    vk::ImageLayout::TRANSFER_SRC_OPTIMAL,
638
                );
639
            })?;
640
        }
641
        let mut data = staging.bytes()[..w as usize * h as usize * BYTES_PER_PIXEL].to_vec();
642
        if swap {
643
            for px in data.as_chunks_mut::<BYTES_PER_PIXEL>().0 {
644
                px.swap(0, 2);
645
            }
646
        }
647
        Ok(VulkanMapping {
648
            data,
649
            width: w,
650
            height: h,
651
            fourcc,
652
        })
653
    }
654
655
    fn own(&self, image: &Arc<ImageInner>) -> Result<(), Error> {
656
        if Arc::ptr_eq(&image.shared, &self.shared) {
657
            Ok(())
658
        } else {
659
            Err(Error::ForeignTexture)
660
        }
661
    }
662
}
663
664
impl Drop for VulkanRenderer {
665
    fn drop(&mut self) {
666
        let d = &self.shared.device.device;
667
        unsafe {
668
            let _ = d.device_wait_idle();
669
            for p in self.pending.drain(..) {
670
                d.free_command_buffers(self.pool, &[p.cmd]);
671
            }
672
            for s in self.samplers {
673
                d.destroy_sampler(s, None);
674
            }
675
            d.destroy_command_pool(self.pool, None);
676
        }
677
    }
678
}
679
680
fn sampler_filter(bit: usize) -> TextureFilter {
681
    if bit == 0 {
682
        TextureFilter::Linear
683
    } else {
684
        TextureFilter::Nearest
685
    }
686
}
687
688
/// A full barrier between all earlier and later work on the image, optionally
689
/// transferring ownership between queue families. Coarse, but correct for
690
/// every transition the renderer makes.
691
unsafe fn barrier(
692
    d: &ash::Device,
693
    cmd: vk::CommandBuffer,
694
    image: vk::Image,
695
    old: vk::ImageLayout,
696
    new: vk::ImageLayout,
697
    src_family: u32,
698
    dst_family: u32,
699
) {
700
    let b = [vk::ImageMemoryBarrier2::default()
701
        .src_stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS)
702
        .src_access_mask(vk::AccessFlags2::MEMORY_WRITE)
703
        .dst_stage_mask(vk::PipelineStageFlags2::ALL_COMMANDS)
704
        .dst_access_mask(vk::AccessFlags2::MEMORY_READ | vk::AccessFlags2::MEMORY_WRITE)
705
        .old_layout(old)
706
        .new_layout(new)
707
        .src_queue_family_index(src_family)
708
        .dst_queue_family_index(dst_family)
709
        .image(image)
710
        .subresource_range(color_range())];
711
    unsafe {
712
        d.cmd_pipeline_barrier2(
713
            cmd,
714
            &vk::DependencyInfo::default().image_memory_barriers(&b),
715
        )
716
    };
717
}
718
719
/// Moves an image into `layout` for use on this queue. Dmabuf images come
720
/// from the foreign queue family in the general layout, others continue from
721
/// their tracked layout.
722
unsafe fn acquire(
723
    d: &ash::Device,
724
    cmd: vk::CommandBuffer,
725
    family: u32,
726
    image: &ImageInner,
727
    layout: vk::ImageLayout,
728
) {
729
    if image.external {
730
        unsafe {
731
            barrier(
732
                d,
733
                cmd,
734
                image.image,
735
                vk::ImageLayout::GENERAL,
736
                layout,
737
                vk::QUEUE_FAMILY_FOREIGN_EXT,
738
                family,
739
            )
740
        };
741
    } else {
742
        let old = image.layout();
743
        if old != layout {
744
            unsafe {
745
                barrier(
746
                    d,
747
                    cmd,
748
                    image.image,
749
                    old,
750
                    layout,
751
                    vk::QUEUE_FAMILY_IGNORED,
752
                    vk::QUEUE_FAMILY_IGNORED,
753
                )
754
            };
755
            image.set_layout(layout);
756
        }
757
    }
758
}
759
760
/// Hands a dmabuf image back to the foreign queue family after use.
761
unsafe fn release(
762
    d: &ash::Device,
763
    cmd: vk::CommandBuffer,
764
    family: u32,
765
    image: &ImageInner,
766
    layout: vk::ImageLayout,
767
) {
768
    if image.external {
769
        unsafe {
770
            barrier(
771
                d,
772
                cmd,
773
                image.image,
774
                layout,
775
                vk::ImageLayout::GENERAL,
776
                family,
777
                vk::QUEUE_FAMILY_FOREIGN_EXT,
778
            )
779
        };
780
    }
781
}
782
783
/// Whether reading `from` as `to` needs red and blue swapped.
784
fn needs_swizzle(from: Fourcc, to: Fourcc) -> Result<bool, Error> {
785
    let bgr = |f: Fourcc| match f {
786
        Fourcc::Argb8888 | Fourcc::Xrgb8888 => Ok(true),
787
        Fourcc::Abgr8888 | Fourcc::Xbgr8888 => Ok(false),
788
        f => Err(Error::UnsupportedFormat(f)),
789
    };
790
    Ok(bgr(from)? != bgr(to)?)
791
}
792
793
impl RendererSuper for VulkanRenderer {
794
    type Error = Error;
795
    type TextureId = VulkanTexture;
796
    type Framebuffer<'buffer> = VulkanTarget<'buffer>;
797
    type Frame<'frame, 'buffer>
798
        = VulkanFrame<'frame, 'buffer>
799
    where
800
        'buffer: 'frame,
801
        Self: 'frame;
802
}
803
804
impl Renderer for VulkanRenderer {
805
    fn context_id(&self) -> ContextId<VulkanTexture> {
806
        self.context.clone()
807
    }
808
809
    fn downscale_filter(&mut self, filter: TextureFilter) -> Result<(), Error> {
810
        self.min_filter = filter;
811
        Ok(())
812
    }
813
814
    fn upscale_filter(&mut self, filter: TextureFilter) -> Result<(), Error> {
815
        self.mag_filter = filter;
816
        Ok(())
817
    }
818
819
    fn set_debug_flags(&mut self, flags: DebugFlags) {
820
        self.debug = flags;
821
    }
822
823
    fn debug_flags(&self) -> DebugFlags {
824
        self.debug
825
    }
826
827
    fn render<'frame, 'buffer>(
828
        &'frame mut self,
829
        framebuffer: &'frame mut VulkanTarget<'buffer>,
830
        output_size: Size<i32, Physical>,
831
        dst_transform: Transform,
832
    ) -> Result<VulkanFrame<'frame, 'buffer>, Error>
833
    where
834
        'buffer: 'frame,
835
    {
836
        self.own(&framebuffer.0)?;
837
        self.reap();
838
        Ok(VulkanFrame::new(
839
            self,
840
            framebuffer,
841
            output_size,
842
            dst_transform,
843
        ))
844
    }
845
846
    fn wait(&mut self, sync: &SyncPoint) -> Result<(), Error> {
847
        sync.wait().map_err(|_| Error::Interrupted)
848
    }
849
850
    fn cleanup_texture_cache(&mut self) -> Result<(), Error> {
851
        self.reap();
852
        Ok(())
853
    }
854
855
    fn invalidate_caches(&mut self) -> Result<(), Error> {
856
        self.reap();
857
        self.dmabuf_textures.clear();
858
        self.dmabuf_targets.clear();
859
        Ok(())
860
    }
861
}
862
863
impl Bind<VulkanImage> for VulkanRenderer {
864
    fn bind<'a>(&mut self, target: &'a mut VulkanImage) -> Result<VulkanTarget<'a>, Error> {
865
        self.own(&target.0)?;
866
        Ok(VulkanTarget(target.0.clone(), PhantomData))
867
    }
868
}
869
870
impl Bind<Dmabuf> for VulkanRenderer {
871
    #[profiling::function]
872
    fn bind<'a>(&mut self, target: &'a mut Dmabuf) -> Result<VulkanTarget<'a>, Error> {
873
        let key = target.weak();
874
        let image = match self.dmabuf_targets.get(&key) {
875
            Some(image) => image.clone(),
876
            None => {
877
                let image = dmabuf::import(&self.shared, &self.caps, target, RENDER_USAGE)?;
878
                self.dmabuf_targets.insert(key, image.clone());
879
                image
880
            }
881
        };
882
        Ok(VulkanTarget(image, PhantomData))
883
    }
884
885
    fn supported_formats(&self) -> Option<FormatSet> {
886
        Some(self.caps.render.clone())
887
    }
888
}
889
890
impl Offscreen<VulkanImage> for VulkanRenderer {
891
    fn create_buffer(
892
        &mut self,
893
        format: Fourcc,
894
        size: Size<i32, Buffer>,
895
    ) -> Result<VulkanImage, Error> {
896
        if size.w <= 0 || size.h <= 0 {
897
            return Err(Error::InvalidSize(size.w, size.h));
898
        }
899
        let usage = vk::ImageUsageFlags::COLOR_ATTACHMENT
900
            | vk::ImageUsageFlags::SAMPLED
901
            | vk::ImageUsageFlags::TRANSFER_SRC
902
            | vk::ImageUsageFlags::TRANSFER_DST;
903
        Ok(VulkanImage(ImageInner::create(
904
            &self.shared,
905
            format,
906
            size.w as u32,
907
            size.h as u32,
908
            usage,
909
        )?))
910
    }
911
}
912
913
impl ImportMem for VulkanRenderer {
914
    fn import_memory(
915
        &mut self,
916
        data: &[u8],
917
        format: Fourcc,
918
        size: Size<i32, Buffer>,
919
        flipped: bool,
920
    ) -> Result<VulkanTexture, Error> {
921
        if size.w <= 0 || size.h <= 0 {
922
            return Err(Error::InvalidSize(size.w, size.h));
923
        }
924
        let usage = vk::ImageUsageFlags::SAMPLED
925
            | vk::ImageUsageFlags::TRANSFER_DST
926
            | vk::ImageUsageFlags::TRANSFER_SRC;
927
        let image = ImageInner::create(&self.shared, format, size.w as u32, size.h as u32, usage)?;
928
        self.upload(&image, data, Rectangle::from_size(size), flipped)?;
929
        Ok(VulkanTexture(image))
930
    }
931
932
    fn update_memory(
933
        &mut self,
934
        texture: &VulkanTexture,
935
        data: &[u8],
936
        region: Rectangle<i32, Buffer>,
937
    ) -> Result<(), Error> {
938
        self.own(&texture.0)?;
939
        let Some(region) = region.intersection(Rectangle::from_size(texture.size())) else {
940
            return Ok(());
941
        };
942
        self.upload(&texture.0, data, region, false)
943
    }
944
945
    fn mem_formats(&self) -> Box<dyn Iterator<Item = Fourcc>> {
946
        Box::new(SUPPORTED.into_iter())
947
    }
948
}
949
950
impl ImportDma for VulkanRenderer {
951
    fn dmabuf_formats(&self) -> FormatSet {
952
        self.caps.sample.clone()
953
    }
954
955
    #[profiling::function]
956
    fn import_dmabuf(
957
        &mut self,
958
        buffer: &Dmabuf,
959
        _damage: Option<&[Rectangle<i32, Buffer>]>,
960
    ) -> Result<VulkanTexture, Error> {
961
        let key = buffer.weak();
962
        if let Some(t) = self.dmabuf_textures.get(&key) {
963
            return Ok(t.clone());
964
        }
965
        let texture = VulkanTexture(dmabuf::import(
966
            &self.shared,
967
            &self.caps,
968
            buffer,
969
            SAMPLE_USAGE,
970
        )?);
971
        self.dmabuf_textures.insert(key, texture.clone());
972
        Ok(texture)
973
    }
974
}
975
976
impl ImportDmaWl for VulkanRenderer {}
977
978
impl ExportMem for VulkanRenderer {
979
    type TextureMapping = VulkanMapping;
980
981
    fn copy_framebuffer(
982
        &mut self,
983
        target: &VulkanTarget<'_>,
984
        region: Rectangle<i32, Buffer>,
985
        format: Fourcc,
986
    ) -> Result<VulkanMapping, Error> {
987
        self.own(&target.0)?;
988
        let image = target.0.clone();
989
        self.read_back(&image, region, format)
990
    }
991
992
    fn copy_texture(
993
        &mut self,
994
        texture: &VulkanTexture,
995
        region: Rectangle<i32, Buffer>,
996
        format: Fourcc,
997
    ) -> Result<VulkanMapping, Error> {
998
        self.own(&texture.0)?;
999
        self.read_back(&texture.0, region, format)
1000
    }
1001
1002
    fn can_read_texture(&mut self, texture: &VulkanTexture) -> Result<bool, Error> {
1003
        Ok(texture.0.external || texture.0.layout() != vk::ImageLayout::UNDEFINED)
1004
    }
1005
1006
    fn map_texture<'a>(&mut self, mapping: &'a VulkanMapping) -> Result<&'a [u8], Error> {
1007
        Ok(&mapping.data)
1008
    }
1009
}
1010
1011
impl ImportMemWl for VulkanRenderer {
1012
    /// Keeps one texture per surface and uploads only the damaged rows into
1013
    /// it, straight from the client's shm pool.
1014
    fn import_shm_buffer(
1015
        &mut self,
1016
        buffer: &WlBuffer,
1017
        surface: Option<&SurfaceData>,
1018
        damage: &[Rectangle<i32, Buffer>],
1019
    ) -> Result<VulkanTexture, Error> {
1020
        let cache = surface.map(|s| {
1021
            s.data_map
1022
                .insert_if_missing_threadsafe(|| ShmCache(Mutex::new(None)));
1023
            s.data_map.get::<ShmCache>().expect("just inserted")
1024
        });
1025
        with_buffer_contents(buffer, |ptr, len, data| {
1026
            let fourcc = shm_format_to_fourcc(data.format).ok_or(Error::ShmAccess)?;
1027
            let (w, h, stride, offset) = (
1028
                data.width as usize,
1029
                data.height as usize,
1030
                data.stride as usize,
1031
                data.offset as usize,
1032
            );
1033
            let line = w * BYTES_PER_PIXEL;
1034
            if w == 0 || h == 0 || stride < line || offset + stride * (h - 1) + line > len {
1035
                return Err(Error::ShmAccess);
1036
            }
1037
            let size = Size::from((data.width, data.height));
1038
            let full = Rectangle::from_size(size);
1039
            let reused = cache
1040
                .and_then(|c| c.0.lock().unwrap_or_else(|e| e.into_inner()).clone())
1041
                .filter(|t| {
1042
                    t.0.fourcc == fourcc
1043
                        && t.size() == size
1044
                        && Arc::ptr_eq(&t.0.shared, &self.shared)
1045
                });
1046
            let (texture, region) = match reused {
1047
                Some(t) => {
1048
                    let dirty = damage
1049
                        .iter()
1050
                        .filter_map(|d| d.intersection(full))
1051
                        .reduce(|a, b| a.merge(b));
1052
                    (t, if damage.is_empty() { Some(full) } else { dirty })
1053
                }
1054
                None => {
1055
                    let usage = vk::ImageUsageFlags::SAMPLED
1056
                        | vk::ImageUsageFlags::TRANSFER_DST
1057
                        | vk::ImageUsageFlags::TRANSFER_SRC;
1058
                    let image =
1059
                        ImageInner::create(&self.shared, fourcc, w as u32, h as u32, usage)?;
1060
                    (VulkanTexture(image), Some(full))
1061
                }
1062
            };
1063
            if let Some(region) = region {
1064
                let x = region.loc.x as usize * BYTES_PER_PIXEL;
1065
                let rw = region.size.w as usize * BYTES_PER_PIXEL;
1066
                self.upload_with(&texture.0, region, |y| unsafe {
1067
                    std::slice::from_raw_parts(ptr.add(offset + y * stride + x), rw)
1068
                })?;
1069
            }
1070
            if let Some(c) = cache {
1071
                *c.0.lock().unwrap_or_else(|e| e.into_inner()) = Some(texture.clone());
1072
            }
1073
            Ok(texture)
1074
        })
1075
        .map_err(|_| Error::ShmAccess)?
1076
    }
1077
}
1078
1079
/// The texture last imported for a surface's shm buffers.
1080
struct ShmCache(Mutex<Option<VulkanTexture>>);