Add multiplexer abstraction

Adds a new "mux" module which can have multiple backends. As of this
commit, it's not wired up at all, but the functionality should be
reasonably complete.

Minor tweaks to the backend trait to accommodate this, mostly changing
Fence and Semaphore to references so they don't need to be Copy.

Part of the work toward #95

Cargo.lock

@@ -851,6 +851,7 @@ dependencies = [
  "ash-window",
  "bitflags",
  "raw-window-handle",
+ "smallvec",
  "winapi 0.3.9",
  "wio",
 ]
@@ -1036,9 +1037,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
 
 [[package]]
 name = "smallvec"
-version = "1.4.2"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"
+checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
 [[package]]
 name = "smithay-client-toolkit"

piet-gpu-hal/Cargo.toml

@@ -11,7 +11,9 @@ ash = "0.31"
 ash-window = "0.5"
 raw-window-handle = "0.3"
 bitflags = "1.2.1"
+smallvec = "1.6.1"
 
+[target.'cfg(target_os="windows")'.dependencies]
 winapi = { version = "0.3.9", features = [
     'd3d12', 'd3d12sdklayers', 'd3dcommon', 'd3dcompiler', 'dxgi',
     'dxgi1_2', 'dxgi1_3', 'dxgi1_4', 'dxgidebug', 'dxgiformat', 'dxgitype',

piet-gpu-hal/examples/dx12_toy.rs

@@ -50,8 +50,8 @@ void main(SPIRV_Cross_Input stage_input)
 "#;
 
 fn toy() -> Result<(), Error> {
-    let instance = dx12::Dx12Instance::new()?;
+    let (instance, _surface) = dx12::Dx12Instance::new(None)?;
-    let device = instance.device()?;
+    let device = instance.device(None)?;
     let buf = device.create_buffer(
         1024,
         BufferUsage::MAP_READ
@@ -87,7 +87,7 @@ fn toy() -> Result<(), Error> {
         cmd_buf.host_barrier();
         cmd_buf.finish();
         device.run_cmd_bufs(&[&cmd_buf], &[], &[], Some(&fence))?;
-        device.wait_and_reset(&[fence])?;
+        device.wait_and_reset(&[&fence])?;
         let mut readback: Vec<u32> = vec![0u32; 256];
         device.read_buffer(&buf, readback.as_mut_ptr() as *mut u8, 0, 1024)?;
         println!("{:?}", readback);

piet-gpu-hal/src/dx12.rs

@@ -9,6 +9,8 @@ use winapi::shared::dxgi1_3;
 use winapi::shared::minwindef::TRUE;
 use winapi::um::d3d12;
 
+use smallvec::SmallVec;
+
 use crate::{BufferUsage, Error, ImageLayout};
 
 use self::wrappers::{
@@ -19,6 +21,9 @@ pub struct Dx12Instance {
     factory: Factory4,
 }
 
+// TODO
+pub struct Dx12Surface;
+
 pub struct Dx12Device {
     device: Device,
     command_allocator: CommandAllocator,
@@ -95,7 +100,7 @@ impl Dx12Instance {
     ///
     /// TODO: take a raw window handle.
     /// TODO: can probably be a trait.
-    pub fn new() -> Result<Dx12Instance, Error> {
+    pub fn new(window_handle: Option<&dyn raw_window_handle::HasRawWindowHandle>) -> Result<(Dx12Instance, Option<Dx12Surface>), Error> {
         unsafe {
             #[cfg(debug_assertions)]
             if let Err(e) = wrappers::enable_debug_layer() {
@@ -110,7 +115,7 @@ impl Dx12Instance {
             let factory_flags: u32 = 0;
 
             let factory = Factory4::create(factory_flags)?;
-            Ok(Dx12Instance { factory })
+            Ok((Dx12Instance { factory }, None))
         }
     }
 
@@ -118,7 +123,7 @@ impl Dx12Instance {
     ///
     /// TODO: handle window.
     /// TODO: probably can also be trait'ified.
-    pub fn device(&self) -> Result<Dx12Device, Error> {
+    pub fn device(&self, surface: Option<&Dx12Surface>) -> Result<Dx12Device, Error> {
         unsafe {
             let device = Device::create_device(&self.factory)?;
             let list_type = d3d12::D3D12_COMMAND_LIST_TYPE_DIRECT;
@@ -262,16 +267,15 @@ impl crate::Device for Dx12Device {
     unsafe fn run_cmd_bufs(
         &self,
         cmd_bufs: &[&Self::CmdBuf],
-        wait_semaphores: &[Self::Semaphore],
+        wait_semaphores: &[&Self::Semaphore],
-        signal_semaphores: &[Self::Semaphore],
+        signal_semaphores: &[&Self::Semaphore],
         fence: Option<&Self::Fence>,
     ) -> Result<(), Error> {
         // TODO: handle semaphores
-        // SmallVec?
         let lists = cmd_bufs
             .iter()
             .map(|c| c.0.as_raw_command_list())
-            .collect::<Vec<_>>();
+            .collect::<SmallVec<[_; 4]>>();
         self.command_queue.execute_command_lists(&lists);
         if let Some(fence) = fence {
             let val = fence.val.get() + 1;
@@ -319,7 +323,7 @@ impl crate::Device for Dx12Device {
         Ok(Fence { fence, event, val })
     }
 
-    unsafe fn wait_and_reset(&self, fences: &[Self::Fence]) -> Result<(), Error> {
+    unsafe fn wait_and_reset(&self, fences: &[&Self::Fence]) -> Result<(), Error> {
         for fence in fences {
             // TODO: probably handle errors here.
             let _status = fence.event.wait(winapi::um::winbase::INFINITE);

piet-gpu-hal/src/hub.rs

@@ -133,7 +133,7 @@ impl Session {
                 if let Ok(true) = self.0.device.get_fence_status(pending[i].fence) {
                     let item = pending.swap_remove(i);
                     // TODO: wait is superfluous, can just reset
-                    let _ = self.0.device.wait_and_reset(&[item.fence]);
+                    let _ = self.0.device.wait_and_reset(&[&item.fence]);
                     let mut pool = self.0.cmd_buf_pool.lock().unwrap();
                     pool.push((item.cmd_buf, item.fence));
                     std::mem::drop(item.resources);
@@ -151,8 +151,8 @@ impl Session {
     pub unsafe fn run_cmd_buf(
         &self,
         cmd_buf: CmdBuf,
-        wait_semaphores: &[Semaphore],
+        wait_semaphores: &[&Semaphore],
-        signal_semaphores: &[Semaphore],
+        signal_semaphores: &[&Semaphore],
     ) -> Result<SubmittedCmdBuf, Error> {
         // Again, SmallVec here?
         let mut cmd_bufs = Vec::with_capacity(2);
@@ -322,7 +322,7 @@ impl SubmittedCmdBuf {
         let item = self.0.take().unwrap();
         if let Some(session) = Weak::upgrade(&self.1) {
             unsafe {
-                session.device.wait_and_reset(&[item.fence])?;
+                session.device.wait_and_reset(&[&item.fence])?;
             }
             session
                 .cmd_buf_pool

piet-gpu-hal/src/lib.rs

@@ -6,8 +6,17 @@ use bitflags::bitflags;
 
 pub mod hub;
 
-#[cfg(target_os = "windows")]
+#[macro_use]
-pub mod dx12;
+mod macros;
+
+// TODO make this not pub
+pub mod mux;
+
+mux! {
+    #[cfg(vk)]
+    pub mod dx12;
+}
+#[cfg(not(target_os = "macos"))]
 pub mod vulkan;
 
 /// This isn't great but is expedient.
@@ -178,8 +187,8 @@ pub trait Device: Sized {
     unsafe fn run_cmd_bufs(
         &self,
         cmd_buf: &[&Self::CmdBuf],
-        wait_semaphores: &[Self::Semaphore],
+        wait_semaphores: &[&Self::Semaphore],
-        signal_semaphores: &[Self::Semaphore],
+        signal_semaphores: &[&Self::Semaphore],
         fence: Option<&Self::Fence>,
     ) -> Result<(), Error>;
 
@@ -217,7 +226,7 @@ pub trait Device: Sized {
 
     unsafe fn create_semaphore(&self) -> Result<Self::Semaphore, Error>;
     unsafe fn create_fence(&self, signaled: bool) -> Result<Self::Fence, Error>;
-    unsafe fn wait_and_reset(&self, fences: &[Self::Fence]) -> Result<(), Error>;
+    unsafe fn wait_and_reset(&self, fences: &[&Self::Fence]) -> Result<(), Error>;
     unsafe fn get_fence_status(&self, fence: Self::Fence) -> Result<bool, Error>;
 
     unsafe fn create_sampler(&self, params: SamplerParams) -> Result<Self::Sampler, Error>;

piet-gpu-hal/src/macros.rs

@@ -0,0 +1,79 @@
+// Copyright 2021 The piet-gpu authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Also licensed under MIT license, at your choice.
+
+//! Macros, mostly to automate backend selection tedium.
+
+#[macro_export]
+macro_rules! mux {
+    ( #[cfg(vk)] $($tokens:tt)* ) => {
+        #[cfg(not(target_os="macos"))] $( $tokens )*
+    };
+
+    ( #[cfg(dx12)] $($tokens:tt)* ) => {
+        #[cfg(target_os="windows")] $( $tokens )*
+    };
+}
+
+#[macro_export]
+macro_rules! mux_enum {
+    ( $(#[$outer:meta])* $v:vis enum $name:ident {
+        Vk($vk:ty),
+        Dx12($dx12:ty),
+    } ) => {
+        $(#[$outer])* $v enum $name {
+            #[cfg(not(target_os="macos"))]
+            Vk($vk),
+            #[cfg(target_os="windows")]
+            Dx12($dx12),
+        }
+
+        impl $name {
+            $crate::mux! {
+                #[cfg(vk)]
+                #[allow(unused)]
+                fn vk(&self) -> &$vk {
+                    match self {
+                        $name::Vk(x) => x,
+                        _ => panic!("downcast error")
+                    }
+                }
+            }
+
+            $crate::mux! {
+                #[cfg(dx12)]
+                #[allow(unused)]
+                fn dx12(&self) -> &$dx12 {
+                    match self {
+                        $name::Dx12(x) => x,
+                        _ => panic!("downcast error")
+                    }
+                }
+            }
+        }
+    };
+}
+
+macro_rules! mux_device_enum {
+    ( $(#[$outer:meta])* $assoc_type: ident) => {
+        $crate::mux_enum! {
+            $(#[$outer])*
+            pub enum $assoc_type {
+                Vk(<$crate::vulkan::VkDevice as $crate::Device>::$assoc_type),
+                Dx12(<$crate::dx12::Dx12Device as $crate::Device>::$assoc_type),
+            }
+        }
+    }
+}

piet-gpu-hal/src/mux.rs

@@ -0,0 +1,461 @@
+// Copyright 2021 The piet-gpu authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Also licensed under MIT license, at your choice.
+
+//! A multiplexer module that selects a back-end at runtime.
+
+use smallvec::SmallVec;
+
+use crate::dx12;
+use crate::vulkan;
+use crate::CmdBuf as CmdBufTrait;
+use crate::DescriptorSetBuilder as DescriptorSetBuilderTrait;
+use crate::Device as DeviceTrait;
+use crate::PipelineBuilder as PipelineBuilderTrait;
+use crate::{BufferUsage, Error, GpuInfo, ImageLayout};
+
+mux_enum! {
+    /// An instance, selected from multiple backends.
+    pub enum Instance {
+        Vk(vulkan::VkInstance),
+        Dx12(dx12::Dx12Instance),
+    }
+}
+
+mux_enum! {
+    /// A device, selected from multiple backends.
+    pub enum Device {
+        Vk(vulkan::VkDevice),
+        Dx12(dx12::Dx12Device),
+    }
+}
+
+mux_enum! {
+    /// A surface, which can apply to one of multiple backends.
+    pub enum Surface {
+        Vk(vulkan::VkSurface),
+        Dx12(dx12::Dx12Surface),
+    }
+}
+
+mux_device_enum! { Buffer }
+mux_device_enum! { Image }
+mux_device_enum! { Fence }
+mux_device_enum! { Semaphore }
+mux_device_enum! { PipelineBuilder }
+mux_device_enum! { Pipeline }
+mux_device_enum! { DescriptorSetBuilder }
+mux_device_enum! { DescriptorSet }
+mux_device_enum! { CmdBuf }
+mux_device_enum! { QueryPool }
+
+/// The code for a shader, either as source or intermediate representation.
+pub enum ShaderCode<'a> {
+    Spv(&'a [u8]),
+    Hlsl(&'a str),
+}
+
+impl Instance {
+    pub fn new(
+        window_handle: Option<&dyn raw_window_handle::HasRawWindowHandle>,
+    ) -> Result<(Instance, Option<Surface>), Error> {
+        mux! {
+            #[cfg(vk)]
+            {
+                let result = vulkan::VkInstance::new(window_handle);
+                if let Ok((instance, surface)) = result {
+                    return Ok((Instance::Vk(instance), surface.map(Surface::Vk)));
+                }
+            }
+        }
+        mux! {
+            #[cfg(dx12)]
+            {
+                let result = dx12::Dx12Instance::new(window_handle);
+                if let Ok((instance, surface)) = result {
+                    return Ok((Instance::Dx12(instance), surface.map(Surface::Dx12)));
+                }
+            }
+        }
+        // TODO plumb creation errors through.
+        Err("No suitable instances found".into())
+    }
+
+    pub unsafe fn device(&self, surface: Option<&Surface>) -> Result<Device, Error> {
+        match self {
+            Instance::Vk(i) => i.device(surface.map(Surface::vk)).map(Device::Vk),
+            Instance::Dx12(i) => i.device(surface.map(Surface::dx12)).map(Device::Dx12),
+        }
+    }
+}
+
+// This is basically re-exporting the backend device trait, and we could do that,
+// but not doing so lets us diverge more easily (at the moment, the divergence is
+// missing functionality).
+impl Device {
+    pub fn query_gpu_info(&self) -> GpuInfo {
+        match self {
+            Device::Vk(d) => d.query_gpu_info(),
+            Device::Dx12(d) => d.query_gpu_info(),
+        }
+    }
+
+    pub fn create_buffer(&self, size: u64, usage: BufferUsage) -> Result<Buffer, Error> {
+        match self {
+            Device::Vk(d) => d.create_buffer(size, usage).map(Buffer::Vk),
+            Device::Dx12(d) => d.create_buffer(size, usage).map(Buffer::Dx12),
+        }
+    }
+
+    pub unsafe fn destroy_buffer(&self, buffer: &Buffer) -> Result<(), Error> {
+        match self {
+            Device::Vk(d) => d.destroy_buffer(buffer.vk()),
+            Device::Dx12(d) => d.destroy_buffer(buffer.dx12()),
+        }
+    }
+
+    pub unsafe fn create_fence(&self, signaled: bool) -> Result<Fence, Error> {
+        match self {
+            Device::Vk(d) => d.create_fence(signaled).map(Fence::Vk),
+            Device::Dx12(d) => d.create_fence(signaled).map(Fence::Dx12),
+        }
+    }
+
+    pub unsafe fn wait_and_reset(&self, fences: &[&Fence]) -> Result<(), Error> {
+        match self {
+            Device::Vk(d) => {
+                let fences = fences
+                    .iter()
+                    .copied()
+                    .map(Fence::vk)
+                    .collect::<SmallVec<[_; 4]>>();
+                d.wait_and_reset(&*fences)
+            }
+            Device::Dx12(d) => {
+                let fences = fences
+                    .iter()
+                    .copied()
+                    .map(Fence::dx12)
+                    .collect::<SmallVec<[_; 4]>>();
+                d.wait_and_reset(&*fences)
+            }
+            // Probably need to change device trait to accept &Fence
+            _ => todo!(),
+        }
+    }
+
+    pub unsafe fn pipeline_builder(&self) -> PipelineBuilder {
+        match self {
+            Device::Vk(d) => PipelineBuilder::Vk(d.pipeline_builder()),
+            Device::Dx12(d) => PipelineBuilder::Dx12(d.pipeline_builder()),
+        }
+    }
+
+    pub unsafe fn descriptor_set_builder(&self) -> DescriptorSetBuilder {
+        match self {
+            Device::Vk(d) => DescriptorSetBuilder::Vk(d.descriptor_set_builder()),
+            Device::Dx12(d) => DescriptorSetBuilder::Dx12(d.descriptor_set_builder()),
+        }
+    }
+
+    pub fn create_cmd_buf(&self) -> Result<CmdBuf, Error> {
+        match self {
+            Device::Vk(d) => d.create_cmd_buf().map(CmdBuf::Vk),
+            Device::Dx12(d) => d.create_cmd_buf().map(CmdBuf::Dx12),
+        }
+    }
+
+    pub fn create_query_pool(&self, n_queries: u32) -> Result<QueryPool, Error> {
+        match self {
+            Device::Vk(d) => d.create_query_pool(n_queries).map(QueryPool::Vk),
+            Device::Dx12(d) => d.create_query_pool(n_queries).map(QueryPool::Dx12),
+        }
+    }
+
+    pub unsafe fn fetch_query_pool(&self, pool: &QueryPool) -> Result<Vec<f64>, Error> {
+        match self {
+            Device::Vk(d) => d.fetch_query_pool(pool.vk()),
+            Device::Dx12(d) => d.fetch_query_pool(pool.dx12()),
+        }
+    }
+
+    pub unsafe fn run_cmd_bufs(
+        &self,
+        cmd_bufs: &[&CmdBuf],
+        wait_semaphores: &[&Semaphore],
+        signal_semaphores: &[&Semaphore],
+        fence: Option<&Fence>,
+    ) -> Result<(), Error> {
+        match self {
+            Device::Vk(d) => d.run_cmd_bufs(
+                &cmd_bufs
+                    .iter()
+                    .map(|c| c.vk())
+                    .collect::<SmallVec<[_; 4]>>(),
+                &wait_semaphores.iter().copied().map(Semaphore::vk).collect::<SmallVec<[_; 4]>>(),
+                &signal_semaphores.iter().copied().map(Semaphore::vk).collect::<SmallVec<[_; 4]>>(),
+                fence.map(Fence::vk),
+            ),
+            Device::Dx12(d) => d.run_cmd_bufs(
+                &cmd_bufs
+                    .iter()
+                    .map(|c| c.dx12())
+                    .collect::<SmallVec<[_; 4]>>(),
+                &wait_semaphores.iter().copied().map(Semaphore::dx12).collect::<SmallVec<[_; 4]>>(),
+                &signal_semaphores.iter().copied().map(Semaphore::dx12).collect::<SmallVec<[_; 4]>>(),
+                fence.map(Fence::dx12),
+            ),
+        }
+    }
+
+    pub unsafe fn read_buffer(
+        &self,
+        buffer: &Buffer,
+        dst: *mut u8,
+        offset: u64,
+        size: u64,
+    ) -> Result<(), Error> {
+        match self {
+            Device::Vk(d) => d.read_buffer(buffer.vk(), dst, offset, size),
+            Device::Dx12(d) => d.read_buffer(buffer.dx12(), dst, offset, size),
+        }
+    }
+
+    pub unsafe fn write_buffer(
+        &self,
+        buffer: &Buffer,
+        contents: *const u8,
+        offset: u64,
+        size: u64,
+    ) -> Result<(), Error> {
+        match self {
+            Device::Vk(d) => d.write_buffer(buffer.vk(), contents, offset, size),
+            Device::Dx12(d) => d.write_buffer(buffer.dx12(), contents, offset, size),
+        }
+    }
+}
+
+impl PipelineBuilder {
+    pub fn add_buffers(&mut self, n_buffers: u32) {
+        match self {
+            PipelineBuilder::Vk(x) => x.add_buffers(n_buffers),
+            PipelineBuilder::Dx12(x) => x.add_buffers(n_buffers),
+        }
+    }
+
+    pub fn add_images(&mut self, n_buffers: u32) {
+        match self {
+            PipelineBuilder::Vk(x) => x.add_images(n_buffers),
+            PipelineBuilder::Dx12(x) => x.add_images(n_buffers),
+        }
+    }
+
+    pub fn add_textures(&mut self, n_buffers: u32) {
+        match self {
+            PipelineBuilder::Vk(x) => x.add_textures(n_buffers),
+            PipelineBuilder::Dx12(x) => x.add_textures(n_buffers),
+        }
+    }
+
+    pub unsafe fn create_compute_pipeline<'a>(
+        self,
+        device: &Device,
+        code: ShaderCode<'a>,
+    ) -> Result<Pipeline, Error> {
+        match self {
+            PipelineBuilder::Vk(x) => {
+                let shader_code = match code {
+                    ShaderCode::Spv(spv) => spv,
+                    // Panic or return "incompatible shader" error here?
+                    _ => panic!("Vulkan backend requires shader code in SPIR-V format"),
+                };
+                x.create_compute_pipeline(device.vk(), shader_code)
+                    .map(Pipeline::Vk)
+            }
+            PipelineBuilder::Dx12(x) => {
+                let shader_code = match code {
+                    ShaderCode::Hlsl(hlsl) => hlsl,
+                    // Panic or return "incompatible shader" error here?
+                    _ => panic!("DX12 backend requires shader code in HLSL format"),
+                };
+                x.create_compute_pipeline(device.dx12(), shader_code)
+                    .map(Pipeline::Dx12)
+            }
+        }
+    }
+}
+
+impl DescriptorSetBuilder {
+    pub fn add_buffers(&mut self, buffers: &[Buffer]) {
+        match self {
+            DescriptorSetBuilder::Vk(x) => {
+                x.add_buffers(&buffers.iter().map(Buffer::vk).collect::<SmallVec<[_; 8]>>())
+            }
+            DescriptorSetBuilder::Dx12(x) => x.add_buffers(
+                &buffers
+                    .iter()
+                    .map(Buffer::dx12)
+                    .collect::<SmallVec<[_; 8]>>(),
+            ),
+        }
+    }
+
+    pub fn add_images(&mut self, images: &[Image]) {
+        match self {
+            DescriptorSetBuilder::Vk(x) => {
+                x.add_images(&images.iter().map(Image::vk).collect::<SmallVec<[_; 8]>>())
+            }
+            DescriptorSetBuilder::Dx12(x) => {
+                x.add_images(&images.iter().map(Image::dx12).collect::<SmallVec<[_; 8]>>())
+            }
+        }
+    }
+
+    pub fn add_textures(&mut self, images: &[Image]) {
+        match self {
+            DescriptorSetBuilder::Vk(x) => {
+                x.add_textures(&images.iter().map(Image::vk).collect::<SmallVec<[_; 8]>>())
+            }
+            DescriptorSetBuilder::Dx12(x) => {
+                x.add_textures(&images.iter().map(Image::dx12).collect::<SmallVec<[_; 8]>>())
+            }
+        }
+    }
+
+    pub unsafe fn build(
+        self,
+        device: &Device,
+        pipeline: &Pipeline,
+    ) -> Result<DescriptorSet, Error> {
+        match self {
+            DescriptorSetBuilder::Vk(x) => {
+                x.build(device.vk(), pipeline.vk()).map(DescriptorSet::Vk)
+            }
+            DescriptorSetBuilder::Dx12(x) => x
+                .build(device.dx12(), pipeline.dx12())
+                .map(DescriptorSet::Dx12),
+        }
+    }
+}
+
+impl CmdBuf {
+    pub unsafe fn begin(&mut self) {
+        match self {
+            CmdBuf::Vk(c) => c.begin(),
+            CmdBuf::Dx12(c) => c.begin(),
+        }
+    }
+
+    pub unsafe fn finish(&mut self) {
+        match self {
+            CmdBuf::Vk(c) => c.finish(),
+            CmdBuf::Dx12(c) => c.finish(),
+        }
+    }
+
+    pub unsafe fn dispatch(
+        &mut self,
+        pipeline: &Pipeline,
+        descriptor_set: &DescriptorSet,
+        size: (u32, u32, u32),
+    ) {
+        match self {
+            CmdBuf::Vk(c) => c.dispatch(pipeline.vk(), descriptor_set.vk(), size),
+            CmdBuf::Dx12(c) => c.dispatch(pipeline.dx12(), descriptor_set.dx12(), size),
+        }
+    }
+
+    pub unsafe fn memory_barrier(&mut self) {
+        match self {
+            CmdBuf::Vk(c) => c.memory_barrier(),
+            CmdBuf::Dx12(c) => c.memory_barrier(),
+        }
+    }
+
+    pub unsafe fn host_barrier(&mut self) {
+        match self {
+            CmdBuf::Vk(c) => c.host_barrier(),
+            CmdBuf::Dx12(c) => c.host_barrier(),
+        }
+    }
+
+    pub unsafe fn image_barrier(
+        &mut self,
+        image: &Image,
+        src_layout: ImageLayout,
+        dst_layout: ImageLayout,
+    ) {
+        match self {
+            CmdBuf::Vk(c) => c.image_barrier(image.vk(), src_layout, dst_layout),
+            CmdBuf::Dx12(c) => c.image_barrier(image.dx12(), src_layout, dst_layout),
+        }
+    }
+
+    pub unsafe fn clear_buffer(&mut self, buffer: &Buffer, size: Option<u64>) {
+        match self {
+            CmdBuf::Vk(c) => c.clear_buffer(buffer.vk(), size),
+            CmdBuf::Dx12(c) => c.clear_buffer(buffer.dx12(), size),
+        }
+    }
+
+    pub unsafe fn copy_buffer(&mut self, src: &Buffer, dst: &Buffer) {
+        match self {
+            CmdBuf::Vk(c) => c.copy_buffer(src.vk(), dst.vk()),
+            CmdBuf::Dx12(c) => c.copy_buffer(src.dx12(), dst.dx12()),
+        }
+    }
+
+    pub unsafe fn copy_image_to_buffer(&mut self, src: &Image, dst: &Buffer) {
+        match self {
+            CmdBuf::Vk(c) => c.copy_image_to_buffer(src.vk(), dst.vk()),
+            CmdBuf::Dx12(c) => c.copy_image_to_buffer(src.dx12(), dst.dx12()),
+        }
+    }
+
+    pub unsafe fn copy_buffer_to_image(&mut self, src: &Buffer, dst: &Image) {
+        match self {
+            CmdBuf::Vk(c) => c.copy_buffer_to_image(src.vk(), dst.vk()),
+            CmdBuf::Dx12(c) => c.copy_buffer_to_image(src.dx12(), dst.dx12()),
+        }
+    }
+
+    pub unsafe fn blit_image(&mut self, src: &Image, dst: &Image) {
+        match self {
+            CmdBuf::Vk(c) => c.blit_image(src.vk(), dst.vk()),
+            CmdBuf::Dx12(c) => c.blit_image(src.dx12(), dst.dx12()),
+        }
+    }
+
+    pub unsafe fn reset_query_pool(&mut self, pool: &QueryPool) {
+        match self {
+            CmdBuf::Vk(c) => c.reset_query_pool(pool.vk()),
+            CmdBuf::Dx12(c) => c.reset_query_pool(pool.dx12()),
+        }
+    }
+
+    pub unsafe fn write_timestamp(&mut self, pool: &QueryPool, query: u32) {
+        match self {
+            CmdBuf::Vk(c) => c.write_timestamp(pool.vk(), query),
+            CmdBuf::Dx12(c) => c.write_timestamp(pool.dx12(), query),
+        }
+    }
+
+    pub unsafe fn finish_timestamps(&mut self, pool: &QueryPool) {
+        match self {
+            CmdBuf::Vk(c) => c.finish_timestamps(pool.vk()),
+            CmdBuf::Dx12(c) => c.finish_timestamps(pool.dx12()),
+        }
+    }
+}

piet-gpu-hal/src/vulkan.rs

@@ -10,6 +10,8 @@ use ash::extensions::{ext::DebugUtils, khr};
 use ash::version::{DeviceV1_0, EntryV1_0, InstanceV1_0, InstanceV1_1};
 use ash::{vk, Device, Entry, Instance};
 
+use smallvec::SmallVec;
+
 use crate::{
     BufferUsage, Device as DeviceTrait, Error, GpuInfo, ImageLayout, SamplerParams, SubgroupSize,
 };
@@ -617,10 +619,15 @@ impl crate::Device for VkDevice {
         Ok(device.create_semaphore(&vk::SemaphoreCreateInfo::default(), None)?)
     }
 
-    unsafe fn wait_and_reset(&self, fences: &[Self::Fence]) -> Result<(), Error> {
+    unsafe fn wait_and_reset(&self, fences: &[&Self::Fence]) -> Result<(), Error> {
         let device = &self.device.device;
-        device.wait_for_fences(fences, true, !0)?;
+        let fences = fences
-        device.reset_fences(fences)?;
+            .iter()
+            .copied()
+            .copied()
+            .collect::<SmallVec<[_; 4]>>();
+        device.wait_for_fences(&fences, true, !0)?;
+        device.reset_fences(&fences)?;
         Ok(())
     }
 
@@ -709,8 +716,8 @@ impl crate::Device for VkDevice {
     unsafe fn run_cmd_bufs(
         &self,
         cmd_bufs: &[&CmdBuf],
-        wait_semaphores: &[Self::Semaphore],
+        wait_semaphores: &[&Self::Semaphore],
-        signal_semaphores: &[Self::Semaphore],
+        signal_semaphores: &[&Self::Semaphore],
         fence: Option<&Self::Fence>,
     ) -> Result<(), Error> {
         let device = &self.device.device;
@@ -722,16 +729,28 @@ impl crate::Device for VkDevice {
         let wait_stages = wait_semaphores
             .iter()
             .map(|_| vk::PipelineStageFlags::ALL_COMMANDS)
-            .collect::<Vec<_>>();
+            .collect::<SmallVec<[_; 4]>>();
-        // Use SmallVec or similar here to reduce allocation?
+        let cmd_bufs = cmd_bufs
-        let cmd_bufs = cmd_bufs.iter().map(|c| c.cmd_buf).collect::<Vec<_>>();
+            .iter()
+            .map(|c| c.cmd_buf)
+            .collect::<SmallVec<[_; 4]>>();
+        let wait_semaphores = wait_semaphores
+            .iter()
+            .copied()
+            .copied()
+            .collect::<SmallVec<[_; 2]>>();
+        let signal_semaphores = signal_semaphores
+            .iter()
+            .copied()
+            .copied()
+            .collect::<SmallVec<[_; 2]>>();
         device.queue_submit(
             self.queue,
             &[vk::SubmitInfo::builder()
                 .command_buffers(&cmd_bufs)
-                .wait_semaphores(wait_semaphores)
+                .wait_semaphores(&wait_semaphores)
                 .wait_dst_stage_mask(&wait_stages)
-                .signal_semaphores(signal_semaphores)
+                .signal_semaphores(&signal_semaphores)
                 .build()],
             fence,
         )?;

piet-gpu/bin/android.rs

@@ -159,8 +159,8 @@ impl GfxState {
                 self.session
                     .run_cmd_buf(
                         cmd_buf,
-                        &[acquisition_semaphore],
+                        &[&acquisition_semaphore],
-                        &[self.present_semaphores[frame_idx]],
+                        &[&self.present_semaphores[frame_idx]],
                     )
                     .unwrap(),
             );

piet-gpu/bin/winit.rs

@@ -110,8 +110,8 @@ fn main() -> Result<(), Error> {
                     submitted = Some(session
                         .run_cmd_buf(
                             cmd_buf,
-                            &[acquisition_semaphore],
+                            &[&acquisition_semaphore],
-                            &[present_semaphores[frame_idx]],
+                            &[&present_semaphores[frame_idx]],
                         )
                         .unwrap());
                     last_frame_idx = frame_idx;