External memory fd implementation

examples/standalone/custom_backend/src/custom.rs

@@ -180,6 +180,15 @@ impl DeviceInterface for CustomDevice {
         unimplemented!()
     }
 
+    fn create_buffer_external_memory_fd(
+        &self,
+        _fd: i32,
+        _offset: u64,
+        _desc: &wgpu::BufferDescriptor<'_>,
+    ) -> wgpu::custom::DispatchBuffer {
+        unimplemented!()
+    }
+
     fn create_texture(&self, _desc: &wgpu::TextureDescriptor<'_>) -> wgpu::custom::DispatchTexture {
         unimplemented!()
     }

wgpu-core/src/device/global.rs

@@ -142,6 +142,58 @@ impl Global {
         (id, Some(error))
     }
 
+    pub fn device_create_buffer_external_memory_fd(
+        &self,
+        device_id: DeviceId,
+        fd: i32,
+        offset: u64,
+        desc: &resource::BufferDescriptor,
+        id_in: Option<id::BufferId>,
+    ) -> (id::BufferId, Option<CreateBufferError>) {
+        profiling::scope!("Device::create_buffer_external_memory_fd");
+
+        let hub = &self.hub;
+        let fid = hub.buffers.prepare(id_in);
+
+        let error = 'error: {
+            let device = self.hub.devices.get(device_id);
+
+            #[cfg(feature = "trace")]
+            if let Some(ref mut trace) = *device.trace.lock() {
+                let mut desc = desc.clone();
+                let mapped_at_creation = core::mem::replace(&mut desc.mapped_at_creation, false);
+                if mapped_at_creation && !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
+                    desc.usage |= wgt::BufferUsages::COPY_DST;
+                }
+                trace.add(trace::Action::CreateBuffer(fid.id(), desc));
+            }
+
+            let buffer = match device.create_buffer_external_memory_fd(fd, offset, desc) {
+                Ok(buffer) => buffer,
+                Err(e) => {
+                    break 'error e;
+                }
+            };
+
+            let id = fid.assign(Fallible::Valid(buffer));
+
+            api_log!(
+                "Device::create_buffer_external_memory_fd({fd}, {offset}, {:?}{}) -> {id:?}",
+                desc.label.as_deref().unwrap_or(""),
+                if desc.mapped_at_creation {
+                    ", mapped_at_creation"
+                } else {
+                    ""
+                }
+            );
+
+            return (id, None);
+        };
+
+        let id = fid.assign(Fallible::Invalid(Arc::new(desc.label.to_string())));
+        (id, Some(error))
+    }
+
     /// Assign `id_in` an error with the given `label`.
     ///
     /// Ensure that future attempts to use `id_in` as a buffer ID will propagate
@@ -390,7 +442,7 @@ impl Global {
         desc: &resource::BufferDescriptor,
         id_in: Option<id::BufferId>,
     ) -> (id::BufferId, Option<CreateBufferError>) {
-        profiling::scope!("Device::create_buffer");
+        profiling::scope!("Device::create_buffer_from_hal");
 
         let hub = &self.hub;
         let fid = hub.buffers.prepare(id_in);
@@ -407,7 +459,7 @@ impl Global {
         let (buffer, err) = device.create_buffer_from_hal(Box::new(hal_buffer), desc);
 
         let id = fid.assign(buffer);
-        api_log!("Device::create_buffer -> {id:?}");
+        api_log!("Device::create_buffer_from_hal -> {id:?}");
 
         (id, err)
     }

wgpu-core/src/device/resource.rs

@@ -779,6 +779,160 @@ impl Device {
         Ok(buffer)
     }
 
+    pub(crate) fn create_buffer_external_memory_fd(
+        self: &Arc<Self>,
+        fd: i32,
+        offset: u64,
+        desc: &resource::BufferDescriptor,
+    ) -> Result<Arc<Buffer>, resource::CreateBufferError> {
+        self.check_is_valid()?;
+
+        self.require_features(wgt::Features::VULKAN_EXTERNAL_MEMORY_FD)
+            .map_err(|_| resource::CreateBufferError::ExternalMemoryFeatureNotEnabled)?;
+
+        if desc.size > self.limits.max_buffer_size {
+            return Err(resource::CreateBufferError::MaxBufferSize {
+                requested: desc.size,
+                maximum: self.limits.max_buffer_size,
+            });
+        }
+
+        if desc.usage.contains(wgt::BufferUsages::INDEX)
+            && desc.usage.contains(
+                wgt::BufferUsages::VERTEX
+                    | wgt::BufferUsages::UNIFORM
+                    | wgt::BufferUsages::INDIRECT
+                    | wgt::BufferUsages::STORAGE,
+            )
+        {
+            self.require_downlevel_flags(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)?;
+        }
+        if desc.mapped_at_creation {
+            return Err(resource::CreateBufferError::ExternalMemoryMappedAtCreation);
+        }
+
+        if desc.usage.is_empty()
+            || desc.usage.contains_unknown_bits()
+            || desc
+                .usage
+                .intersects(wgt::BufferUsages::MAP_READ | wgt::BufferUsages::MAP_WRITE)
+        {
+            return Err(resource::CreateBufferError::InvalidUsage(desc.usage));
+        }
+
+        if !self
+            .features
+            .contains(wgt::Features::MAPPABLE_PRIMARY_BUFFERS)
+        {
+            use wgt::BufferUsages as Bu;
+            let write_mismatch = desc.usage.contains(Bu::MAP_WRITE)
+                && !(Bu::MAP_WRITE | Bu::COPY_SRC).contains(desc.usage);
+            let read_mismatch = desc.usage.contains(Bu::MAP_READ)
+                && !(Bu::MAP_READ | Bu::COPY_DST).contains(desc.usage);
+            if write_mismatch || read_mismatch {
+                return Err(resource::CreateBufferError::UsageMismatch(desc.usage));
+            }
+        }
+
+        let mut usage = conv::map_buffer_usage(desc.usage);
+
+        if desc.usage.contains(wgt::BufferUsages::INDIRECT) {
+            self.require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)?;
+            // We are going to be reading from it, internally;
+            // when validating the content of the buffer
+            usage |= wgt::BufferUses::STORAGE_READ_ONLY | wgt::BufferUses::STORAGE_READ_WRITE;
+        }
+
+        if desc.usage.contains(wgt::BufferUsages::QUERY_RESOLVE) {
+            usage |= TIMESTAMP_NORMALIZATION_BUFFER_USES;
+        }
+
+        if desc.mapped_at_creation {
+            if desc.size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
+                return Err(resource::CreateBufferError::UnalignedSize);
+            }
+            if !desc.usage.contains(wgt::BufferUsages::MAP_WRITE) {
+                // we are going to be copying into it, internally
+                usage |= wgt::BufferUses::COPY_DST;
+            }
+        } else {
+            // We are required to zero out (initialize) all memory. This is done
+            // on demand using clear_buffer which requires write transfer usage!
+            usage |= wgt::BufferUses::COPY_DST;
+        }
+
+        let actual_size = if desc.size == 0 {
+            wgt::COPY_BUFFER_ALIGNMENT
+        } else if desc.usage.contains(wgt::BufferUsages::VERTEX) {
+            // Bumping the size by 1 so that we can bind an empty range at the
+            // end of the buffer.
+            desc.size + 1
+        } else {
+            desc.size
+        };
+        let clear_remainder = actual_size % wgt::COPY_BUFFER_ALIGNMENT;
+        let aligned_size = if clear_remainder != 0 {
+            actual_size + wgt::COPY_BUFFER_ALIGNMENT - clear_remainder
+        } else {
+            actual_size
+        };
+
+        let hal_desc = hal::BufferDescriptor {
+            label: desc.label.to_hal(self.instance_flags),
+            size: aligned_size,
+            usage,
+            memory_flags: hal::MemoryFlags::empty(),
+        };
+        let buffer = unsafe {
+            self.raw()
+                .create_buffer_external_memory_fd(fd, offset, &hal_desc)
+        }
+        .map_err(|e| self.handle_hal_error_with_nonfatal_oom(e))?;
+
+        let timestamp_normalization_bind_group = Snatchable::new(
+            self.timestamp_normalizer
+                .get()
+                .unwrap()
+                .create_normalization_bind_group(
+                    self,
+                    &*buffer,
+                    desc.label.as_deref(),
+                    desc.size,
+                    desc.usage,
+                )?,
+        );
+
+        let indirect_validation_bind_groups =
+            self.create_indirect_validation_bind_groups(buffer.as_ref(), desc.size, desc.usage)?;
+
+        let mut init_tracker = BufferInitTracker::new(aligned_size);
+        init_tracker.drain(0..aligned_size);
+        let buffer = Buffer {
+            raw: Snatchable::new(buffer),
+            device: self.clone(),
+            usage: desc.usage,
+            size: desc.size,
+            initialization_status: RwLock::new(rank::BUFFER_INITIALIZATION_STATUS, init_tracker),
+            map_state: Mutex::new(rank::BUFFER_MAP_STATE, resource::BufferMapState::Idle),
+            label: desc.label.to_string(),
+            tracking_data: TrackingData::new(self.tracker_indices.buffers.clone()),
+            bind_groups: Mutex::new(rank::BUFFER_BIND_GROUPS, WeakVec::new()),
+            timestamp_normalization_bind_group,
+            indirect_validation_bind_groups,
+        };
+
+        let buffer = Arc::new(buffer);
+
+        // Don't zero initialize the memory
+
+        self.trackers
+            .lock()
+            .buffers
+            .insert_single(&buffer, wgt::BufferUses::empty());
+
+        Ok(buffer)
+    }
+
     pub(crate) fn create_texture_from_hal(
         self: &Arc<Self>,
         hal_texture: Box<dyn hal::DynTexture>,

wgpu-core/src/resource.rs

@@ -777,6 +777,12 @@ pub enum CreateBufferError {
     MissingDownlevelFlags(#[from] MissingDownlevelFlags),
     #[error("Failed to create bind group for indirect buffer validation: {0}")]
     IndirectValidationBindGroup(DeviceError),
+    #[error(
+        "Buffers created with `create_buffer_external_memory_fd` cannot be mapped at creation"
+    )]
+    ExternalMemoryMappedAtCreation,
+    #[error("The VULKAN_EXTERNAL_MEMORY_FD feature must be enabled in order to create buffers from external memory file descriptors")]
+    ExternalMemoryFeatureNotEnabled,
 }
 
 crate::impl_resource_type!(Buffer);

wgpu-hal/src/dx12/device.rs

@@ -458,6 +458,14 @@ impl crate::Device for super::Device {
             allocation,
         })
     }
+    unsafe fn create_buffer_external_memory_fd(
+        &self,
+        _fd: i32,
+        _offset: u64,
+        _desc: &crate::BufferDescriptor,
+    ) -> Result<<Self::A as crate::Api>::Buffer, crate::DeviceError> {
+        unreachable!()
+    }
 
     unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
         suballocation::DeviceAllocationContext::from(self)

wgpu-hal/src/dynamic/device.rs

@@ -22,6 +22,12 @@ pub trait DynDevice: DynResource {
         &self,
         desc: &BufferDescriptor,
     ) -> Result<Box<dyn DynBuffer>, DeviceError>;
+    unsafe fn create_buffer_external_memory_fd(
+        &self,
+        fd: i32,
+        offset: u64,
+        desc: &BufferDescriptor,
+    ) -> Result<Box<dyn DynBuffer>, DeviceError>;
 
     unsafe fn destroy_buffer(&self, buffer: Box<dyn DynBuffer>);
     unsafe fn add_raw_buffer(&self, buffer: &dyn DynBuffer);
@@ -182,6 +188,17 @@ impl<D: Device + DynResource> DynDevice for D {
     ) -> Result<Box<dyn DynBuffer>, DeviceError> {
         unsafe { D::create_buffer(self, desc) }.map(|b| -> Box<dyn DynBuffer> { Box::new(b) })
     }
+    unsafe fn create_buffer_external_memory_fd(
+        &self,
+        fd: i32,
+        offset: u64,
+        desc: &BufferDescriptor,
+    ) -> Result<Box<dyn DynBuffer>, DeviceError> {
+        unsafe {
+            D::create_buffer_external_memory_fd(self, fd, offset, desc)
+                .map(|b| -> Box<dyn DynBuffer> { Box::new(b) })
+        }
+    }
 
     unsafe fn destroy_buffer(&self, buffer: Box<dyn DynBuffer>) {
         unsafe { D::destroy_buffer(self, buffer.unbox()) };

wgpu-hal/src/gles/device.rs

@@ -640,6 +640,14 @@ impl crate::Device for super::Device {
             offset_of_current_mapping: Arc::new(MaybeMutex::new(0)),
         })
     }
+    unsafe fn create_buffer_external_memory_fd(
+        &self,
+        _fd: i32,
+        _offset: u64,
+        _desc: &crate::BufferDescriptor,
+    ) -> Result<<Self::A as crate::Api>::Buffer, crate::DeviceError> {
+        unreachable!()
+    }
 
     unsafe fn destroy_buffer(&self, buffer: super::Buffer) {
         if let Some(raw) = buffer.raw {

wgpu-hal/src/lib.rs

@@ -735,6 +735,13 @@ pub trait Device: WasmNotSendSync {
         desc: &BufferDescriptor,
     ) -> Result<<Self::A as Api>::Buffer, DeviceError>;
 
+    unsafe fn create_buffer_external_memory_fd(
+        &self,
+        fd: i32,
+        offset: u64,
+        desc: &BufferDescriptor,
+    ) -> Result<<Self::A as Api>::Buffer, DeviceError>;
+
     /// Free `buffer` and any GPU resources it owns.
     ///
     /// Note that backends are allowed to allocate GPU memory for buffers from

wgpu-hal/src/metal/device.rs

@@ -379,6 +379,14 @@ impl crate::Device for super::Device {
             })
         })
     }
+    unsafe fn create_buffer_external_memory_fd(
+        &self,
+        _fd: i32,
+        _offset: u64,
+        _desc: &crate::BufferDescriptor,
+    ) -> Result<<Self::A as crate::Api>::Buffer, crate::DeviceError> {
+        unreachable!()
+    }
     unsafe fn destroy_buffer(&self, _buffer: super::Buffer) {
         self.counters.buffers.sub(1);
     }

wgpu-hal/src/noop/mod.rs

@@ -286,6 +286,14 @@ impl crate::Device for Context {
     unsafe fn create_buffer(&self, desc: &crate::BufferDescriptor) -> DeviceResult<Buffer> {
         Buffer::new(desc)
     }
+    unsafe fn create_buffer_external_memory_fd(
+        &self,
+        fd: i32,
+        offset: u64,
+        desc: &crate::BufferDescriptor,
+    ) -> Result<<Self::A as crate::Api>::Buffer, crate::DeviceError> {
+        unreachable!()
+    }
 
     unsafe fn destroy_buffer(&self, buffer: Buffer) {}
     unsafe fn add_raw_buffer(&self, _buffer: &Buffer) {}