From 884b03fe6c85b7602ee0ce5865bdee290a0b741b Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 20 May 2025 18:31:28 +0000 Subject: [PATCH 1/5] vfio-ioctls: Adopt more generic 'KVM_DEV_VFIO_FILE*' uAPIs The 'KVM_DEV_VFIO_FILE*' uAPIs are more generic ways to accept vfio device files from userspace, supporting both legacy vfio group fd and the new vfio cdev fd. For the same reason, `device_set_group()` is replaced with `device_set_fd()` to to support both legacy vfio mode and vfio cdev mode. Signed-off-by: Bo Chen --- vfio-ioctls/src/vfio_device.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/vfio-ioctls/src/vfio_device.rs b/vfio-ioctls/src/vfio_device.rs index 12221f2..9bc6d58 100644 --- a/vfio-ioctls/src/vfio_device.rs +++ b/vfio-ioctls/src/vfio_device.rs @@ -25,7 +25,7 @@ use crate::vfio_ioctls::*; use crate::{Result, VfioError}; #[cfg(all(feature = "kvm", not(test)))] use kvm_bindings::{ - kvm_device_attr, KVM_DEV_VFIO_GROUP, KVM_DEV_VFIO_GROUP_ADD, KVM_DEV_VFIO_GROUP_DEL, + kvm_device_attr, KVM_DEV_VFIO_FILE, KVM_DEV_VFIO_FILE_ADD, KVM_DEV_VFIO_FILE_DEL, }; #[cfg(all(feature = "kvm", not(test)))] use kvm_ioctls::DeviceFd as KvmDeviceFd; @@ -347,23 +347,23 @@ impl VfioContainer { } #[cfg(all(any(feature = "kvm", feature = "mshv"), not(test)))] - fn device_set_group(&self, group: &VfioGroup, add: bool) -> Result<()> { - let group_fd_ptr = &group.as_raw_fd() as *const i32; + fn device_set_fd(&self, dev_fd: RawFd, add: bool) -> Result<()> { + let dev_fd_ptr = &dev_fd as *const i32; if let Some(device_fd) = self.device_fd.as_ref() { match &device_fd.0 { #[cfg(feature = "kvm")] DeviceFdInner::Kvm(fd) => { let flag = if add { - KVM_DEV_VFIO_GROUP_ADD + KVM_DEV_VFIO_FILE_ADD } else { - KVM_DEV_VFIO_GROUP_DEL + KVM_DEV_VFIO_FILE_DEL }; let dev_attr = kvm_device_attr { flags: 0, - group: KVM_DEV_VFIO_GROUP, + group: KVM_DEV_VFIO_FILE, attr: u64::from(flag), - addr: group_fd_ptr as u64, + addr: dev_fd_ptr as u64, }; fd.set_device_attr(&dev_attr) .map_err(|e| VfioError::SetDeviceAttr(Error::new(e.errno()))) @@ -379,7 +379,7 @@ impl VfioContainer { flags: 0, group: MSHV_DEV_VFIO_FILE, attr: u64::from(flag), - addr: group_fd_ptr as u64, + addr: dev_fd_ptr as u64, }; fd.set_device_attr(&dev_attr) .map_err(|e| VfioError::SetDeviceAttr(Error::new(e.errno()))) @@ -398,7 +398,7 @@ impl VfioContainer { /// * group: target VFIO group #[cfg(all(any(feature = "kvm", feature = "mshv"), not(test)))] fn device_add_group(&self, group: &VfioGroup) -> Result<()> { - self.device_set_group(group, true) + self.device_set_fd(group.as_raw_fd(), true) } /// Delete a device from a VFIO group @@ -409,7 +409,7 @@ impl VfioContainer { /// * group: target VFIO group #[cfg(all(any(feature = "kvm", feature = "mshv"), not(test)))] fn device_del_group(&self, group: &VfioGroup) -> Result<()> { - self.device_set_group(group, false) + self.device_set_fd(group.as_raw_fd(), false) } #[cfg(test)] From 292ec4d8f969e3633997dc7c7bff9a1ed5231e9c Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 20 May 2025 22:16:01 +0000 Subject: [PATCH 2/5] vfio_ioctls: Introduce struct `VfioCommon` This is to hold common data structure used by both the legacy vfio mode (using vfio container and groups) and the new vfio cdev mode (using iommufd). As for now, the only shared data structure is the `device fd` from the hypervisor (e.g. kvm or mshv). Signed-off-by: Bo Chen --- vfio-ioctls/src/vfio_device.rs | 107 ++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 50 deletions(-) diff --git a/vfio-ioctls/src/vfio_device.rs b/vfio-ioctls/src/vfio_device.rs index 9bc6d58..c9413b2 100644 --- a/vfio-ioctls/src/vfio_device.rs +++ b/vfio-ioctls/src/vfio_device.rs @@ -149,6 +149,57 @@ impl vfio_region_info_with_cap { } } +struct VfioCommon { + #[allow(dead_code)] + device_fd: Option, +} + +impl VfioCommon { + #[cfg(all(any(feature = "kvm", feature = "mshv"), not(test)))] + fn device_set_fd(&self, dev_fd: RawFd, add: bool) -> Result<()> { + let dev_fd_ptr = &dev_fd as *const i32; + + if let Some(device_fd) = self.device_fd.as_ref() { + match &device_fd.0 { + #[cfg(feature = "kvm")] + DeviceFdInner::Kvm(fd) => { + let flag = if add { + KVM_DEV_VFIO_FILE_ADD + } else { + KVM_DEV_VFIO_FILE_DEL + }; + let dev_attr = kvm_device_attr { + flags: 0, + group: KVM_DEV_VFIO_FILE, + attr: u64::from(flag), + addr: dev_fd_ptr as u64, + }; + fd.set_device_attr(&dev_attr) + .map_err(|e| VfioError::SetDeviceAttr(Error::new(e.errno()))) + } + #[cfg(feature = "mshv")] + DeviceFdInner::Mshv(fd) => { + let flag = if add { + MSHV_DEV_VFIO_FILE_ADD + } else { + MSHV_DEV_VFIO_FILE_DEL + }; + let dev_attr = mshv_device_attr { + flags: 0, + group: MSHV_DEV_VFIO_FILE, + attr: u64::from(flag), + addr: dev_fd_ptr as u64, + }; + fd.set_device_attr(&dev_attr) + .map_err(|e| VfioError::SetDeviceAttr(Error::new(e.errno()))) + } + } + } else { + Ok(()) + } + } +} + /// A safe wrapper over a VFIO container object. /// /// A VFIO container represents an IOMMU domain, or a set of IO virtual address translation tables. @@ -161,9 +212,9 @@ impl vfio_region_info_with_cap { /// address translation mapping tables. pub struct VfioContainer { pub(crate) container: File, - #[allow(dead_code)] - pub(crate) device_fd: Option, pub(crate) groups: Mutex>>, + #[allow(dead_code)] + common: VfioCommon, } impl VfioContainer { @@ -180,7 +231,7 @@ impl VfioContainer { let container = VfioContainer { container, - device_fd, + common: VfioCommon { device_fd }, groups: Mutex::new(HashMap::new()), }; container.check_api_version()?; @@ -346,50 +397,6 @@ impl VfioContainer { }) } - #[cfg(all(any(feature = "kvm", feature = "mshv"), not(test)))] - fn device_set_fd(&self, dev_fd: RawFd, add: bool) -> Result<()> { - let dev_fd_ptr = &dev_fd as *const i32; - - if let Some(device_fd) = self.device_fd.as_ref() { - match &device_fd.0 { - #[cfg(feature = "kvm")] - DeviceFdInner::Kvm(fd) => { - let flag = if add { - KVM_DEV_VFIO_FILE_ADD - } else { - KVM_DEV_VFIO_FILE_DEL - }; - let dev_attr = kvm_device_attr { - flags: 0, - group: KVM_DEV_VFIO_FILE, - attr: u64::from(flag), - addr: dev_fd_ptr as u64, - }; - fd.set_device_attr(&dev_attr) - .map_err(|e| VfioError::SetDeviceAttr(Error::new(e.errno()))) - } - #[cfg(feature = "mshv")] - DeviceFdInner::Mshv(fd) => { - let flag = if add { - MSHV_DEV_VFIO_FILE_ADD - } else { - MSHV_DEV_VFIO_FILE_DEL - }; - let dev_attr = mshv_device_attr { - flags: 0, - group: MSHV_DEV_VFIO_FILE, - attr: u64::from(flag), - addr: dev_fd_ptr as u64, - }; - fd.set_device_attr(&dev_attr) - .map_err(|e| VfioError::SetDeviceAttr(Error::new(e.errno()))) - } - } - } else { - Ok(()) - } - } - /// Add a device to a VFIO group /// /// The VFIO device fd should have been set. @@ -398,7 +405,7 @@ impl VfioContainer { /// * group: target VFIO group #[cfg(all(any(feature = "kvm", feature = "mshv"), not(test)))] fn device_add_group(&self, group: &VfioGroup) -> Result<()> { - self.device_set_fd(group.as_raw_fd(), true) + self.common.device_set_fd(group.as_raw_fd(), true) } /// Delete a device from a VFIO group @@ -409,7 +416,7 @@ impl VfioContainer { /// * group: target VFIO group #[cfg(all(any(feature = "kvm", feature = "mshv"), not(test)))] fn device_del_group(&self, group: &VfioGroup) -> Result<()> { - self.device_set_fd(group.as_raw_fd(), false) + self.common.device_set_fd(group.as_raw_fd(), false) } #[cfg(test)] @@ -1328,7 +1335,7 @@ mod tests { VfioContainer { container, - device_fd: None, + common: VfioCommon { device_fd: None }, groups: Mutex::new(HashMap::new()), } } From ad0fcb7276c57a0d2b50ba97270d70b1cfb716f9 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 20 May 2025 23:00:26 +0000 Subject: [PATCH 3/5] vfio-ioctls: Introduce public trait `VfioOps` This trait defines common operations exposed to user-space drivers for vfio device wrappers that are either backed by a legacy VfioContainer or a vfio cdev device using iommufd . Signed-off-by: Bo Chen --- vfio-ioctls/src/lib.rs | 6 ++-- vfio-ioctls/src/vfio_device.rs | 58 ++++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/vfio-ioctls/src/lib.rs b/vfio-ioctls/src/lib.rs index 8098896..c642809 100644 --- a/vfio-ioctls/src/lib.rs +++ b/vfio-ioctls/src/lib.rs @@ -66,9 +66,9 @@ mod vfio_device; mod vfio_ioctls; pub use vfio_device::{ - VfioContainer, VfioDevice, VfioDeviceFd, VfioGroup, VfioIrq, VfioRegion, VfioRegionInfoCap, - VfioRegionInfoCapNvlink2Lnkspd, VfioRegionInfoCapNvlink2Ssatgt, VfioRegionInfoCapSparseMmap, - VfioRegionInfoCapType, VfioRegionSparseMmapArea, + VfioContainer, VfioDevice, VfioDeviceFd, VfioGroup, VfioIrq, VfioOps, VfioRegion, + VfioRegionInfoCap, VfioRegionInfoCapNvlink2Lnkspd, VfioRegionInfoCapNvlink2Ssatgt, + VfioRegionInfoCapSparseMmap, VfioRegionInfoCapType, VfioRegionSparseMmapArea, }; /// Error codes for VFIO operations. diff --git a/vfio-ioctls/src/vfio_device.rs b/vfio-ioctls/src/vfio_device.rs index c9413b2..4cf751a 100644 --- a/vfio-ioctls/src/vfio_device.rs +++ b/vfio-ioctls/src/vfio_device.rs @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause +use std::any::Any; use std::collections::HashMap; use std::ffi::CString; use std::fs::{File, OpenOptions}; @@ -148,6 +149,39 @@ impl vfio_region_info_with_cap { region_with_cap } } +/// Trait to define common operations exposed to user-space drivers for +/// VFIO device wrappers that are either backed by a legacy VfioContainer or +/// a VFIO cdev device using iommufd. +pub trait VfioOps: Any + Send + Sync { + /// Map a region of user space memory (e.g. guest memory) into an IO + /// address space managed by IOMMU hardware to enable DMA for + /// associated VFIO devices + /// + /// # Parameters + /// * iova: IO virtual address to map the memory. + /// * size: size of the memory region. + /// * user_addr: user space address (e.g. host virtual address) for + /// the guest memory region to map. + fn vfio_dma_map(&self, _iova: u64, _size: u64, _user_addr: u64) -> Result<()> { + unimplemented!() + } + + /// Unmap a region of user space memory (e.g. guest memory) from an IO + /// address space managed by IOMMU hardware to disable DMA for + /// associated VFIO devices + /// + /// # Parameters + /// * iova: IO virtual address to unmap the memory. + /// * size: size of the memory region. + fn vfio_dma_unmap(&self, _iova: u64, _size: u64) -> Result<()> { + unimplemented!() + } + + /// Downcast to the underlying vfio wrapper type + fn as_any(&self) -> &dyn Any { + unimplemented!() + } +} struct VfioCommon { #[allow(dead_code)] @@ -327,7 +361,9 @@ impl VfioContainer { } } - /// Map a region of guest memory regions into the vfio container's iommu table. + /// Map a region of user space memory (e.g. guest memory) into an IO + /// address space managed by IOMMU hardware to enable DMA for + /// associated VFIO devices /// /// # Parameters /// * iova: IO virtual address to mapping the memory. @@ -345,10 +381,12 @@ impl VfioContainer { vfio_syscall::map_dma(self, &dma_map) } - /// Unmap a region of guest memory regions into the vfio container's iommu table. + /// Unmap a region of user space memory (e.g. guest memory) from an IO + /// address space managed by IOMMU hardware to disable DMA for + /// associated VFIO devices /// /// # Parameters - /// * iova: IO virtual address to mapping the memory. + /// * iova: IO virtual address to unmap the memory. /// * size: size of the memory region. pub fn vfio_dma_unmap(&self, iova: u64, size: u64) -> Result<()> { let mut dma_unmap = vfio_iommu_type1_dma_unmap { @@ -436,6 +474,20 @@ impl AsRawFd for VfioContainer { } } +impl VfioOps for VfioContainer { + fn vfio_dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<()> { + self.vfio_dma_map(iova, size, user_addr) + } + + fn vfio_dma_unmap(&self, iova: u64, size: u64) -> Result<()> { + self.vfio_dma_unmap(iova, size) + } + + fn as_any(&self) -> &dyn Any { + self + } +} + /// A safe wrapper over a VFIO group object. /// /// The Linux VFIO frameworks supports multiple devices per group, and multiple groups per From 18b1476974c2c6687f7ca495a469f01220744e54 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Tue, 20 May 2025 22:21:10 +0000 Subject: [PATCH 4/5] vfio-ioctls: Decouple `get_device_info` from `VfioGroup` In this way, the refactored `get_device_info` can take a vfio device file that is opened via either vfio group or vfio cdev, making `struct VfioDeviceInfo` mode agnostic. Signed-off-by: Bo Chen --- vfio-ioctls/src/vfio_device.rs | 93 ++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/vfio-ioctls/src/vfio_device.rs b/vfio-ioctls/src/vfio_device.rs index 4cf751a..5b74472 100644 --- a/vfio-ioctls/src/vfio_device.rs +++ b/vfio-ioctls/src/vfio_device.rs @@ -532,50 +532,12 @@ impl VfioGroup { self.id } - #[inline] - /// Get device type from device_info flags. - /// - /// # Parameters - /// * `flags`: flags field in device_info structure. - fn get_device_type(flags: &u32) -> u32 { - // There may be more types of device here later according to vfio_bindings. - let device_type: u32 = VFIO_DEVICE_FLAGS_PCI - | VFIO_DEVICE_FLAGS_PLATFORM - | VFIO_DEVICE_FLAGS_AMBA - | VFIO_DEVICE_FLAGS_CCW - | VFIO_DEVICE_FLAGS_AP; - - flags & device_type - } - fn get_device(&self, name: &Path) -> Result { let uuid_osstr = name.file_name().ok_or(VfioError::InvalidPath)?; let uuid_str = uuid_osstr.to_str().ok_or(VfioError::InvalidPath)?; let path: CString = CString::new(uuid_str.as_bytes()).expect("CString::new() failed"); let device = vfio_syscall::get_group_device_fd(self, &path)?; - - let mut dev_info = vfio_device_info { - argsz: mem::size_of::() as u32, - flags: 0, - num_regions: 0, - num_irqs: 0, - cap_offset: 0, - pad: 0, - }; - vfio_syscall::get_device_info(&device, &mut dev_info)?; - match VfioGroup::get_device_type(&dev_info.flags) { - VFIO_DEVICE_FLAGS_PLATFORM => {} - VFIO_DEVICE_FLAGS_PCI => { - if dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1 - || dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1 - { - return Err(VfioError::VfioDeviceGetInfoPCI); - } - } - _ => { - return Err(VfioError::VfioDeviceGetInfoOther); - } - } + let dev_info = VfioDeviceInfo::get_device_info(&device)?; Ok(VfioDeviceInfo::new(device, &dev_info)) } @@ -669,6 +631,49 @@ pub(crate) struct VfioDeviceInfo { } impl VfioDeviceInfo { + #[inline] + /// Get device type from device_info flags. + /// + /// # Parameters + /// * `flags`: flags field in device_info structure. + fn get_device_type(flags: &u32) -> u32 { + // There may be more types of device here later according to vfio_bindings. + let device_type: u32 = VFIO_DEVICE_FLAGS_PCI + | VFIO_DEVICE_FLAGS_PLATFORM + | VFIO_DEVICE_FLAGS_AMBA + | VFIO_DEVICE_FLAGS_CCW + | VFIO_DEVICE_FLAGS_AP; + + flags & device_type + } + + fn get_device_info(device: &File) -> Result { + let mut dev_info = vfio_device_info { + argsz: mem::size_of::() as u32, + flags: 0, + num_regions: 0, + num_irqs: 0, + cap_offset: 0, + pad: 0, + }; + vfio_syscall::get_device_info(device, &mut dev_info)?; + match VfioDeviceInfo::get_device_type(&dev_info.flags) { + VFIO_DEVICE_FLAGS_PLATFORM => {} + VFIO_DEVICE_FLAGS_PCI => { + if dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1 + || dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1 + { + return Err(VfioError::VfioDeviceGetInfoPCI); + } + } + _ => { + return Err(VfioError::VfioDeviceGetInfoOther); + } + } + + Ok(dev_info) + } + fn new(device: File, dev_info: &vfio_device_info) -> Self { VfioDeviceInfo { device, @@ -1586,18 +1591,18 @@ mod tests { #[test] fn test_get_device_type() { let flags: u32 = VFIO_DEVICE_FLAGS_PCI; - assert_eq!(flags, VfioGroup::get_device_type(&flags)); + assert_eq!(flags, VfioDeviceInfo::get_device_type(&flags)); let flags: u32 = VFIO_DEVICE_FLAGS_PLATFORM; - assert_eq!(flags, VfioGroup::get_device_type(&flags)); + assert_eq!(flags, VfioDeviceInfo::get_device_type(&flags)); let flags: u32 = VFIO_DEVICE_FLAGS_AMBA; - assert_eq!(flags, VfioGroup::get_device_type(&flags)); + assert_eq!(flags, VfioDeviceInfo::get_device_type(&flags)); let flags: u32 = VFIO_DEVICE_FLAGS_CCW; - assert_eq!(flags, VfioGroup::get_device_type(&flags)); + assert_eq!(flags, VfioDeviceInfo::get_device_type(&flags)); let flags: u32 = VFIO_DEVICE_FLAGS_AP; - assert_eq!(flags, VfioGroup::get_device_type(&flags)); + assert_eq!(flags, VfioDeviceInfo::get_device_type(&flags)); } } From 2d9d44a5e355a4a1dee884df823b1f58d80c5159 Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Wed, 21 May 2025 20:40:25 +0000 Subject: [PATCH 5/5] vfio-ioctls: Decouple VfioDevice and VfioContainer/VfioGroup The refactored `struct VfioDevice` now is mode agnostic. It can support both legacy vfio mode (using vfio container and group) and the vfio cdev mode (using iommufd). Signed-off-by: Bo Chen --- vfio-ioctls/src/lib.rs | 2 ++ vfio-ioctls/src/vfio_device.rs | 46 ++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/vfio-ioctls/src/lib.rs b/vfio-ioctls/src/lib.rs index c642809..8b06705 100644 --- a/vfio-ioctls/src/lib.rs +++ b/vfio-ioctls/src/lib.rs @@ -137,6 +137,8 @@ pub enum VfioError { GetHostAddress, #[error("invalid dma unmap size")] InvalidDmaUnmapSize, + #[error("failed to downcast VfioOps")] + DowncastVfioOps, } /// Specialized version of `Result` for VFIO subsystem. diff --git a/vfio-ioctls/src/vfio_device.rs b/vfio-ioctls/src/vfio_device.rs index 5b74472..298065c 100644 --- a/vfio-ioctls/src/vfio_device.rs +++ b/vfio-ioctls/src/vfio_device.rs @@ -10,9 +10,7 @@ use std::fs::{File, OpenOptions}; use std::mem::{self, ManuallyDrop}; use std::os::unix::io::{AsRawFd, RawFd}; use std::os::unix::prelude::FileExt; -use std::path::Path; -#[cfg(not(test))] -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use byteorder::{ByteOrder, NativeEndian}; @@ -341,10 +339,9 @@ impl VfioContainer { // Clean up the group when the last user releases reference to the group, three reference // count for: - // - one reference held by the last device object // - one reference cloned in VfioDevice.drop() and passed into here // - one reference held by the groups hashmap - if Arc::strong_count(&group) == 3 { + if Arc::strong_count(&group) == 2 { #[cfg(any(feature = "kvm", feature = "mshv"))] match self.device_del_group(&group) { Ok(_) => {} @@ -891,8 +888,8 @@ pub struct VfioDevice { pub(crate) flags: u32, pub(crate) regions: Vec, pub(crate) irqs: HashMap, - pub(crate) group: Arc, - pub(crate) container: Arc, + pub(crate) sysfspath: PathBuf, + pub(crate) vfio_ops: Arc, } impl VfioDevice { @@ -910,11 +907,17 @@ impl VfioDevice { /// /// # Parameters /// * `sysfspath`: specify the vfio device path in sys file system. - /// * `container`: the new VFIO device object will bind to this container object. - pub fn new(sysfspath: &Path, container: Arc) -> Result { - let group_id = Self::get_group_id_from_path(sysfspath)?; - let group = container.get_group(group_id)?; - let device_info = group.get_device(sysfspath)?; + /// * `vfio_ops`: the vfio device wrapper object that the new VFIO device object will bind to. + pub fn new(sysfspath: &Path, vfio_ops: Arc) -> Result { + let device_info = + if let Some(vfio_container) = vfio_ops.as_any().downcast_ref::() { + let group_id = Self::get_group_id_from_path(sysfspath)?; + let group = vfio_container.get_group(group_id)?; + group.get_device(sysfspath)? + } else { + return Err(VfioError::DowncastVfioOps); + }; + let regions = device_info.get_regions()?; let irqs = device_info.get_irqs()?; @@ -923,8 +926,8 @@ impl VfioDevice { flags: device_info.flags, regions, irqs, - group, - container, + sysfspath: sysfspath.to_path_buf(), + vfio_ops, }) } @@ -1278,12 +1281,16 @@ impl Drop for VfioDevice { // ManuallyDrop is needed here because we need to ensure that VfioDevice::device is closed // before dropping VfioDevice::group, otherwise it will cause EBUSY when putting the // group object. + if let Some(container) = self.vfio_ops.as_any().downcast_ref::() { + // SAFETY: we own the File object. + unsafe { + ManuallyDrop::drop(&mut self.device); + } - // SAFETY: we own the File object. - unsafe { - ManuallyDrop::drop(&mut self.device); + let group_id = Self::get_group_id_from_path(&self.sysfspath).unwrap(); + let group = container.get_group(group_id).unwrap(); + container.put_group(group); } - self.container.put_group(self.group.clone()); } } @@ -1423,7 +1430,8 @@ mod tests { container.put_group(group4); assert_eq!(Arc::strong_count(&group), 3); container.put_group(group3); - assert_eq!(Arc::strong_count(&group), 1); + assert_eq!(Arc::strong_count(&group), 2); + container.put_group(group); container.vfio_dma_map(0x1000, 0x1000, 0x8000).unwrap(); container.vfio_dma_map(0x2000, 0x2000, 0x8000).unwrap_err();