Import textures through metal on macos

Author: TrueDoctor

desktop/Cargo.toml

@@ -13,9 +13,9 @@ default = ["gpu", "accelerated_paint"]
 gpu = ["graphite-desktop-wrapper/gpu"]
 
 # Hardware acceleration features
-accelerated_paint = ["ash", "accelerated_paint_dmabuf", "accelerated_paint_d3d11", "accelerated_paint_iosurface"]
-accelerated_paint_dmabuf = ["libc"]
-accelerated_paint_d3d11 = ["windows"]
+accelerated_paint = ["accelerated_paint_dmabuf", "accelerated_paint_d3d11", "accelerated_paint_iosurface"]
+accelerated_paint_dmabuf = ["libc", "ash"]
+accelerated_paint_d3d11 = ["windows", "ash"]
 accelerated_paint_iosurface = ["objc2-io-surface", "objc2-metal", "core-foundation"]
 
 [dependencies]

desktop/src/app.rs

@@ -1,9 +1,10 @@
 use crate::CustomEvent;
 use crate::cef::WindowSize;
-use crate::consts::APP_NAME;
+use crate::consts::{APP_NAME, CEF_MESSAGE_LOOP_MAX_ITERATIONS};
 use crate::render::GraphicsState;
 use graphite_desktop_wrapper::messages::{DesktopFrontendMessage, DesktopWrapperMessage};
 use graphite_desktop_wrapper::{DesktopWrapper, NodeGraphExecutionResult, WgpuContext, serialize_frontend_messages};
+
 use rfd::AsyncFileDialog;
 use std::sync::Arc;
 use std::sync::mpsc::Sender;
@@ -174,7 +175,7 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 		{
 			self.cef_schedule = None;
 			// Poll cef message loop multiple times to avoid message loop starvation
-			for _ in 0..10 {
+			for _ in 0..CEF_MESSAGE_LOOP_MAX_ITERATIONS {
 				self.cef_context.work();
 			}
 		}

desktop/src/cef/context.rs

@@ -86,7 +86,6 @@ impl Context<Setup> {
 		let mut client = Client::new(BrowserProcessClientImpl::new(render_handler, event_handler.clone()));
 
 		let url = CefString::from(format!("{GRAPHITE_SCHEME}://{FRONTEND_DOMAIN}/").as_str());
-		// let url = CefString::from(format!("chrome://gpu").as_str());
 
 		let window_info = WindowInfo {
 			windowless_rendering_enabled: 1,
@@ -96,7 +95,7 @@ impl Context<Setup> {
 		};
 
 		let settings = BrowserSettings {
-			windowless_frame_rate: 120,
+			windowless_frame_rate: crate::consts::CEF_WINDOWLESS_FRAME_RATE,
 			background_color: 0x0,
 			..Default::default()
 		};

desktop/src/cef/texture_import/common.rs

@@ -50,7 +50,7 @@ pub mod format {
 		}
 	}
 
-	#[cfg(feature = "accelerated_paint")]
+	#[cfg(all(feature = "accelerated_paint", not(target_os = "macos")))]
 	/// Convert CEF color type to Vulkan format
 	pub fn cef_to_vulkan(format: cef_color_type_t) -> Result<vk::Format, TextureImportError> {
 		match format {
@@ -84,7 +84,12 @@ pub mod texture {
 			view_formats: &[],
 		});
 
-		tracing::warn!("Using fallback CPU texture - hardware acceleration not available");
+		tracing::warn!(
+			"Using fallback CPU texture for CEF rendering ({}x{}, {:?}) - hardware acceleration failed or unavailable. Consider checking GPU driver support.",
+			width,
+			height,
+			format
+		);
 		Ok(texture)
 	}
 }
@@ -105,6 +110,7 @@ pub mod vulkan {
 	}
 
 	/// Check if the wgpu device is using Vulkan backend
+	#[cfg(not(target_os = "macos"))]
 	pub fn is_vulkan_backend(device: &Device) -> bool {
 		use wgpu::hal::api;
 		let mut is_vulkan = false;

desktop/src/cef/texture_import/d3d11.rs

@@ -38,7 +38,7 @@ impl TextureImporter for D3D11Importer {
 						}
 					}
 				}
-				
+
 				// Try Vulkan as fallback
 				if vulkan::is_vulkan_backend(device) {
 					match self.import_via_vulkan(device) {
@@ -298,13 +298,11 @@ impl D3D11Importer {
 		// Open D3D11 shared handle on D3D12 device
 		unsafe {
 			let mut shared_resource: Option<ID3D12Resource> = None;
-			d3d12_device.OpenSharedHandle(
-				windows::Win32::Foundation::HANDLE(self.handle as isize),
-				&ID3D12Resource::IID,
-				&mut shared_resource as *mut _ as *mut _,
-			).map_err(|e| TextureImportError::PlatformError {
-				message: format!("Failed to open D3D11 shared handle on D3D12: {:?}", e),
-			})?;
+			d3d12_device
+				.OpenSharedHandle(windows::Win32::Foundation::HANDLE(self.handle as isize), &ID3D12Resource::IID, &mut shared_resource as *mut _ as *mut _)
+				.map_err(|e| TextureImportError::PlatformError {
+					message: format!("Failed to open D3D11 shared handle on D3D12: {:?}", e),
+				})?;
 
 			shared_resource.ok_or_else(|| TextureImportError::InvalidHandle("Failed to get D3D12 resource from shared handle".to_string()))
 		}

desktop/src/cef/texture_import/iosurface.rs

@@ -3,16 +3,16 @@
 #[cfg(target_os = "macos")]
 use std::os::raw::c_void;
 
-#[cfg(all(feature = "accelerated_paint", target_os = "macos"))]
-use ash::vk;
 #[cfg(all(feature = "accelerated_paint", target_os = "macos"))]
 use core_foundation::base::{CFType, TCFType};
 #[cfg(all(feature = "accelerated_paint", target_os = "macos"))]
 use objc2_io_surface::{IOSurface, IOSurfaceRef};
 #[cfg(all(feature = "accelerated_paint", target_os = "macos"))]
+use objc2_metal::{MTLDevice, MTLPixelFormat, MTLTexture, MTLTextureDescriptor, MTLTextureType, MTLTextureUsage};
+#[cfg(all(feature = "accelerated_paint", target_os = "macos"))]
 use wgpu::hal::api;
 
-use super::common::{TextureImportError, TextureImportResult, TextureImporter, format, texture, vulkan};
+use super::common::{TextureImportError, TextureImportResult, TextureImporter, format, texture};
 use cef::sys::cef_color_type_t;
 
 #[cfg(target_os = "macos")]
@@ -30,13 +30,13 @@ impl TextureImporter for IOSurfaceImporter {
 		#[cfg(feature = "accelerated_paint")]
 		{
 			if self.supports_hardware_acceleration(device) {
-				match self.import_via_vulkan(device) {
+				match self.import_via_metal(device) {
 					Ok(texture) => {
-						tracing::trace!("Successfully imported IOSurface texture via Vulkan");
+						tracing::trace!("Successfully imported IOSurface texture via Metal");
 						return Ok(texture);
 					}
 					Err(e) => {
-						tracing::warn!("Failed to import IOSurface via Vulkan: {}, falling back to CPU texture", e);
+						tracing::warn!("Failed to import IOSurface via Metal: {}, falling back to CPU texture", e);
 					}
 				}
 			}
@@ -54,8 +54,8 @@ impl TextureImporter for IOSurfaceImporter {
 				return false;
 			}
 
-			// Check if wgpu is using Vulkan backend
-			vulkan::is_vulkan_backend(device)
+			// Check if wgpu is using Metal backend
+			self.is_metal_backend(device)
 		}
 		#[cfg(not(feature = "accelerated_paint"))]
 		{
@@ -68,23 +68,23 @@ impl TextureImporter for IOSurfaceImporter {
 #[cfg(target_os = "macos")]
 impl IOSurfaceImporter {
 	#[cfg(feature = "accelerated_paint")]
-	fn import_via_vulkan(&self, device: &wgpu::Device) -> TextureImportResult {
-		// Get wgpu's Vulkan instance and device
-		use wgpu::{TextureUses, wgc::api::Vulkan};
+	fn import_via_metal(&self, device: &wgpu::Device) -> TextureImportResult {
+		// Get wgpu's Metal device
+		use wgpu::{hal::Api, wgc::api::Metal};
 		let hal_texture = unsafe {
-			device.as_hal::<api::Vulkan, _, _>(|device| {
+			device.as_hal::<api::Metal, _, _>(|device| {
 				let Some(device) = device else {
 					return Err(TextureImportError::HardwareUnavailable {
-						reason: "Device is not using Vulkan backend".to_string(),
+						reason: "Device is not using Metal backend".to_string(),
 					});
 				};
 
-				// Import IOSurface handle into Vulkan via Metal
-				let vk_image = self.import_iosurface_to_vulkan(device)?;
+				// Import IOSurface handle into Metal texture
+				let metal_texture = self.import_iosurface_to_metal(device)?;
 
-				// Wrap VkImage in wgpu-hal texture
-				let hal_texture = <api::Vulkan as wgpu::hal::Api>::Device::texture_from_raw(
-					vk_image,
+				// Wrap Metal texture in wgpu-hal texture
+				let hal_texture = <api::Metal as wgpu::hal::Api>::Device::texture_from_raw(
+					metal_texture,
 					&wgpu::hal::TextureDescriptor {
 						label: Some("CEF IOSurface Texture"),
 						size: wgpu::Extent3d {
@@ -96,7 +96,7 @@ impl IOSurfaceImporter {
 						sample_count: 1,
 						dimension: wgpu::TextureDimension::D2,
 						format: format::cef_to_wgpu(self.format)?,
-						usage: TextureUses::COPY_DST | TextureUses::RESOURCE,
+						usage: wgpu::hal::TextureUses::RESOURCE,
 						memory_flags: wgpu::hal::MemoryFlags::empty(),
 						view_formats: vec![],
 					},
@@ -109,7 +109,7 @@ impl IOSurfaceImporter {
 
 		// Import hal texture into wgpu
 		let texture = unsafe {
-			device.create_texture_from_hal::<Vulkan>(
+			device.create_texture_from_hal::<Metal>(
 				hal_texture,
 				&wgpu::TextureDescriptor {
 					label: Some("CEF IOSurface Texture"),
@@ -132,64 +132,66 @@ impl IOSurfaceImporter {
 	}
 
 	#[cfg(feature = "accelerated_paint")]
-	fn import_iosurface_to_vulkan(&self, hal_device: &<api::Vulkan as wgpu::hal::Api>::Device) -> Result<vk::Image, TextureImportError> {
-		// Get raw Vulkan handles
-		let device = hal_device.raw_device();
-		let _instance = hal_device.shared_instance().raw_instance();
-
+	fn import_iosurface_to_metal(&self, hal_device: &<api::Metal as wgpu::hal::Api>::Device) -> Result<<api::Metal as wgpu::hal::Api>::Texture, TextureImportError> {
 		// Validate dimensions
 		if self.width == 0 || self.height == 0 {
 			return Err(TextureImportError::InvalidHandle("Invalid IOSurface texture dimensions".to_string()));
 		}
 
 		// Convert handle to IOSurface
-		let _iosurface = unsafe {
+		let iosurface = unsafe {
 			let cf_type = CFType::wrap_under_get_rule(self.handle as IOSurfaceRef);
 			IOSurface::from(cf_type)
 		};
 
-		// Note: Full Metal-to-Vulkan import would require:
-		// 1. Creating Metal texture from IOSurface
-		// 2. Using VK_EXT_metal_objects to import Metal texture into Vulkan
-		// 3. Proper synchronization between Metal and Vulkan
-		//
-		// This is complex and not fully supported by current objc2 bindings.
-		// For now, we create a minimal Vulkan image and rely on fallback.
-
-		// Create external memory image info for Metal objects
-		let mut external_memory_info = vk::ExternalMemoryImageCreateInfo::default().handle_types(vk::ExternalMemoryHandleTypeFlags::MTLTEXTURE_EXT);
-
-		// Create image create info
-		let image_create_info = vk::ImageCreateInfo::default()
-			.image_type(vk::ImageType::TYPE_2D)
-			.format(format::cef_to_vulkan(self.format)?)
-			.extent(vk::Extent3D {
-				width: self.width,
-				height: self.height,
-				depth: 1,
-			})
-			.mip_levels(1)
-			.array_layers(1)
-			.samples(vk::SampleCountFlags::TYPE_1)
-			.tiling(vk::ImageTiling::OPTIMAL)
-			.usage(vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::COLOR_ATTACHMENT)
-			.sharing_mode(vk::SharingMode::EXCLUSIVE)
-			.push_next(&mut external_memory_info);
-
-		// Create the image
-		let image = unsafe {
-			device.create_image(&image_create_info, None).map_err(|e| TextureImportError::VulkanError {
-				operation: format!("Failed to create Vulkan image: {:?}", e),
-			})?
+		// Get the Metal device from wgpu-hal
+		let metal_device = hal_device.raw_device();
+
+		// Convert CEF format to Metal pixel format
+		let metal_format = self.cef_to_metal_format(self.format)?;
+
+		// Create Metal texture descriptor
+		let texture_descriptor = MTLTextureDescriptor::new();
+		texture_descriptor.setTextureType(MTLTextureType::Type2D);
+		texture_descriptor.setPixelFormat(metal_format);
+		texture_descriptor.setWidth(self.width as usize);
+		texture_descriptor.setHeight(self.height as usize);
+		texture_descriptor.setDepth(1);
+		texture_descriptor.setMipmapLevelCount(1);
+		texture_descriptor.setSampleCount(1);
+		texture_descriptor.setUsage(MTLTextureUsage::ShaderRead);
+
+		// Create Metal texture from IOSurface
+		let metal_texture = unsafe { metal_device.newTextureWithDescriptor_iosurface_plane(&texture_descriptor, &iosurface, 0) };
+
+		let Some(metal_texture) = metal_texture else {
+			return Err(TextureImportError::PlatformError {
+				message: "Failed to create Metal texture from IOSurface".to_string(),
+			});
 		};
 
-		// Note: The actual Metal-to-Vulkan import would require VK_EXT_metal_objects
-		// and proper Metal texture handle extraction, which is complex and not
-		// fully supported in the current objc2 bindings. For now, we return the
-		// image and rely on fallback behavior.
+		tracing::trace!("Successfully created Metal texture from IOSurface");
+		Ok(metal_texture)
+	}
 
-		tracing::warn!("Metal-to-Vulkan texture import not fully implemented");
+	#[cfg(feature = "accelerated_paint")]
+	fn cef_to_metal_format(&self, format: cef_color_type_t) -> Result<MTLPixelFormat, TextureImportError> {
+		match format {
+			cef_color_type_t::CEF_COLOR_TYPE_BGRA_8888 => Ok(MTLPixelFormat::BGRA8Unorm_sRGB),
+			cef_color_type_t::CEF_COLOR_TYPE_RGBA_8888 => Ok(MTLPixelFormat::RGBA8Unorm_sRGB),
+			_ => Err(TextureImportError::UnsupportedFormat { format }),
+		}
+	}
 
-		Ok(image)
+	#[cfg(feature = "accelerated_paint")]
+	fn is_metal_backend(&self, device: &wgpu::Device) -> bool {
+		use wgpu::hal::api;
+		let mut is_metal = false;
+		unsafe {
+			device.as_hal::<api::Metal, _, _>(|device| {
+				is_metal = device.is_some();
+			});
+		}
+		is_metal
 	}
 }

desktop/src/cef/texture_import/mod.rs

@@ -8,7 +8,7 @@
 //!
 //! - **Linux**: DMA-BUF via Vulkan external memory
 //! - **Windows**: D3D11 shared textures via Vulkan interop
-//! - **macOS**: IOSurface via Metal/Vulkan interop
+//! - **macOS**: IOSurface via Metal native API
 //!
 //! # Usage
 //!

desktop/src/consts.rs

@@ -1,3 +1,7 @@
 pub(crate) static APP_NAME: &str = "Graphite";
 pub(crate) static APP_ID: &str = "rs.graphite.GraphiteEditor";
 pub(crate) static APP_DIRECTORY_NAME: &str = "graphite-editor";
+
+// CEF configuration constants
+pub(crate) const CEF_WINDOWLESS_FRAME_RATE: i32 = 120;
+pub(crate) const CEF_MESSAGE_LOOP_MAX_ITERATIONS: usize = 10;

desktop/src/main.rs

@@ -17,7 +17,7 @@ use app::WinitApp;
 mod dirs;
 
 use graphite_desktop_wrapper::messages::DesktopWrapperMessage;
-use graphite_desktop_wrapper::{DesktopWrapper, NodeGraphExecutionResult, WgpuContext};
+use graphite_desktop_wrapper::{NodeGraphExecutionResult, WgpuContext};
 
 pub(crate) enum CustomEvent {
 	UiUpdate(wgpu::Texture),