From ca9a70e10a21775c889a5772c08721f8ffcfe082 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 05:18:23 -0400 Subject: [PATCH] export: run the zero-copy H.264 encoder on the shared device (Stage 3c-export pt 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the common Linux H.264 export fully GPU-resident: decode (HW NV12) → composite → VAAPI encode all on one device, no CPU round-trips. - gpu-video-encoder: ZeroCopyEncoder now holds wgpu (device,queue,adapter) handles instead of owning a DrmDevice. New `new_on_device(device,queue,adapter,...)` runs the RGBA→NV12 render + DMA-BUF import on a passed device; `new` keeps building its own. The encoder only *imports* VAAPI surfaces (not export), so the shared import-capable device works directly. - editor: stash the shared device handles on EditorApp (set in the creation closure from the eframe render_state when the shared device is active) and thread them through start_video_export / start_video_with_audio_export → try_build_zero_copy, which uses new_on_device when available. ZeroCopyVideo carries on_shared_device → the export composite uses hardware_ok=true (consuming HW-decoded GPU frames from pt 1) only then. Falls back to the own-device encoder (decode downloads to CPU) when no shared device. Tradeoff: the export now shares the GPU with the UI thread (was a separate device); acceptable for the GPU-resident-decode win. Compiles; crate tests build. Co-Authored-By: Claude Opus 4.8 --- .../gpu-video-encoder/src/encoder.rs | 50 +++++++++++++++---- .../lightningbeam-editor/src/export/mod.rs | 43 +++++++++++----- .../lightningbeam-editor/src/main.rs | 19 +++++-- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs index 4347f0e..7ec9c36 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs @@ -6,7 +6,7 @@ use crate::dmabuf::{self, ImportedNv12, Nv12DmaBuf}; use crate::render_nv12::Rgba2Nv12; -use crate::vk_device::{self, DrmDevice}; +use crate::vk_device; use ffmpeg_sys_next as ff; use std::collections::HashMap; use std::ffi::CString; @@ -19,7 +19,11 @@ fn averror(e: i32) -> i32 { } pub struct ZeroCopyEncoder { - drm: DrmDevice, + /// wgpu handles the NV12 render runs on — either an own `DrmDevice`'s (via `new`) or the + /// editor's shared device (via `new_on_device`). Cloned (Arc-backed) so the source can drop. + device: wgpu::Device, + queue: wgpu::Queue, + adapter: wgpu::Adapter, renderer: Rgba2Nv12, hw_device: *mut ff::AVBufferRef, frames_ref: *mut ff::AVBufferRef, @@ -51,8 +55,30 @@ impl ZeroCopyEncoder { output_path: &Path, full_range: bool, ) -> Result { + // Build a dedicated DMA-BUF-import device and run the encoder on it. let drm = vk_device::create()?; - let renderer = Rgba2Nv12::new(&drm.device, full_range); + Self::new_on_device( + drm.device, drm.queue, drm.adapter, + width, height, framerate, bitrate_kbps, output_path, full_range, + ) + } + + /// Build the encoder running its NV12 render + DMA-BUF import on an existing wgpu device (the + /// editor's shared device), so decode→composite→encode stay GPU-resident on one device. The + /// device must have the DMA-BUF import extensions (a `DrmDevice` / `vk_device::create_windowed`). + #[allow(clippy::too_many_arguments)] + pub fn new_on_device( + device: wgpu::Device, + queue: wgpu::Queue, + adapter: wgpu::Adapter, + width: u32, + height: u32, + framerate: i32, + bitrate_kbps: u32, + output_path: &Path, + full_range: bool, + ) -> Result { + let renderer = Rgba2Nv12::new(&device, full_range); unsafe { let mut hw_device = crate::vaapi::create_device()?; let name = CString::new("h264_vaapi").unwrap(); @@ -153,7 +179,9 @@ impl ZeroCopyEncoder { let stream_tb = (*stream).time_base; Ok(Self { - drm, + device, + queue, + adapter, renderer, hw_device, frames_ref, @@ -172,10 +200,10 @@ impl ZeroCopyEncoder { /// The wgpu device frames must be rendered on (so the RGBA texture is importable). pub fn device(&self) -> &wgpu::Device { - &self.drm.device + &self.device } pub fn queue(&self) -> &wgpu::Queue { - &self.drm.queue + &self.queue } /// Render `rgba` (an `Rgba8Unorm` texture on [`Self::device`], `TEXTURE_BINDING`) @@ -216,7 +244,7 @@ impl ZeroCopyEncoder { uv_pitch: uv.pitch as u64, ten_bit: false, }; - let imported = match dmabuf::import_raw(&self.drm.device, &self.drm.adapter, &buf) { + let imported = match dmabuf::import_raw(&self.device, &self.adapter, &buf) { Ok(i) => i, Err(e) => { ff::av_frame_free(&mut (drm_f as *mut _)); @@ -233,10 +261,10 @@ impl ZeroCopyEncoder { let rgba_view = rgba.create_view(&Default::default()); let y_view = imp.y().create_view(&Default::default()); let uv_view = imp.uv().create_view(&Default::default()); - let mut cmd = self.drm.device.create_command_encoder(&Default::default()); - self.renderer.convert(&self.drm.device, &mut cmd, &rgba_view, &y_view, &uv_view); - self.drm.queue.submit(Some(cmd.finish())); - let _ = self.drm.device.poll(wgpu::PollType::wait_indefinitely()); + let mut cmd = self.device.create_command_encoder(&Default::default()); + self.renderer.convert(&self.device, &mut cmd, &rgba_view, &y_view, &uv_view); + self.queue.submit(Some(cmd.finish())); + let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); // Encode the surface. (*surf).pts = self.pts; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index d435749..5673da1 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -82,6 +82,8 @@ struct ZeroCopyVideo { gpu_resources: video_exporter::ExportGpuResources, /// Reused RGBA target (RENDER_ATTACHMENT | TEXTURE_BINDING) on the encoder's device. rgba: wgpu::Texture, + /// True when running on the shared device → compositing can consume hardware-decoded GPU frames. + on_shared_device: bool, } /// State for a single-frame image export (runs on the GPU render thread, one frame per update). @@ -895,6 +897,7 @@ impl ExportOrchestrator { height: u32, framerate: f64, output_path: &std::path::Path, + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Option { // Zero-copy is 8-bit H.264 only; HDR needs the 10-bit HEVC software path. if settings.hdr.is_hdr() @@ -902,14 +905,25 @@ impl ExportOrchestrator { { return None; } - let encoder = match gpu_video_encoder::encoder::ZeroCopyEncoder::new( - width, - height, - framerate.round() as i32, - settings.quality.bitrate_kbps(), - output_path, - settings.color_range.is_full(), - ) { + let bitrate = settings.quality.bitrate_kbps(); + let fr = framerate.round() as i32; + let full = settings.color_range.is_full(); + let on_shared_device = shared_device.is_some(); + // Prefer the shared device → decode→composite→encode stay GPU-resident on one device. + // Without it, the encoder builds its own device (decode still downloads to CPU per Step 1's + // hardware_ok=false on this path). + let encoder_result = match shared_device { + Some((device, queue, adapter)) => { + println!("🎬 [EXPORT] zero-copy on shared device (GPU-resident decode)"); + gpu_video_encoder::encoder::ZeroCopyEncoder::new_on_device( + device, queue, adapter, width, height, fr, bitrate, output_path, full, + ) + } + None => gpu_video_encoder::encoder::ZeroCopyEncoder::new( + width, height, fr, bitrate, output_path, full, + ), + }; + let encoder = match encoder_result { Ok(e) => e, Err(e) => { println!("🎬 [EXPORT] zero-copy unavailable ({e}); software path"); @@ -945,7 +959,7 @@ impl ExportOrchestrator { view_formats: &[], }); println!("🎬 [EXPORT] zero-copy VAAPI H.264 enabled"); - Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba }) + Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba, on_shared_device }) } /// Start a video export in the background. @@ -963,6 +977,7 @@ impl ExportOrchestrator { /// # Returns /// Ok(()) on success, Err on failure #[allow(clippy::too_many_arguments)] + #[allow(clippy::too_many_arguments)] pub fn start_video_export( &mut self, settings: VideoExportSettings, @@ -971,6 +986,8 @@ impl ExportOrchestrator { video_manager: Arc>, raster_store: lightningbeam_core::raster_store::RasterStore, container_path: Option, + // The shared VAAPI device, `Some` only when active → zero-copy encode runs on it. + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Result<(), String> { println!("🎬 [VIDEO EXPORT] Starting video export"); @@ -998,7 +1015,7 @@ impl ExportOrchestrator { // success it returns here; otherwise we fall through to the software encoder thread. #[cfg(target_os = "linux")] { - if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path) { + if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path, shared_device) { drop(frame_rx); let document_snapshot = document.clone(); let mut image_cache = ImageCache::new(); @@ -1062,6 +1079,8 @@ impl ExportOrchestrator { video_manager: Arc>, raster_store: lightningbeam_core::raster_store::RasterStore, container_path: Option, + // The shared VAAPI device, `Some` only when active → zero-copy encode runs on it. + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Result<(), String> { println!("🎬🎵 [PARALLEL EXPORT] Starting parallel video+audio export"); @@ -1128,7 +1147,7 @@ impl ExportOrchestrator { // by `render_next_video_frame` on the UI thread. #[cfg(target_os = "linux")] let (video_thread, video_state) = match Self::try_build_zero_copy( - &video_settings, video_width, video_height, video_framerate, &temp_video_path, + &video_settings, video_width, video_height, video_framerate, &temp_video_path, shared_device, ) { Some(zc) => { drop(frame_rx); // the zero-copy path renders internally, no frame channel @@ -1500,7 +1519,7 @@ impl ExportOrchestrator { None, false, Some(&raster_store), - false, // zero-copy runs on its own device → download HW frames to CPU + zc.on_shared_device, // GPU-resident decode only when on the shared device ) { Ok(cmd) => cmd, Err(e) => { diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index d9152ad..8c77c48 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -299,14 +299,18 @@ fn main() -> eframe::Result { options, Box::new(move |cc| { #[cfg(debug_assertions)] - let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); + #[allow(unused_mut)] + let mut app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); #[cfg(not(debug_assertions))] - let app = EditorApp::new(cc, layouts, theme); - // Wire hardware video decode into the VideoManager now that the shared device exists. + #[allow(unused_mut)] + let mut app = EditorApp::new(cc, layouts, theme); + // Wire hardware video decode into the VideoManager now that the shared device exists, and + // stash the shared device handles so the zero-copy export encoder can run on it too. #[cfg(target_os = "linux")] if shared_device_active { if let Some(rs) = cc.wgpu_render_state.as_ref() { hw_video::install(&app.video_manager, &rs.device, &rs.adapter); + app.shared_device = Some((rs.device.clone(), rs.queue.clone(), rs.adapter.clone())); } } Ok(Box::new(app)) @@ -990,6 +994,9 @@ struct EditorApp { audio_channels: u32, // Video decoding and management video_manager: std::sync::Arc>, // Shared video manager + /// The shared VAAPI-capable wgpu device (device, queue, adapter), `Some` only when active. Lets + /// the zero-copy export encoder run on it (GPU-resident decode→composite→encode). + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, // Webcam capture state webcam: Option, /// Latest polled webcam frame (updated each frame for preview) @@ -1327,6 +1334,7 @@ impl EditorApp { video_manager: std::sync::Arc::new(std::sync::Mutex::new( lightningbeam_core::video::VideoManager::new() )), + shared_device: None, webcam: None, webcam_frame: None, webcam_record_command: None, @@ -6125,6 +6133,9 @@ impl eframe::App for EditorApp { self.export_orchestrator = Some(export::ExportOrchestrator::new()); } + // Clone before the &mut self.export_orchestrator borrow below. + let shared_device = self.shared_device.clone(); + let export_started = if let Some(orchestrator) = &mut self.export_orchestrator { match export_result { ExportResult::Image(settings, output_path) => { @@ -6163,6 +6174,7 @@ impl eframe::App for EditorApp { Arc::clone(&self.video_manager), self.raster_store.clone(), self.current_file_path.clone(), + shared_device.clone(), ) { Ok(()) => true, Err(err) => { @@ -6184,6 +6196,7 @@ impl eframe::App for EditorApp { Arc::clone(&self.video_manager), self.raster_store.clone(), self.current_file_path.clone(), + shared_device.clone(), ) { Ok(()) => true, Err(err) => {