skyler
/
Lightningbeam
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

export: run the zero-copy H.264 encoder on the shared device (Stage 3c-export pt 2) 

Makes the common Linux H.264 export fully GPU-resident: decode (HW NV12) →
composite → VAAPI encode all on one device, no CPU round-trips.

- gpu-video-encoder: ZeroCopyEncoder now holds wgpu (device,queue,adapter) handles
  instead of owning a DrmDevice. New `new_on_device(device,queue,adapter,...)` runs
  the RGBA→NV12 render + DMA-BUF import on a passed device; `new` keeps building its
  own. The encoder only *imports* VAAPI surfaces (not export), so the shared
  import-capable device works directly.
- editor: stash the shared device handles on EditorApp (set in the creation closure
  from the eframe render_state when the shared device is active) and thread them
  through start_video_export / start_video_with_audio_export → try_build_zero_copy,
  which uses new_on_device when available. ZeroCopyVideo carries on_shared_device →
  the export composite uses hardware_ok=true (consuming HW-decoded GPU frames from
  pt 1) only then.

Falls back to the own-device encoder (decode downloads to CPU) when no shared
device. Tradeoff: the export now shares the GPU with the UI thread (was a separate
device); acceptable for the GPU-resident-decode win. Compiles; crate tests build.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Skyler Lehmkuhl 2026-06-26 05:18:23 -04:00
parent bb3369b709
commit ca9a70e10a
3 changed files with 86 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
lightningbeam-ui/gpu-video-encoder/src/encoder.rs
View File

@ -6,7 +6,7 @@
use crate::dmabuf::{self, ImportedNv12, Nv12DmaBuf};
use crate::render_nv12::Rgba2Nv12;
use crate::vk_device::{self, DrmDevice};
use crate::vk_device;
use ffmpeg_sys_next as ff;
use std::collections::HashMap;
use std::ffi::CString;
@ -19,7 +19,11 @@ fn averror(e: i32) -> i32 {
}
pub struct ZeroCopyEncoder {
drm: DrmDevice,
/// wgpu handles the NV12 render runs on — either an own `DrmDevice`'s (via `new`) or the
/// editor's shared device (via `new_on_device`). Cloned (Arc-backed) so the source can drop.
device: wgpu::Device,
queue: wgpu::Queue,
adapter: wgpu::Adapter,
renderer: Rgba2Nv12,
hw_device: *mut ff::AVBufferRef,
frames_ref: *mut ff::AVBufferRef,
@ -51,8 +55,30 @@ impl ZeroCopyEncoder {
output_path: &Path,
full_range: bool,
) -> Result<Self, String> {
// Build a dedicated DMA-BUF-import device and run the encoder on it.
let drm = vk_device::create()?;
let renderer = Rgba2Nv12::new(&drm.device, full_range);
Self::new_on_device(
drm.device, drm.queue, drm.adapter,
width, height, framerate, bitrate_kbps, output_path, full_range,
)
}
/// Build the encoder running its NV12 render + DMA-BUF import on an existing wgpu device (the
/// editor's shared device), so decode→composite→encode stay GPU-resident on one device. The
/// device must have the DMA-BUF import extensions (a `DrmDevice` / `vk_device::create_windowed`).
#[allow(clippy::too_many_arguments)]
pub fn new_on_device(
device: wgpu::Device,
queue: wgpu::Queue,
adapter: wgpu::Adapter,
width: u32,
height: u32,
framerate: i32,
bitrate_kbps: u32,
output_path: &Path,
full_range: bool,
) -> Result<Self, String> {
let renderer = Rgba2Nv12::new(&device, full_range);
unsafe {
let mut hw_device = crate::vaapi::create_device()?;
let name = CString::new("h264_vaapi").unwrap();
@ -153,7 +179,9 @@ impl ZeroCopyEncoder {
let stream_tb = (*stream).time_base;
Ok(Self {
drm,
device,
queue,
adapter,
renderer,
hw_device,
frames_ref,
@ -172,10 +200,10 @@ impl ZeroCopyEncoder {
/// The wgpu device frames must be rendered on (so the RGBA texture is importable).
pub fn device(&self) -> &wgpu::Device {
&self.drm.device
&self.device
}
pub fn queue(&self) -> &wgpu::Queue {
&self.drm.queue
&self.queue
}
/// Render `rgba` (an `Rgba8Unorm` texture on [`Self::device`], `TEXTURE_BINDING`)
@ -216,7 +244,7 @@ impl ZeroCopyEncoder {
uv_pitch: uv.pitch as u64,
ten_bit: false,
};
let imported = match dmabuf::import_raw(&self.drm.device, &self.drm.adapter, &buf) {
let imported = match dmabuf::import_raw(&self.device, &self.adapter, &buf) {
Ok(i) => i,
Err(e) => {
ff::av_frame_free(&mut (drm_f as *mut _));
@ -233,10 +261,10 @@ impl ZeroCopyEncoder {
let rgba_view = rgba.create_view(&Default::default());
let y_view = imp.y().create_view(&Default::default());
let uv_view = imp.uv().create_view(&Default::default());
let mut cmd = self.drm.device.create_command_encoder(&Default::default());
self.renderer.convert(&self.drm.device, &mut cmd, &rgba_view, &y_view, &uv_view);
self.drm.queue.submit(Some(cmd.finish()));
let _ = self.drm.device.poll(wgpu::PollType::wait_indefinitely());
let mut cmd = self.device.create_command_encoder(&Default::default());
self.renderer.convert(&self.device, &mut cmd, &rgba_view, &y_view, &uv_view);
self.queue.submit(Some(cmd.finish()));
let _ = self.device.poll(wgpu::PollType::wait_indefinitely());
// Encode the surface.
(*surf).pts = self.pts;

43
lightningbeam-ui/lightningbeam-editor/src/export/mod.rs
View File

@ -82,6 +82,8 @@ struct ZeroCopyVideo {
gpu_resources: video_exporter::ExportGpuResources,
/// Reused RGBA target (RENDER_ATTACHMENT | TEXTURE_BINDING) on the encoder's device.
rgba: wgpu::Texture,
/// True when running on the shared device → compositing can consume hardware-decoded GPU frames.
on_shared_device: bool,
}
/// State for a single-frame image export (runs on the GPU render thread, one frame per update).
@ -895,6 +897,7 @@ impl ExportOrchestrator {
height: u32,
framerate: f64,
output_path: &std::path::Path,
shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>,
) -> Option<ZeroCopyVideo> {
// Zero-copy is 8-bit H.264 only; HDR needs the 10-bit HEVC software path.
if settings.hdr.is_hdr()
@ -902,14 +905,25 @@ impl ExportOrchestrator {
{
return None;
}
let encoder = match gpu_video_encoder::encoder::ZeroCopyEncoder::new(
width,
height,
framerate.round() as i32,
settings.quality.bitrate_kbps(),
output_path,
settings.color_range.is_full(),
) {
let bitrate = settings.quality.bitrate_kbps();
let fr = framerate.round() as i32;
let full = settings.color_range.is_full();
let on_shared_device = shared_device.is_some();
// Prefer the shared device → decode→composite→encode stay GPU-resident on one device.
// Without it, the encoder builds its own device (decode still downloads to CPU per Step 1's
// hardware_ok=false on this path).
let encoder_result = match shared_device {
Some((device, queue, adapter)) => {
println!("🎬 [EXPORT] zero-copy on shared device (GPU-resident decode)");
gpu_video_encoder::encoder::ZeroCopyEncoder::new_on_device(
device, queue, adapter, width, height, fr, bitrate, output_path, full,
)
}
None => gpu_video_encoder::encoder::ZeroCopyEncoder::new(
width, height, fr, bitrate, output_path, full,
),
};
let encoder = match encoder_result {
Ok(e) => e,
Err(e) => {
println!("🎬 [EXPORT] zero-copy unavailable ({e}); software path");
@ -945,7 +959,7 @@ impl ExportOrchestrator {
view_formats: &[],
});
println!("🎬 [EXPORT] zero-copy VAAPI H.264 enabled");
Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba })
Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba, on_shared_device })
}
/// Start a video export in the background.
@ -963,6 +977,7 @@ impl ExportOrchestrator {
/// # Returns
/// Ok(()) on success, Err on failure
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments)]
pub fn start_video_export(
&mut self,
settings: VideoExportSettings,
@ -971,6 +986,8 @@ impl ExportOrchestrator {
video_manager: Arc<std::sync::Mutex<VideoManager>>,
raster_store: lightningbeam_core::raster_store::RasterStore,
container_path: Option<PathBuf>,
// The shared VAAPI device, `Some` only when active → zero-copy encode runs on it.
shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>,
) -> Result<(), String> {
println!("🎬 [VIDEO EXPORT] Starting video export");
@ -998,7 +1015,7 @@ impl ExportOrchestrator {
// success it returns here; otherwise we fall through to the software encoder thread.
#[cfg(target_os = "linux")]
{
if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path) {
if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path, shared_device) {
drop(frame_rx);
let document_snapshot = document.clone();
let mut image_cache = ImageCache::new();
@ -1062,6 +1079,8 @@ impl ExportOrchestrator {
video_manager: Arc<std::sync::Mutex<VideoManager>>,
raster_store: lightningbeam_core::raster_store::RasterStore,
container_path: Option<PathBuf>,
// The shared VAAPI device, `Some` only when active → zero-copy encode runs on it.
shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>,
) -> Result<(), String> {
println!("🎬🎵 [PARALLEL EXPORT] Starting parallel video+audio export");
@ -1128,7 +1147,7 @@ impl ExportOrchestrator {
// by `render_next_video_frame` on the UI thread.
#[cfg(target_os = "linux")]
let (video_thread, video_state) = match Self::try_build_zero_copy(
&video_settings, video_width, video_height, video_framerate, &temp_video_path,
&video_settings, video_width, video_height, video_framerate, &temp_video_path, shared_device,
) {
Some(zc) => {
drop(frame_rx); // the zero-copy path renders internally, no frame channel
@ -1500,7 +1519,7 @@ impl ExportOrchestrator {
None,
false,
Some(&raster_store),
false, // zero-copy runs on its own device → download HW frames to CPU
zc.on_shared_device, // GPU-resident decode only when on the shared device
) {
Ok(cmd) => cmd,
Err(e) => {

19
lightningbeam-ui/lightningbeam-editor/src/main.rs
View File

@ -299,14 +299,18 @@ fn main() -> eframe::Result {
options,
Box::new(move |cc| {
#[cfg(debug_assertions)]
let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app);
#[allow(unused_mut)]
let mut app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app);
#[cfg(not(debug_assertions))]
let app = EditorApp::new(cc, layouts, theme);
// Wire hardware video decode into the VideoManager now that the shared device exists.
#[allow(unused_mut)]
let mut app = EditorApp::new(cc, layouts, theme);
// Wire hardware video decode into the VideoManager now that the shared device exists, and
// stash the shared device handles so the zero-copy export encoder can run on it too.
#[cfg(target_os = "linux")]
if shared_device_active {
if let Some(rs) = cc.wgpu_render_state.as_ref() {
hw_video::install(&app.video_manager, &rs.device, &rs.adapter);
app.shared_device = Some((rs.device.clone(), rs.queue.clone(), rs.adapter.clone()));
}
}
Ok(Box::new(app))
@ -990,6 +994,9 @@ struct EditorApp {
audio_channels: u32,
// Video decoding and management
video_manager: std::sync::Arc<std::sync::Mutex<lightningbeam_core::video::VideoManager>>, // Shared video manager
/// The shared VAAPI-capable wgpu device (device, queue, adapter), `Some` only when active. Lets
/// the zero-copy export encoder run on it (GPU-resident decode→composite→encode).
shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>,
// Webcam capture state
webcam: Option<lightningbeam_core::webcam::WebcamCapture>,
/// Latest polled webcam frame (updated each frame for preview)
@ -1327,6 +1334,7 @@ impl EditorApp {
video_manager: std::sync::Arc::new(std::sync::Mutex::new(
lightningbeam_core::video::VideoManager::new()
)),
shared_device: None,
webcam: None,
webcam_frame: None,
webcam_record_command: None,
@ -6125,6 +6133,9 @@ impl eframe::App for EditorApp {
self.export_orchestrator = Some(export::ExportOrchestrator::new());
}
// Clone before the &mut self.export_orchestrator borrow below.
let shared_device = self.shared_device.clone();
let export_started = if let Some(orchestrator) = &mut self.export_orchestrator {
match export_result {
ExportResult::Image(settings, output_path) => {
@ -6163,6 +6174,7 @@ impl eframe::App for EditorApp {
Arc::clone(&self.video_manager),
self.raster_store.clone(),
self.current_file_path.clone(),
shared_device.clone(),
) {
Ok(()) => true,
Err(err) => {
@ -6184,6 +6196,7 @@ impl eframe::App for EditorApp {
Arc::clone(&self.video_manager),
self.raster_store.clone(),
self.current_file_path.clone(),
shared_device.clone(),
) {
Ok(()) => true,
Err(err) => {