Wire zero-copy VAAPI H.264 into video+audio export

When exporting H.264 with audio, try the gpu-video-encoder ZeroCopyEncoder:
render each frame to RGBA and hardware-encode it into a VAAPI surface
inline, on the encoder's own VAAPI-capable wgpu device — no GPU->CPU
readback, no swscale, no software-encoder thread. Falls back to the
existing software path verbatim when VAAPI/the device is unavailable
(non-Linux, non-H264, or init failure), so it's additive.

- VideoExportState gains zero_copy: Option<ZeroCopyVideo> (encoder + its own
  vello renderer + ExportGpuResources + a reused RGBA target, all on the
  encoder's device).
- start_video_with_audio_export builds it for H.264 and skips spawning the
  software encoder thread when present.
- render_next_video_frame routes to a zero-copy arm that reuses
  render_frame_to_gpu_rgba on the encoder's device, then encode_rgba; on the
  last frame finish() writes the temp .mp4 and sets video_progress=Complete
  so the existing mux runs. video_thread=None makes the mux join a no-op.

Separate export device (vs modifying the eframe device) keeps this contained
to export. Video-only export stays on the software path for now. Runtime
verification (an actual H.264 export) is pending — cannot run the editor in
the dev container.
This commit is contained in:
Skyler Lehmkuhl 2026-06-25 15:43:46 -04:00
parent a00e73c4b3
commit 3e4f29c297
3 changed files with 170 additions and 17 deletions

View File

@ -3543,6 +3543,7 @@ dependencies = [
"egui_extras", "egui_extras",
"egui_node_graph2", "egui_node_graph2",
"ffmpeg-next", "ffmpeg-next",
"gpu-video-encoder",
"half", "half",
"image", "image",
"kurbo 0.12.0", "kurbo 0.12.0",

View File

@ -9,6 +9,7 @@ license = "GPL-3.0-or-later"
lightningbeam-core = { path = "../lightningbeam-core" } lightningbeam-core = { path = "../lightningbeam-core" }
daw-backend = { path = "../../daw-backend" } daw-backend = { path = "../../daw-backend" }
beamdsp = { path = "../beamdsp" } beamdsp = { path = "../beamdsp" }
gpu-video-encoder = { path = "../gpu-video-encoder" }
rtrb = "0.3" rtrb = "0.3"
cpal = "0.17" cpal = "0.17"
ffmpeg-next = { version = "8.0", features = ["static"] } ffmpeg-next = { version = "8.0", features = ["static"] }

View File

@ -66,6 +66,20 @@ pub struct VideoExportState {
next_frame_to_encode: usize, next_frame_to_encode: usize,
/// Performance metrics for instrumentation /// Performance metrics for instrumentation
perf_metrics: Option<perf_metrics::ExportMetrics>, perf_metrics: Option<perf_metrics::ExportMetrics>,
/// When `Some`, the video is produced zero-copy on its own VAAPI-capable device
/// (renders + hardware-encodes inline), bypassing the readback pipeline + encoder
/// thread. Only set for the parallel video+audio H.264 path when VAAPI is available.
zero_copy: Option<ZeroCopyVideo>,
}
/// Zero-copy VAAPI video production: renders each frame to RGBA and hardware-encodes it
/// into a VAAPI surface, all on the encoder's own wgpu device (no readback / swscale).
struct ZeroCopyVideo {
encoder: gpu_video_encoder::encoder::ZeroCopyEncoder,
renderer: vello::Renderer,
gpu_resources: video_exporter::ExportGpuResources,
/// Reused RGBA target (RENDER_ATTACHMENT | TEXTURE_BINDING) on the encoder's device.
rgba: wgpu::Texture,
} }
/// State for a single-frame image export (runs on the GPU render thread, one frame per update). /// State for a single-frame image export (runs on the GPU render thread, one frame per update).
@ -860,6 +874,7 @@ impl ExportOrchestrator {
frames_in_flight: 0, frames_in_flight: 0,
next_frame_to_encode: 0, next_frame_to_encode: 0,
perf_metrics: Some(perf_metrics::ExportMetrics::new()), perf_metrics: Some(perf_metrics::ExportMetrics::new()),
zero_copy: None, // video-only export uses the software path for now
}); });
println!("🎬 [VIDEO EXPORT] Encoder thread spawned, ready for frames"); println!("🎬 [VIDEO EXPORT] Encoder thread spawned, ready for frames");
@ -946,10 +961,66 @@ impl ExportOrchestrator {
let video_cancel_flag = Arc::clone(&self.cancel_flag); let video_cancel_flag = Arc::clone(&self.cancel_flag);
let audio_cancel_flag = Arc::clone(&self.cancel_flag); let audio_cancel_flag = Arc::clone(&self.cancel_flag);
// Spawn video encoder thread // Try the zero-copy VAAPI path for H.264: render + hardware-encode each frame inline
// on its own device, writing the temp .mp4 directly (no readback / swscale / encoder
// thread). Falls back to the software encoder thread when unavailable.
let zero_copy = if matches!(video_settings.codec, lightningbeam_core::export::VideoCodec::H264) {
match gpu_video_encoder::encoder::ZeroCopyEncoder::new(
video_width,
video_height,
video_framerate.round() as i32,
video_settings.quality.bitrate_kbps(),
&temp_video_path,
) {
Ok(encoder) => match vello::Renderer::new(
encoder.device(),
vello::RendererOptions {
use_cpu: false,
antialiasing_support: vello::AaSupport::all(),
num_init_threads: None,
pipeline_cache: None,
},
) {
Ok(renderer) => {
let gpu_resources = video_exporter::ExportGpuResources::new(
encoder.device(),
video_width,
video_height,
);
let rgba = encoder.device().create_texture(&wgpu::TextureDescriptor {
label: Some("zerocopy_export_rgba"),
size: wgpu::Extent3d { width: video_width, height: video_height, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT
| wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::COPY_SRC,
view_formats: &[],
});
println!("🎬 [PARALLEL EXPORT] zero-copy VAAPI H.264 enabled");
Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba })
}
Err(e) => {
println!("🎬 [PARALLEL EXPORT] zero-copy renderer init failed ({e}); software path");
None
}
},
Err(e) => {
println!("🎬 [PARALLEL EXPORT] zero-copy unavailable ({e}); software path");
None
}
}
} else {
None
};
// Spawn the software video encoder thread only when not zero-copy.
let video_thread = if zero_copy.is_none() {
let video_settings_clone = video_settings.clone(); let video_settings_clone = video_settings.clone();
let temp_video_path_clone = temp_video_path.clone(); let temp_video_path_clone = temp_video_path.clone();
let video_thread = std::thread::spawn(move || { Some(std::thread::spawn(move || {
Self::run_video_encoder( Self::run_video_encoder(
video_settings_clone, video_settings_clone,
temp_video_path_clone, temp_video_path_clone,
@ -958,7 +1029,14 @@ impl ExportOrchestrator {
video_cancel_flag, video_cancel_flag,
total_frames, total_frames,
); );
}); }))
} else {
// In zero-copy mode the render loop writes the temp .mp4 directly and sets
// `video_progress` on completion, so these are unused.
drop(frame_rx);
drop(video_progress_tx);
None
};
// Spawn audio export thread // Spawn audio export thread
let temp_audio_path_clone = temp_audio_path.clone(); let temp_audio_path_clone = temp_audio_path.clone();
@ -972,8 +1050,9 @@ impl ExportOrchestrator {
); );
}); });
// Initialize video export state for incremental rendering // Initialize video export state for incremental rendering. GPU resources + readback
// GPU resources and readback pipeline will be initialized lazily on first frame (needs device) // pipeline init lazily on the first frame (software path); zero-copy carries its own.
let frame_tx = if zero_copy.is_some() { None } else { Some(frame_tx) };
self.video_state = Some(VideoExportState { self.video_state = Some(VideoExportState {
current_frame: 0, current_frame: 0,
total_frames, total_frames,
@ -982,20 +1061,21 @@ impl ExportOrchestrator {
framerate: video_framerate, framerate: video_framerate,
width: video_width, width: video_width,
height: video_height, height: video_height,
frame_tx: Some(frame_tx), frame_tx,
gpu_resources: None, gpu_resources: None,
readback_pipeline: None, readback_pipeline: None,
cpu_yuv_converter: None, cpu_yuv_converter: None,
frames_in_flight: 0, frames_in_flight: 0,
next_frame_to_encode: 0, next_frame_to_encode: 0,
perf_metrics: Some(perf_metrics::ExportMetrics::new()), perf_metrics: Some(perf_metrics::ExportMetrics::new()),
zero_copy,
}); });
// Initialize parallel export state // Initialize parallel export state
self.parallel_export = Some(ParallelExportState { self.parallel_export = Some(ParallelExportState {
video_progress_rx, video_progress_rx,
audio_progress_rx, audio_progress_rx,
video_thread: Some(video_thread), video_thread,
audio_thread: Some(audio_thread), audio_thread: Some(audio_thread),
temp_video_path, temp_video_path,
temp_audio_path, temp_audio_path,
@ -1036,6 +1116,12 @@ impl ExportOrchestrator {
) -> Result<bool, String> { ) -> Result<bool, String> {
use std::time::Instant; use std::time::Instant;
// Zero-copy VAAPI path renders + hardware-encodes inline on its own device,
// ignoring the passed eframe device/queue/renderer.
if self.video_state.as_ref().map_or(false, |s| s.zero_copy.is_some()) {
return self.render_next_video_frame_zerocopy(document, image_cache, video_manager, raster_store);
}
let state = self.video_state.as_mut() let state = self.video_state.as_mut()
.ok_or("No video export in progress")?; .ok_or("No video export in progress")?;
@ -1204,6 +1290,71 @@ impl ExportOrchestrator {
Ok(true) // More work to do Ok(true) // More work to do
} }
/// Zero-copy video production: render the document to RGBA on the encoder's own device
/// and hardware-encode it into a VAAPI surface, writing the temp `.mp4` directly. Drives
/// the parallel export's `video_progress` and triggers the mux on completion.
fn render_next_video_frame_zerocopy(
&mut self,
document: &mut Document,
image_cache: &mut ImageCache,
video_manager: &Arc<std::sync::Mutex<VideoManager>>,
raster_store: Option<&lightningbeam_core::raster_store::RasterStore>,
) -> Result<bool, String> {
let (current, total) = {
let s = self.video_state.as_ref().ok_or("No video export in progress")?;
(s.current_frame, s.total_frames)
};
if current >= total {
// All frames rendered: flush + write the container trailer, then signal Complete
// so `poll_parallel_progress` muxes with the audio track.
let zc = self.video_state.as_mut().unwrap().zero_copy.take().unwrap();
let ZeroCopyVideo { encoder, .. } = zc;
encoder.finish()?;
if let Some(p) = self.parallel_export.as_mut() {
let path = p.temp_video_path.clone();
p.video_progress = Some(ExportProgress::Complete { output_path: path });
}
self.video_state = None;
println!("🎬 [VIDEO EXPORT] zero-copy complete: {} frames", total);
return Ok(false);
}
// Render one frame to RGBA on the encoder's device, then hardware-encode it.
{
let state = self.video_state.as_mut().unwrap();
let zc = state.zero_copy.as_mut().unwrap();
let (w, h) = (state.width, state.height);
let timestamp = state.start_time + (current as f64 / state.framerate);
let rgba_view = zc.rgba.create_view(&Default::default());
let cmd = video_exporter::render_frame_to_gpu_rgba(
document,
timestamp,
w,
h,
zc.encoder.device(),
zc.encoder.queue(),
&mut zc.renderer,
image_cache,
video_manager,
&mut zc.gpu_resources,
&rgba_view,
None,
false,
raster_store,
)?;
zc.encoder.queue().submit(Some(cmd.finish()));
zc.encoder.encode_rgba(&zc.rgba)?;
state.current_frame += 1;
}
let cur = self.video_state.as_ref().unwrap().current_frame;
if let Some(p) = self.parallel_export.as_mut() {
p.video_progress = Some(ExportProgress::FrameRendered { frame: cur, total });
}
Ok(true)
}
/// Background thread that receives frames and encodes them /// Background thread that receives frames and encodes them
fn run_video_encoder( fn run_video_encoder(
settings: VideoExportSettings, settings: VideoExportSettings,