core: hardware video decode engine in VideoDecoder (Stage 3b)

Extend the existing VideoDecoder with an optional hardware path, reusing its
demux/seek/keyframe/blob engine (no duplication):

- GpuVideoFrame (NV12 plane wgpu textures) + HwVideoImporter trait (editor
  implements the DMA-BUF import; the AVFrame crosses as an opaque pointer so
  core needn't reference the GPU crate) + HwDeviceHandle (opaque AVBufferRef).
- VideoManager::set_hardware_decode injects the VAAPI device + importer; each
  decoder attaches hw_device_ctx + a get_format(VAAPI) callback before opening,
  decodes into VAAPI surfaces, and imports them (no CPU copy).
- get_frame returns DecodedFrame::{Cpu,Gpu}; VideoFrame/VideoRenderInstance gain
  an optional `gpu`. The frame cache budgets GPU frames as ~w*h*3/2 and keys on
  whether the consumer wanted GPU.
- A hardware decoder serving a CPU consumer (export, render_hardware_ok=false)
  downloads the surface via av_hwframe_transfer_data then swscales — so export
  stays software/correct and only the preview goes GPU-resident. HW init or
  import failure falls back to software per clip.

Dormant until the editor injects an importer (next): no importer => software,
unchanged.
This commit is contained in:
Skyler Lehmkuhl 2026-06-26 02:06:28 -04:00
parent 7909b51df1
commit 863edc80fc
2 changed files with 334 additions and 93 deletions

View File

@ -221,8 +221,11 @@ fn decode_image_brush(data: &[u8]) -> Option<ImageBrush> {
/// A single decoded video frame ready for GPU upload, with its document-space transform. /// A single decoded video frame ready for GPU upload, with its document-space transform.
pub struct VideoRenderInstance { pub struct VideoRenderInstance {
/// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). /// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). Empty when `gpu` is `Some`.
pub rgba_data: Arc<Vec<u8>>, pub rgba_data: Arc<Vec<u8>>,
/// Hardware-decoded frame as GPU NV12 textures (preview path). When `Some`, the compositor
/// samples it directly (NV12→RGB) instead of uploading `rgba_data`.
pub gpu: Option<crate::video::GpuVideoFrame>,
pub width: u32, pub width: u32,
pub height: u32, pub height: u32,
/// Affine transform that maps from video-pixel space to document space. /// Affine transform that maps from video-pixel space to document space.
@ -609,6 +612,7 @@ pub fn render_layer_isolated(
}; };
instances.push(VideoRenderInstance { instances.push(VideoRenderInstance {
rgba_data: frame.rgba_data.clone(), rgba_data: frame.rgba_data.clone(),
gpu: frame.gpu.clone(),
width: frame.width, width: frame.width,
height: frame.height, height: frame.height,
transform: base_transform * clip_transform * frame_to_clip, transform: base_transform * clip_transform * frame_to_clip,
@ -629,6 +633,7 @@ pub fn render_layer_isolated(
* Affine::scale(scale); * Affine::scale(scale);
instances.push(VideoRenderInstance { instances.push(VideoRenderInstance {
rgba_data: frame.rgba_data.clone(), rgba_data: frame.rgba_data.clone(),
gpu: None, // webcam frames are always CPU
width: frame.width, width: frame.width,
height: frame.height, height: frame.height,
transform: cam_transform, transform: cam_transform,
@ -1260,6 +1265,7 @@ fn render_video_layer(
if let Some(ex) = extract.as_deref_mut() { if let Some(ex) = extract.as_deref_mut() {
ex.instances.push(VideoRenderInstance { ex.instances.push(VideoRenderInstance {
rgba_data: frame.rgba_data.clone(), rgba_data: frame.rgba_data.clone(),
gpu: frame.gpu.clone(),
width: frame.width, width: frame.width,
height: frame.height, height: frame.height,
transform: instance_transform * brush_transform, transform: instance_transform * brush_transform,
@ -1314,6 +1320,7 @@ fn render_video_layer(
if let Some(ex) = extract.as_deref_mut() { if let Some(ex) = extract.as_deref_mut() {
ex.instances.push(VideoRenderInstance { ex.instances.push(VideoRenderInstance {
rgba_data: frame.rgba_data.clone(), rgba_data: frame.rgba_data.clone(),
gpu: None, // webcam frames are always CPU
width: frame.width, width: frame.width,
height: frame.height, height: frame.height,
transform: preview_transform, transform: preview_transform,

View File

@ -101,6 +101,34 @@ pub struct VideoDecoder {
/// Building an swscale context isn't free; a stream's frames share one input format/size and a /// Building an swscale context isn't free; a stream's frames share one input format/size and a
/// consumer keeps one output size, so it's built once and rebuilt only when either changes. /// consumer keeps one output size, so it's built once and rebuilt only when either changes.
scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>, scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>,
/// When set (and `hw_failed` is false), decode on the GPU: attach `hw_device` as the decoder's
/// `hw_device_ctx`, decode into VAAPI surfaces, and hand each surface to `importer` to import as
/// wgpu NV12 textures (no CPU copy). `None`/failure → the software swscale path.
hw_device: Option<HwDeviceHandle>,
importer: Option<Arc<dyn HwVideoImporter>>,
/// Set if hardware decode init failed for this clip — fall back to software permanently.
hw_failed: bool,
}
/// A decoded frame: CPU RGBA (software) or GPU NV12 textures (hardware).
enum DecodedFrame {
Cpu { rgba: Vec<u8>, width: u32, height: u32 },
Gpu(GpuVideoFrame),
}
/// `get_format` callback for hardware decode: select VAAPI surfaces. With `hw_device_ctx` set,
/// FFmpeg auto-allocates the frames context.
unsafe extern "C" fn get_vaapi_format(
_ctx: *mut ffmpeg::ffi::AVCodecContext,
mut fmts: *const ffmpeg::ffi::AVPixelFormat,
) -> ffmpeg::ffi::AVPixelFormat {
while *fmts != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE {
if *fmts == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI {
return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI;
}
fmts = fmts.add(1);
}
ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE
} }
/// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only /// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only
@ -197,9 +225,28 @@ impl VideoDecoder {
last_decoded_ts: -1, last_decoded_ts: -1,
keyframe_positions, keyframe_positions,
scaler: None, scaler: None,
hw_device: None,
importer: None,
hw_failed: false,
}) })
} }
/// Configure hardware (VAAPI) decode for this clip. The next decoder open attaches `hw_device`
/// and decodes into VAAPI surfaces imported by `importer`. Resets any prior decoder so the new
/// mode takes effect on the next `get_frame`.
fn set_hardware(&mut self, hw_device: HwDeviceHandle, importer: Arc<dyn HwVideoImporter>) {
self.hw_device = Some(hw_device);
self.importer = Some(importer);
self.hw_failed = false;
self.decoder = None; // force a rebuild with hw_device_ctx
self.input = None;
}
/// Whether this decoder will hardware-decode (configured + not failed).
fn hw_active(&self) -> bool {
self.hw_device.is_some() && self.importer.is_some() && !self.hw_failed
}
/// The source this decoder reads from (file path or packed container blob). /// The source this decoder reads from (file path or packed container blob).
pub fn source(&self) -> VideoSource { pub fn source(&self) -> VideoSource {
self.source.clone() self.source.clone()
@ -226,7 +273,11 @@ impl VideoDecoder {
/// Decode a frame at the specified timestamp, at native resolution (public wrapper). /// Decode a frame at the specified timestamp, at native resolution (public wrapper).
pub fn decode_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> { pub fn decode_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> {
self.get_frame(timestamp, self.native_width, self.native_height).map(|(d, _, _)| d) // Software-only helper; request CPU output.
match self.get_frame(timestamp, self.native_width, self.native_height, false)? {
DecodedFrame::Cpu { rgba, .. } => Ok(rgba),
DecodedFrame::Gpu(_) => Err("decode_frame: unexpected GPU frame".into()),
}
} }
/// Build an index of all keyframe positions in the video by scanning packets /// Build an index of all keyframe positions in the video by scanning packets
@ -268,13 +319,19 @@ impl VideoDecoder {
} }
} }
/// Get a decoded frame at the specified timestamp /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns GPU
/// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns the /// NV12 textures when hardware-decoding and `want_gpu` (the consumer is on the shared device,
/// RGBA bytes and the actual output dimensions. /// i.e. the preview); otherwise CPU RGBA. A hardware decoder serving a CPU consumer (export)
fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32) -> Result<(Vec<u8>, u32, u32), String> { /// downloads the surface via `av_hwframe_transfer_data` then swscales. The `VideoManager` caches
/// the result, so the inner RGBA cache here is for CPU output only.
fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Result<DecodedFrame, String> {
use std::time::Instant; use std::time::Instant;
let t_start = Instant::now(); let t_start = Instant::now();
// `hw` = decoder is opened in hardware mode (produces VAAPI surfaces).
// `gpu_out` = return GPU textures (hw + the consumer can use them).
let hw = self.hw_active();
let gpu_out = hw && want_gpu;
let (out_w, out_h) = self.capped_output(target_w, target_h); let (out_w, out_h) = self.capped_output(target_w, target_h);
// Round timestamp to nearest frame boundary to improve cache hits // Round timestamp to nearest frame boundary to improve cache hits
@ -286,12 +343,14 @@ impl VideoDecoder {
let frame_ts = (rounded_timestamp / self.time_base) as i64; let frame_ts = (rounded_timestamp / self.time_base) as i64;
let cache_key = (frame_ts, out_w, out_h); let cache_key = (frame_ts, out_w, out_h);
// Check cache // Check the inner RGBA cache (CPU output only; GPU frames are cached by VideoManager).
if let Some(cached_frame) = self.frame_cache.get(&cache_key) { if !gpu_out {
if video_debug() { if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); if video_debug() {
eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis());
}
return Ok(DecodedFrame::Cpu { rgba: cached_frame.clone(), width: out_w, height: out_h });
} }
return Ok((cached_frame.clone(), out_w, out_h));
} }
// Determine if we need to seek // Determine if we need to seek
@ -336,9 +395,29 @@ impl VideoDecoder {
input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters()
).map_err(|e| e.to_string())?; ).map_err(|e| e.to_string())?;
let decoder = context_decoder.decoder().video() let mut dec_ctx = context_decoder.decoder();
.map_err(|e| e.to_string())?; if hw {
self.decoder = Some(decoder); // Attach the VAAPI device + format selector before opening so the decoder
// produces hardware surfaces.
unsafe {
let ctx = dec_ctx.as_mut_ptr();
let hwdev = self.hw_device.unwrap().0 as *mut ffmpeg::ffi::AVBufferRef;
(*ctx).hw_device_ctx = ffmpeg::ffi::av_buffer_ref(hwdev);
(*ctx).get_format = Some(get_vaapi_format);
}
}
match dec_ctx.video() {
Ok(decoder) => self.decoder = Some(decoder),
Err(e) if hw => {
// Hardware decode unavailable for this clip — fall back to software. This
// frame fails; the next call rebuilds a software decoder.
eprintln!("[Video] hardware decode unavailable ({e}); falling back to software");
self.hw_failed = true;
self.decoder = None;
return Err(format!("hw decode init failed: {e}"));
}
Err(e) => return Err(e.to_string()),
}
} }
self.input = Some(owned); self.input = Some(owned);
// Set last_decoded_ts to just before the seek target so forward playback works // Set last_decoded_ts to just before the seek target so forward playback works
@ -351,12 +430,14 @@ impl VideoDecoder {
// Decode frames until we find the one closest to our target timestamp // Decode frames until we find the one closest to our target timestamp
let mut best_frame_data: Option<Vec<u8>> = None; let mut best_frame_data: Option<Vec<u8>> = None;
let mut best_gpu: Option<GpuVideoFrame> = None;
let mut best_frame_ts: Option<i64> = None; let mut best_frame_ts: Option<i64> = None;
let t_decode_start = Instant::now(); let t_decode_start = Instant::now();
let mut decode_count = 0; let mut decode_count = 0;
let mut scale_time_ms = 0u128; let mut scale_time_ms = 0u128;
let mut hw_import_failed = false;
for (stream, packet) in input.packets() { 'decode: for (stream, packet) in input.packets() {
if stream.index() == self.stream_index { if stream.index() == self.stream_index {
decoder.send_packet(&packet) decoder.send_packet(&packet)
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
@ -376,73 +457,127 @@ impl VideoDecoder {
}; };
if is_better { if is_better {
let t_scale_start = Instant::now(); if gpu_out {
// Hardware + GPU consumer: import the VAAPI surface as wgpu NV12 textures
// Reuse the RGBA scaler across frames; rebuild only if the input // (no CPU copy).
// format/size or the requested output size changes. let importer = self.importer.as_ref().unwrap();
let need_new = match &self.scaler { match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } {
Some((fmt, w, h, ow, oh, _)) => { Some(gpu) => {
*fmt != frame.format() || *w != frame.width() || *h != frame.height() best_gpu = Some(gpu);
|| *ow != out_w || *oh != out_h best_frame_ts = Some(current_frame_ts);
}
None => {
// Import failed → fall back to software for this clip.
self.hw_failed = true;
hw_import_failed = true;
break 'decode;
}
} }
None => true, } else {
}; let t_scale_start = Instant::now();
if need_new {
let ctx = ffmpeg::software::scaling::context::Context::get( // A hardware decoder produces VAAPI surfaces; a CPU consumer (export)
frame.format(), // downloads to system memory first, then swscales like the software path.
frame.width(), let downloaded;
frame.height(), let src: &ffmpeg::util::frame::Video = if hw {
ffmpeg::format::Pixel::RGBA, let mut dl = ffmpeg::util::frame::Video::empty();
out_w, let r = unsafe {
out_h, ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0)
ffmpeg::software::scaling::flag::Flags::BILINEAR, };
).map_err(|e| e.to_string())?; if r < 0 {
self.scaler = Some((frame.format(), frame.width(), frame.height(), out_w, out_h, SendScaler(ctx))); return Err(format!("av_hwframe_transfer_data failed: {r}"));
}
downloaded = dl;
&downloaded
} else {
&frame
};
// Reuse the RGBA scaler across frames; rebuild only if the input
// format/size or the requested output size changes.
let need_new = match &self.scaler {
Some((fmt, w, h, ow, oh, _)) => {
*fmt != src.format() || *w != src.width() || *h != src.height()
|| *ow != out_w || *oh != out_h
}
None => true,
};
if need_new {
let ctx = ffmpeg::software::scaling::context::Context::get(
src.format(),
src.width(),
src.height(),
ffmpeg::format::Pixel::RGBA,
out_w,
out_h,
ffmpeg::software::scaling::flag::Flags::BILINEAR,
).map_err(|e| e.to_string())?;
self.scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx)));
}
let scaler = &mut self.scaler.as_mut().unwrap().5.0;
let mut rgb_frame = ffmpeg::util::frame::Video::empty();
scaler.run(src, &mut rgb_frame)
.map_err(|e| e.to_string())?;
// Remove stride padding to create tightly packed RGBA data
let width = out_w as usize;
let height = out_h as usize;
let stride = rgb_frame.stride(0);
let row_size = width * 4; // RGBA = 4 bytes per pixel
let source_data = rgb_frame.data(0);
let mut packed_data = Vec::with_capacity(row_size * height);
for y in 0..height {
let row_start = y * stride;
let row_end = row_start + row_size;
packed_data.extend_from_slice(&source_data[row_start..row_end]);
}
scale_time_ms += t_scale_start.elapsed().as_millis();
best_frame_data = Some(packed_data);
best_frame_ts = Some(current_frame_ts);
} }
let scaler = &mut self.scaler.as_mut().unwrap().5.0;
let mut rgb_frame = ffmpeg::util::frame::Video::empty();
scaler.run(&frame, &mut rgb_frame)
.map_err(|e| e.to_string())?;
// Remove stride padding to create tightly packed RGBA data
let width = out_w as usize;
let height = out_h as usize;
let stride = rgb_frame.stride(0);
let row_size = width * 4; // RGBA = 4 bytes per pixel
let source_data = rgb_frame.data(0);
let mut packed_data = Vec::with_capacity(row_size * height);
for y in 0..height {
let row_start = y * stride;
let row_end = row_start + row_size;
packed_data.extend_from_slice(&source_data[row_start..row_end]);
}
scale_time_ms += t_scale_start.elapsed().as_millis();
best_frame_data = Some(packed_data);
best_frame_ts = Some(current_frame_ts);
} }
// If we've reached or passed the target timestamp, we can stop // If we've reached or passed the target timestamp, we can stop
if current_frame_ts >= frame_ts { if current_frame_ts >= frame_ts {
// Found our frame, cache and return it if video_debug() {
if let Some(data) = best_frame_data { let total_time = t_start.elapsed().as_millis();
if video_debug() { let decode_time = t_decode_start.elapsed().as_millis();
let total_time = t_start.elapsed().as_millis(); eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}",
let decode_time = t_decode_start.elapsed().as_millis(); timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" });
eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms",
timestamp, decode_count, decode_time, scale_time_ms, total_time);
}
self.frame_cache.put(cache_key, data.clone());
return Ok((data, out_w, out_h));
} }
break; if gpu_out {
if let Some(gpu) = best_gpu.take() {
return Ok(DecodedFrame::Gpu(gpu));
}
} else if let Some(data) = best_frame_data {
self.frame_cache.put(cache_key, data.clone());
return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h });
}
break 'decode;
} }
} }
} }
} }
// Reached EOF without hitting the target, or HW import failed mid-stream.
if hw_import_failed {
self.decoder = None; // force a software rebuild next call (decoder borrow ended here)
self.input = None;
return Err("hardware frame import failed; retrying software".to_string());
}
// EOF: return the closest frame we found, if any.
if gpu_out {
if let Some(gpu) = best_gpu.take() {
return Ok(DecodedFrame::Gpu(gpu));
}
} else if let Some(data) = best_frame_data {
self.frame_cache.put(cache_key, data.clone());
return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h });
}
eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp); eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp);
Err("Failed to decode frame".to_string()) Err("Failed to decode frame".to_string())
} }
@ -494,7 +629,8 @@ pub fn generate_keyframe_thumbnails(
continue; continue;
} }
// Decode at the thumbnail width (large height so width is the constraint), capped to native. // Decode at the thumbnail width (large height so width is the constraint), capped to native.
if let Ok((rgba, _, _)) = decoder.get_frame(ks, thumb_width, 100_000) { // Thumbnail decoders are always software (no hardware importer).
if let Ok(DecodedFrame::Cpu { rgba, .. }) = decoder.get_frame(ks, thumb_width, 100_000, false) {
on_thumb(ks, Arc::new(rgba)); on_thumb(ks, Arc::new(rgba));
} }
} }
@ -559,10 +695,57 @@ pub fn probe_video(source: &VideoSource) -> Result<VideoMetadata, String> {
pub struct VideoFrame { pub struct VideoFrame {
pub width: u32, pub width: u32,
pub height: u32, pub height: u32,
/// CPU-decoded sRGB RGBA8 (software path). Empty when `gpu` is `Some`.
pub rgba_data: Arc<Vec<u8>>, pub rgba_data: Arc<Vec<u8>>,
/// Hardware-decoded frame living on the GPU (NV12 plane textures). When `Some`, the compositor
/// samples it directly and `rgba_data` is empty.
pub gpu: Option<GpuVideoFrame>,
pub timestamp: f64, pub timestamp: f64,
} }
/// A hardware-decoded video frame on the GPU: two NV12 plane textures (Y = R8, UV = RG8) imported
/// from a VAAPI DMA-BUF on the editor's shared wgpu device. The compositor samples these directly
/// (NV12→RGB), no CPU copy.
#[derive(Clone, Debug)]
pub struct GpuVideoFrame {
pub y: Arc<wgpu::Texture>,
pub uv: Arc<wgpu::Texture>,
pub width: u32,
pub height: u32,
/// Source YUV range: true = full/PC (0255), false = limited/TV (16235). Drives the NV12→RGB
/// offset/scale in the compositor.
pub full_range: bool,
}
/// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't
/// reference the GPU crate's ffmpeg-sys types) into [`GpuVideoFrame`] textures on the shared device.
/// Implemented by the editor; `gpu-video-encoder` does the actual DMA-BUF import.
pub trait HwVideoImporter: Send + Sync {
/// # Safety
/// `av_frame` must be a valid `*mut ffmpeg_sys_next::AVFrame` holding a VAAPI surface.
unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option<GpuVideoFrame>;
}
/// Opaque handle to the FFmpeg VAAPI hardware device (`*mut AVBufferRef`), created by the editor and
/// handed to core so decoders can attach it as `hw_device_ctx`. Core never frees it (the editor owns
/// it for the app's lifetime).
#[derive(Clone, Copy)]
pub struct HwDeviceHandle(pub *mut std::ffi::c_void);
// SAFETY: the pointer is an AVBufferRef whose refcount is managed by FFmpeg; we only `av_buffer_ref`
// it (atomic) and never free it, so sharing the handle across threads is sound.
unsafe impl Send for HwDeviceHandle {}
unsafe impl Sync for HwDeviceHandle {}
/// Approximate resident bytes of a cached frame for the byte budget: the CPU RGBA buffer, or for a
/// GPU (NV12) frame ~`w*h*3/2` of VRAM, so GPU frames stay bounded too.
fn frame_cache_bytes(frame: &VideoFrame) -> usize {
if frame.gpu.is_some() {
(frame.width as usize * frame.height as usize * 3) / 2
} else {
frame.rgba_data.len()
}
}
/// Manages video decoders and frame caching for multiple video clips /// Manages video decoders and frame caching for multiple video clips
pub struct VideoManager { pub struct VideoManager {
/// Pool of video decoders, one per clip /// Pool of video decoders, one per clip
@ -572,7 +755,7 @@ pub struct VideoManager {
/// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which /// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which
/// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600) /// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600)
/// so playback of arbitrarily long video never grows unbounded. /// so playback of arbitrarily long video never grows unbounded.
frame_cache: LruCache<(Uuid, i64, u32, u32), Arc<VideoFrame>>, frame_cache: LruCache<(Uuid, i64, u32, u32, bool), Arc<VideoFrame>>,
/// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len), /// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len),
/// kept in sync on insert/evict/remove so eviction is O(1) per frame. /// kept in sync on insert/evict/remove so eviction is O(1) per frame.
frame_cache_bytes: usize, frame_cache_bytes: usize,
@ -588,6 +771,15 @@ pub struct VideoManager {
/// Maximum number of frames to cache per decoder /// Maximum number of frames to cache per decoder
cache_size: usize, cache_size: usize,
/// Hardware (VAAPI) decode, injected by the editor once the shared device is up. When set, each
/// decoder attaches the VAAPI device and imports frames as GPU textures via `hw_importer`.
hw_device: Option<HwDeviceHandle>,
hw_importer: Option<Arc<dyn HwVideoImporter>>,
/// Whether the current render pass can consume GPU textures (preview = true; export = false,
/// since it composites on a different device → a hardware decoder downloads to CPU instead).
/// Set by the render caller before each pass.
render_hardware_ok: bool,
} }
/// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames). /// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames).
@ -610,9 +802,34 @@ impl VideoManager {
thumbnail_cache: HashMap::new(), thumbnail_cache: HashMap::new(),
thumbnails_complete: std::collections::HashSet::new(), thumbnails_complete: std::collections::HashSet::new(),
cache_size, cache_size,
hw_device: None,
hw_importer: None,
render_hardware_ok: true,
} }
} }
/// Set whether the upcoming render pass can consume GPU video textures (preview = true; export =
/// false). Call before `render_document_for_compositing`.
pub fn set_render_hardware_ok(&mut self, ok: bool) {
self.render_hardware_ok = ok;
}
/// Enable hardware (VAAPI) decode for all clips. Injected by the editor once the shared wgpu
/// device is active; `hw_device` is the FFmpeg VAAPI device and `importer` imports decoded
/// surfaces as GPU textures on that device. Applies to existing and future decoders. Clears the
/// frame cache (cached CPU frames would otherwise hide the new GPU frames).
pub fn set_hardware_decode(&mut self, hw_device: HwDeviceHandle, importer: Arc<dyn HwVideoImporter>) {
self.hw_device = Some(hw_device);
self.hw_importer = Some(Arc::clone(&importer));
for dec in self.decoders.values() {
if let Ok(mut d) = dec.lock() {
d.set_hardware(hw_device, Arc::clone(&importer));
}
}
self.frame_cache.clear();
self.frame_cache_bytes = 0;
}
/// Load a video file and create a decoder for it /// Load a video file and create a decoder for it
/// ///
/// `target_width` and `target_height` specify the maximum dimensions /// `target_width` and `target_height` specify the maximum dimensions
@ -632,7 +849,7 @@ impl VideoManager {
let metadata = probe_video(&source)?; let metadata = probe_video(&source)?;
// Create decoder with target dimensions, without building keyframe index // Create decoder with target dimensions, without building keyframe index
let decoder = VideoDecoder::new( let mut decoder = VideoDecoder::new(
source, source,
self.cache_size, self.cache_size,
Some(target_width), Some(target_width),
@ -640,6 +857,11 @@ impl VideoManager {
false, // Don't build keyframe index synchronously false, // Don't build keyframe index synchronously
)?; )?;
// Inherit hardware decode if the manager has it configured.
if let (Some(hw), Some(imp)) = (self.hw_device, &self.hw_importer) {
decoder.set_hardware(hw, Arc::clone(imp));
}
// Store decoder in pool // Store decoder in pool
self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder))); self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder)));
@ -648,14 +870,16 @@ impl VideoManager {
/// Get a decoded frame for a specific clip at a specific timestamp /// Get a decoded frame for a specific clip at a specific timestamp
/// ///
/// Returns None if the clip is not loaded or decoding fails. /// Returns None if the clip is not loaded or decoding fails. Frames are cached.
/// Frames are cached for performance. /// Whether a hardware decoder returns a GPU texture or downloads to CPU RGBA depends on
/// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the
/// preview, false for export, which composites on a different device).
pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> { pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> {
// Convert timestamp to milliseconds for cache key. The target size is part of the key: the let hardware_ok = self.render_hardware_ok;
// canvas (preview res) and an in-progress export (export res) request the same clip/time at // The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export
// different sizes, and must not collide. // (CPU, export res) request the same clip/time and must not collide or cross representation.
let timestamp_ms = (timestamp * 1000.0) as i64; let timestamp_ms = (timestamp * 1000.0) as i64;
let cache_key = (*clip_id, timestamp_ms, target_w, target_h); let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok);
// Check frame cache first // Check frame cache first
if let Some(cached_frame) = self.frame_cache.get(&cache_key) { if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
@ -668,15 +892,25 @@ impl VideoManager {
let mut decoder = decoder_arc.lock().ok()?; let mut decoder = decoder_arc.lock().ok()?;
// Decode the frame at the requested target (capped to native by the decoder). // Decode the frame at the requested target (capped to native by the decoder).
let (rgba_data, width, height) = decoder.get_frame(timestamp, target_w, target_h).ok()?; let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?;
drop(decoder); // release the lock before touching `self` drop(decoder); // release the lock before touching `self`
// Create VideoFrame and cache it // Create VideoFrame and cache it.
let frame = Arc::new(VideoFrame { let frame = Arc::new(match decoded {
width, DecodedFrame::Cpu { rgba, width, height } => VideoFrame {
height, width,
rgba_data: Arc::new(rgba_data), height,
timestamp, rgba_data: Arc::new(rgba),
gpu: None,
timestamp,
},
DecodedFrame::Gpu(gpu) => VideoFrame {
width: gpu.width,
height: gpu.height,
rgba_data: Arc::new(Vec::new()),
gpu: Some(gpu),
timestamp,
},
}); });
self.cache_frame(cache_key, Arc::clone(&frame)); self.cache_frame(cache_key, Arc::clone(&frame));
@ -686,16 +920,16 @@ impl VideoManager {
/// Insert a frame into the byte-budgeted cache, evicting least-recently-used /// Insert a frame into the byte-budgeted cache, evicting least-recently-used
/// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`]. /// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`].
fn cache_frame(&mut self, key: (Uuid, i64, u32, u32), frame: Arc<VideoFrame>) { fn cache_frame(&mut self, key: (Uuid, i64, u32, u32, bool), frame: Arc<VideoFrame>) {
let bytes = frame.rgba_data.len(); let bytes = frame_cache_bytes(&frame);
if let Some(old) = self.frame_cache.put(key, frame) { if let Some(old) = self.frame_cache.put(key, frame) {
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len()); self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&old));
} }
self.frame_cache_bytes += bytes; self.frame_cache_bytes += bytes;
// Keep at least one frame resident even if it alone exceeds the budget. // Keep at least one frame resident even if it alone exceeds the budget.
while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 { while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 {
if let Some((_, evicted)) = self.frame_cache.pop_lru() { if let Some((_, evicted)) = self.frame_cache.pop_lru() {
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(evicted.rgba_data.len()); self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&evicted));
} else { } else {
break; break;
} }
@ -802,15 +1036,15 @@ impl VideoManager {
// Remove all cached frames for this clip (LruCache has no retain; collect // Remove all cached frames for this clip (LruCache has no retain; collect
// matching keys, then pop each, keeping the byte total in sync). // matching keys, then pop each, keeping the byte total in sync).
let keys: Vec<(Uuid, i64, u32, u32)> = self let keys: Vec<(Uuid, i64, u32, u32, bool)> = self
.frame_cache .frame_cache
.iter() .iter()
.filter(|((id, _, _, _), _)| id == clip_id) .filter(|((id, _, _, _, _), _)| id == clip_id)
.map(|(k, _)| *k) .map(|(k, _)| *k)
.collect(); .collect();
for key in keys { for key in keys {
if let Some(frame) = self.frame_cache.pop(&key) { if let Some(frame) = self.frame_cache.pop(&key) {
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame.rgba_data.len()); self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&frame));
} }
} }