core: hardware video decode engine in VideoDecoder (Stage 3b)
Extend the existing VideoDecoder with an optional hardware path, reusing its
demux/seek/keyframe/blob engine (no duplication):
- GpuVideoFrame (NV12 plane wgpu textures) + HwVideoImporter trait (editor
implements the DMA-BUF import; the AVFrame crosses as an opaque pointer so
core needn't reference the GPU crate) + HwDeviceHandle (opaque AVBufferRef).
- VideoManager::set_hardware_decode injects the VAAPI device + importer; each
decoder attaches hw_device_ctx + a get_format(VAAPI) callback before opening,
decodes into VAAPI surfaces, and imports them (no CPU copy).
- get_frame returns DecodedFrame::{Cpu,Gpu}; VideoFrame/VideoRenderInstance gain
an optional `gpu`. The frame cache budgets GPU frames as ~w*h*3/2 and keys on
whether the consumer wanted GPU.
- A hardware decoder serving a CPU consumer (export, render_hardware_ok=false)
downloads the surface via av_hwframe_transfer_data then swscales — so export
stays software/correct and only the preview goes GPU-resident. HW init or
import failure falls back to software per clip.
Dormant until the editor injects an importer (next): no importer => software,
unchanged.
This commit is contained in:
Skyler Lehmkuhl
parent
7909b51df1
commit
863edc80fc
|
|
@ -221,8 +221,11 @@ fn decode_image_brush(data: &[u8]) -> Option<ImageBrush> {
|
|||
|
||||
/// A single decoded video frame ready for GPU upload, with its document-space transform.
|
||||
pub struct VideoRenderInstance {
|
||||
/// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg).
|
||||
/// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). Empty when `gpu` is `Some`.
|
||||
pub rgba_data: Arc<Vec<u8>>,
|
||||
/// Hardware-decoded frame as GPU NV12 textures (preview path). When `Some`, the compositor
|
||||
/// samples it directly (NV12→RGB) instead of uploading `rgba_data`.
|
||||
pub gpu: Option<crate::video::GpuVideoFrame>,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Affine transform that maps from video-pixel space to document space.
|
||||
|
|
@ -609,6 +612,7 @@ pub fn render_layer_isolated(
|
|||
};
|
||||
instances.push(VideoRenderInstance {
|
||||
rgba_data: frame.rgba_data.clone(),
|
||||
gpu: frame.gpu.clone(),
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
transform: base_transform * clip_transform * frame_to_clip,
|
||||
|
|
@ -629,6 +633,7 @@ pub fn render_layer_isolated(
|
|||
* Affine::scale(scale);
|
||||
instances.push(VideoRenderInstance {
|
||||
rgba_data: frame.rgba_data.clone(),
|
||||
gpu: None, // webcam frames are always CPU
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
transform: cam_transform,
|
||||
|
|
@ -1260,6 +1265,7 @@ fn render_video_layer(
|
|||
if let Some(ex) = extract.as_deref_mut() {
|
||||
ex.instances.push(VideoRenderInstance {
|
||||
rgba_data: frame.rgba_data.clone(),
|
||||
gpu: frame.gpu.clone(),
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
transform: instance_transform * brush_transform,
|
||||
|
|
@ -1314,6 +1320,7 @@ fn render_video_layer(
|
|||
if let Some(ex) = extract.as_deref_mut() {
|
||||
ex.instances.push(VideoRenderInstance {
|
||||
rgba_data: frame.rgba_data.clone(),
|
||||
gpu: None, // webcam frames are always CPU
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
transform: preview_transform,
|
||||
|
|
|
|||
|
|
@ -101,6 +101,34 @@ pub struct VideoDecoder {
|
|||
/// Building an swscale context isn't free; a stream's frames share one input format/size and a
|
||||
/// consumer keeps one output size, so it's built once and rebuilt only when either changes.
|
||||
scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>,
|
||||
/// When set (and `hw_failed` is false), decode on the GPU: attach `hw_device` as the decoder's
|
||||
/// `hw_device_ctx`, decode into VAAPI surfaces, and hand each surface to `importer` to import as
|
||||
/// wgpu NV12 textures (no CPU copy). `None`/failure → the software swscale path.
|
||||
hw_device: Option<HwDeviceHandle>,
|
||||
importer: Option<Arc<dyn HwVideoImporter>>,
|
||||
/// Set if hardware decode init failed for this clip — fall back to software permanently.
|
||||
hw_failed: bool,
|
||||
}
|
||||
|
||||
/// A decoded frame: CPU RGBA (software) or GPU NV12 textures (hardware).
|
||||
enum DecodedFrame {
|
||||
Cpu { rgba: Vec<u8>, width: u32, height: u32 },
|
||||
Gpu(GpuVideoFrame),
|
||||
}
|
||||
|
||||
/// `get_format` callback for hardware decode: select VAAPI surfaces. With `hw_device_ctx` set,
|
||||
/// FFmpeg auto-allocates the frames context.
|
||||
unsafe extern "C" fn get_vaapi_format(
|
||||
_ctx: *mut ffmpeg::ffi::AVCodecContext,
|
||||
mut fmts: *const ffmpeg::ffi::AVPixelFormat,
|
||||
) -> ffmpeg::ffi::AVPixelFormat {
|
||||
while *fmts != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE {
|
||||
if *fmts == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI {
|
||||
return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI;
|
||||
}
|
||||
fmts = fmts.add(1);
|
||||
}
|
||||
ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE
|
||||
}
|
||||
|
||||
/// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only
|
||||
|
|
@ -197,9 +225,28 @@ impl VideoDecoder {
|
|||
last_decoded_ts: -1,
|
||||
keyframe_positions,
|
||||
scaler: None,
|
||||
hw_device: None,
|
||||
importer: None,
|
||||
hw_failed: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Configure hardware (VAAPI) decode for this clip. The next decoder open attaches `hw_device`
|
||||
/// and decodes into VAAPI surfaces imported by `importer`. Resets any prior decoder so the new
|
||||
/// mode takes effect on the next `get_frame`.
|
||||
fn set_hardware(&mut self, hw_device: HwDeviceHandle, importer: Arc<dyn HwVideoImporter>) {
|
||||
self.hw_device = Some(hw_device);
|
||||
self.importer = Some(importer);
|
||||
self.hw_failed = false;
|
||||
self.decoder = None; // force a rebuild with hw_device_ctx
|
||||
self.input = None;
|
||||
}
|
||||
|
||||
/// Whether this decoder will hardware-decode (configured + not failed).
|
||||
fn hw_active(&self) -> bool {
|
||||
self.hw_device.is_some() && self.importer.is_some() && !self.hw_failed
|
||||
}
|
||||
|
||||
/// The source this decoder reads from (file path or packed container blob).
|
||||
pub fn source(&self) -> VideoSource {
|
||||
self.source.clone()
|
||||
|
|
@ -226,7 +273,11 @@ impl VideoDecoder {
|
|||
|
||||
/// Decode a frame at the specified timestamp, at native resolution (public wrapper).
|
||||
pub fn decode_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> {
|
||||
self.get_frame(timestamp, self.native_width, self.native_height).map(|(d, _, _)| d)
|
||||
// Software-only helper; request CPU output.
|
||||
match self.get_frame(timestamp, self.native_width, self.native_height, false)? {
|
||||
DecodedFrame::Cpu { rgba, .. } => Ok(rgba),
|
||||
DecodedFrame::Gpu(_) => Err("decode_frame: unexpected GPU frame".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an index of all keyframe positions in the video by scanning packets
|
||||
|
|
@ -268,13 +319,19 @@ impl VideoDecoder {
|
|||
}
|
||||
}
|
||||
|
||||
/// Get a decoded frame at the specified timestamp
|
||||
/// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns the
|
||||
/// RGBA bytes and the actual output dimensions.
|
||||
fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32) -> Result<(Vec<u8>, u32, u32), String> {
|
||||
/// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns GPU
|
||||
/// NV12 textures when hardware-decoding and `want_gpu` (the consumer is on the shared device,
|
||||
/// i.e. the preview); otherwise CPU RGBA. A hardware decoder serving a CPU consumer (export)
|
||||
/// downloads the surface via `av_hwframe_transfer_data` then swscales. The `VideoManager` caches
|
||||
/// the result, so the inner RGBA cache here is for CPU output only.
|
||||
fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Result<DecodedFrame, String> {
|
||||
use std::time::Instant;
|
||||
let t_start = Instant::now();
|
||||
|
||||
// `hw` = decoder is opened in hardware mode (produces VAAPI surfaces).
|
||||
// `gpu_out` = return GPU textures (hw + the consumer can use them).
|
||||
let hw = self.hw_active();
|
||||
let gpu_out = hw && want_gpu;
|
||||
let (out_w, out_h) = self.capped_output(target_w, target_h);
|
||||
|
||||
// Round timestamp to nearest frame boundary to improve cache hits
|
||||
|
|
@ -286,12 +343,14 @@ impl VideoDecoder {
|
|||
let frame_ts = (rounded_timestamp / self.time_base) as i64;
|
||||
let cache_key = (frame_ts, out_w, out_h);
|
||||
|
||||
// Check cache
|
||||
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
||||
if video_debug() {
|
||||
eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis());
|
||||
// Check the inner RGBA cache (CPU output only; GPU frames are cached by VideoManager).
|
||||
if !gpu_out {
|
||||
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
||||
if video_debug() {
|
||||
eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis());
|
||||
}
|
||||
return Ok(DecodedFrame::Cpu { rgba: cached_frame.clone(), width: out_w, height: out_h });
|
||||
}
|
||||
return Ok((cached_frame.clone(), out_w, out_h));
|
||||
}
|
||||
|
||||
// Determine if we need to seek
|
||||
|
|
@ -336,9 +395,29 @@ impl VideoDecoder {
|
|||
input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters()
|
||||
).map_err(|e| e.to_string())?;
|
||||
|
||||
let decoder = context_decoder.decoder().video()
|
||||
.map_err(|e| e.to_string())?;
|
||||
self.decoder = Some(decoder);
|
||||
let mut dec_ctx = context_decoder.decoder();
|
||||
if hw {
|
||||
// Attach the VAAPI device + format selector before opening so the decoder
|
||||
// produces hardware surfaces.
|
||||
unsafe {
|
||||
let ctx = dec_ctx.as_mut_ptr();
|
||||
let hwdev = self.hw_device.unwrap().0 as *mut ffmpeg::ffi::AVBufferRef;
|
||||
(*ctx).hw_device_ctx = ffmpeg::ffi::av_buffer_ref(hwdev);
|
||||
(*ctx).get_format = Some(get_vaapi_format);
|
||||
}
|
||||
}
|
||||
match dec_ctx.video() {
|
||||
Ok(decoder) => self.decoder = Some(decoder),
|
||||
Err(e) if hw => {
|
||||
// Hardware decode unavailable for this clip — fall back to software. This
|
||||
// frame fails; the next call rebuilds a software decoder.
|
||||
eprintln!("[Video] hardware decode unavailable ({e}); falling back to software");
|
||||
self.hw_failed = true;
|
||||
self.decoder = None;
|
||||
return Err(format!("hw decode init failed: {e}"));
|
||||
}
|
||||
Err(e) => return Err(e.to_string()),
|
||||
}
|
||||
}
|
||||
self.input = Some(owned);
|
||||
// Set last_decoded_ts to just before the seek target so forward playback works
|
||||
|
|
@ -351,12 +430,14 @@ impl VideoDecoder {
|
|||
|
||||
// Decode frames until we find the one closest to our target timestamp
|
||||
let mut best_frame_data: Option<Vec<u8>> = None;
|
||||
let mut best_gpu: Option<GpuVideoFrame> = None;
|
||||
let mut best_frame_ts: Option<i64> = None;
|
||||
let t_decode_start = Instant::now();
|
||||
let mut decode_count = 0;
|
||||
let mut scale_time_ms = 0u128;
|
||||
let mut hw_import_failed = false;
|
||||
|
||||
for (stream, packet) in input.packets() {
|
||||
'decode: for (stream, packet) in input.packets() {
|
||||
if stream.index() == self.stream_index {
|
||||
decoder.send_packet(&packet)
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
|
@ -376,73 +457,127 @@ impl VideoDecoder {
|
|||
};
|
||||
|
||||
if is_better {
|
||||
let t_scale_start = Instant::now();
|
||||
|
||||
// Reuse the RGBA scaler across frames; rebuild only if the input
|
||||
// format/size or the requested output size changes.
|
||||
let need_new = match &self.scaler {
|
||||
Some((fmt, w, h, ow, oh, _)) => {
|
||||
*fmt != frame.format() || *w != frame.width() || *h != frame.height()
|
||||
|| *ow != out_w || *oh != out_h
|
||||
if gpu_out {
|
||||
// Hardware + GPU consumer: import the VAAPI surface as wgpu NV12 textures
|
||||
// (no CPU copy).
|
||||
let importer = self.importer.as_ref().unwrap();
|
||||
match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } {
|
||||
Some(gpu) => {
|
||||
best_gpu = Some(gpu);
|
||||
best_frame_ts = Some(current_frame_ts);
|
||||
}
|
||||
None => {
|
||||
// Import failed → fall back to software for this clip.
|
||||
self.hw_failed = true;
|
||||
hw_import_failed = true;
|
||||
break 'decode;
|
||||
}
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
if need_new {
|
||||
let ctx = ffmpeg::software::scaling::context::Context::get(
|
||||
frame.format(),
|
||||
frame.width(),
|
||||
frame.height(),
|
||||
ffmpeg::format::Pixel::RGBA,
|
||||
out_w,
|
||||
out_h,
|
||||
ffmpeg::software::scaling::flag::Flags::BILINEAR,
|
||||
).map_err(|e| e.to_string())?;
|
||||
self.scaler = Some((frame.format(), frame.width(), frame.height(), out_w, out_h, SendScaler(ctx)));
|
||||
} else {
|
||||
let t_scale_start = Instant::now();
|
||||
|
||||
// A hardware decoder produces VAAPI surfaces; a CPU consumer (export)
|
||||
// downloads to system memory first, then swscales like the software path.
|
||||
let downloaded;
|
||||
let src: &ffmpeg::util::frame::Video = if hw {
|
||||
let mut dl = ffmpeg::util::frame::Video::empty();
|
||||
let r = unsafe {
|
||||
ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0)
|
||||
};
|
||||
if r < 0 {
|
||||
return Err(format!("av_hwframe_transfer_data failed: {r}"));
|
||||
}
|
||||
downloaded = dl;
|
||||
&downloaded
|
||||
} else {
|
||||
&frame
|
||||
};
|
||||
|
||||
// Reuse the RGBA scaler across frames; rebuild only if the input
|
||||
// format/size or the requested output size changes.
|
||||
let need_new = match &self.scaler {
|
||||
Some((fmt, w, h, ow, oh, _)) => {
|
||||
*fmt != src.format() || *w != src.width() || *h != src.height()
|
||||
|| *ow != out_w || *oh != out_h
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
if need_new {
|
||||
let ctx = ffmpeg::software::scaling::context::Context::get(
|
||||
src.format(),
|
||||
src.width(),
|
||||
src.height(),
|
||||
ffmpeg::format::Pixel::RGBA,
|
||||
out_w,
|
||||
out_h,
|
||||
ffmpeg::software::scaling::flag::Flags::BILINEAR,
|
||||
).map_err(|e| e.to_string())?;
|
||||
self.scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx)));
|
||||
}
|
||||
let scaler = &mut self.scaler.as_mut().unwrap().5.0;
|
||||
|
||||
let mut rgb_frame = ffmpeg::util::frame::Video::empty();
|
||||
scaler.run(src, &mut rgb_frame)
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// Remove stride padding to create tightly packed RGBA data
|
||||
let width = out_w as usize;
|
||||
let height = out_h as usize;
|
||||
let stride = rgb_frame.stride(0);
|
||||
let row_size = width * 4; // RGBA = 4 bytes per pixel
|
||||
let source_data = rgb_frame.data(0);
|
||||
|
||||
let mut packed_data = Vec::with_capacity(row_size * height);
|
||||
for y in 0..height {
|
||||
let row_start = y * stride;
|
||||
let row_end = row_start + row_size;
|
||||
packed_data.extend_from_slice(&source_data[row_start..row_end]);
|
||||
}
|
||||
|
||||
scale_time_ms += t_scale_start.elapsed().as_millis();
|
||||
best_frame_data = Some(packed_data);
|
||||
best_frame_ts = Some(current_frame_ts);
|
||||
}
|
||||
let scaler = &mut self.scaler.as_mut().unwrap().5.0;
|
||||
|
||||
let mut rgb_frame = ffmpeg::util::frame::Video::empty();
|
||||
scaler.run(&frame, &mut rgb_frame)
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// Remove stride padding to create tightly packed RGBA data
|
||||
let width = out_w as usize;
|
||||
let height = out_h as usize;
|
||||
let stride = rgb_frame.stride(0);
|
||||
let row_size = width * 4; // RGBA = 4 bytes per pixel
|
||||
let source_data = rgb_frame.data(0);
|
||||
|
||||
let mut packed_data = Vec::with_capacity(row_size * height);
|
||||
for y in 0..height {
|
||||
let row_start = y * stride;
|
||||
let row_end = row_start + row_size;
|
||||
packed_data.extend_from_slice(&source_data[row_start..row_end]);
|
||||
}
|
||||
|
||||
scale_time_ms += t_scale_start.elapsed().as_millis();
|
||||
best_frame_data = Some(packed_data);
|
||||
best_frame_ts = Some(current_frame_ts);
|
||||
}
|
||||
|
||||
// If we've reached or passed the target timestamp, we can stop
|
||||
if current_frame_ts >= frame_ts {
|
||||
// Found our frame, cache and return it
|
||||
if let Some(data) = best_frame_data {
|
||||
if video_debug() {
|
||||
let total_time = t_start.elapsed().as_millis();
|
||||
let decode_time = t_decode_start.elapsed().as_millis();
|
||||
eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms",
|
||||
timestamp, decode_count, decode_time, scale_time_ms, total_time);
|
||||
}
|
||||
self.frame_cache.put(cache_key, data.clone());
|
||||
return Ok((data, out_w, out_h));
|
||||
if video_debug() {
|
||||
let total_time = t_start.elapsed().as_millis();
|
||||
let decode_time = t_decode_start.elapsed().as_millis();
|
||||
eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}",
|
||||
timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" });
|
||||
}
|
||||
break;
|
||||
if gpu_out {
|
||||
if let Some(gpu) = best_gpu.take() {
|
||||
return Ok(DecodedFrame::Gpu(gpu));
|
||||
}
|
||||
} else if let Some(data) = best_frame_data {
|
||||
self.frame_cache.put(cache_key, data.clone());
|
||||
return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h });
|
||||
}
|
||||
break 'decode;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reached EOF without hitting the target, or HW import failed mid-stream.
|
||||
if hw_import_failed {
|
||||
self.decoder = None; // force a software rebuild next call (decoder borrow ended here)
|
||||
self.input = None;
|
||||
return Err("hardware frame import failed; retrying software".to_string());
|
||||
}
|
||||
// EOF: return the closest frame we found, if any.
|
||||
if gpu_out {
|
||||
if let Some(gpu) = best_gpu.take() {
|
||||
return Ok(DecodedFrame::Gpu(gpu));
|
||||
}
|
||||
} else if let Some(data) = best_frame_data {
|
||||
self.frame_cache.put(cache_key, data.clone());
|
||||
return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h });
|
||||
}
|
||||
|
||||
eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp);
|
||||
Err("Failed to decode frame".to_string())
|
||||
}
|
||||
|
|
@ -494,7 +629,8 @@ pub fn generate_keyframe_thumbnails(
|
|||
continue;
|
||||
}
|
||||
// Decode at the thumbnail width (large height so width is the constraint), capped to native.
|
||||
if let Ok((rgba, _, _)) = decoder.get_frame(ks, thumb_width, 100_000) {
|
||||
// Thumbnail decoders are always software (no hardware importer).
|
||||
if let Ok(DecodedFrame::Cpu { rgba, .. }) = decoder.get_frame(ks, thumb_width, 100_000, false) {
|
||||
on_thumb(ks, Arc::new(rgba));
|
||||
}
|
||||
}
|
||||
|
|
@ -559,10 +695,57 @@ pub fn probe_video(source: &VideoSource) -> Result<VideoMetadata, String> {
|
|||
pub struct VideoFrame {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// CPU-decoded sRGB RGBA8 (software path). Empty when `gpu` is `Some`.
|
||||
pub rgba_data: Arc<Vec<u8>>,
|
||||
/// Hardware-decoded frame living on the GPU (NV12 plane textures). When `Some`, the compositor
|
||||
/// samples it directly and `rgba_data` is empty.
|
||||
pub gpu: Option<GpuVideoFrame>,
|
||||
pub timestamp: f64,
|
||||
}
|
||||
|
||||
/// A hardware-decoded video frame on the GPU: two NV12 plane textures (Y = R8, UV = RG8) imported
|
||||
/// from a VAAPI DMA-BUF on the editor's shared wgpu device. The compositor samples these directly
|
||||
/// (NV12→RGB), no CPU copy.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GpuVideoFrame {
|
||||
pub y: Arc<wgpu::Texture>,
|
||||
pub uv: Arc<wgpu::Texture>,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Source YUV range: true = full/PC (0–255), false = limited/TV (16–235). Drives the NV12→RGB
|
||||
/// offset/scale in the compositor.
|
||||
pub full_range: bool,
|
||||
}
|
||||
|
||||
/// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't
|
||||
/// reference the GPU crate's ffmpeg-sys types) into [`GpuVideoFrame`] textures on the shared device.
|
||||
/// Implemented by the editor; `gpu-video-encoder` does the actual DMA-BUF import.
|
||||
pub trait HwVideoImporter: Send + Sync {
|
||||
/// # Safety
|
||||
/// `av_frame` must be a valid `*mut ffmpeg_sys_next::AVFrame` holding a VAAPI surface.
|
||||
unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option<GpuVideoFrame>;
|
||||
}
|
||||
|
||||
/// Opaque handle to the FFmpeg VAAPI hardware device (`*mut AVBufferRef`), created by the editor and
|
||||
/// handed to core so decoders can attach it as `hw_device_ctx`. Core never frees it (the editor owns
|
||||
/// it for the app's lifetime).
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct HwDeviceHandle(pub *mut std::ffi::c_void);
|
||||
// SAFETY: the pointer is an AVBufferRef whose refcount is managed by FFmpeg; we only `av_buffer_ref`
|
||||
// it (atomic) and never free it, so sharing the handle across threads is sound.
|
||||
unsafe impl Send for HwDeviceHandle {}
|
||||
unsafe impl Sync for HwDeviceHandle {}
|
||||
|
||||
/// Approximate resident bytes of a cached frame for the byte budget: the CPU RGBA buffer, or for a
|
||||
/// GPU (NV12) frame ~`w*h*3/2` of VRAM, so GPU frames stay bounded too.
|
||||
fn frame_cache_bytes(frame: &VideoFrame) -> usize {
|
||||
if frame.gpu.is_some() {
|
||||
(frame.width as usize * frame.height as usize * 3) / 2
|
||||
} else {
|
||||
frame.rgba_data.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Manages video decoders and frame caching for multiple video clips
|
||||
pub struct VideoManager {
|
||||
/// Pool of video decoders, one per clip
|
||||
|
|
@ -572,7 +755,7 @@ pub struct VideoManager {
|
|||
/// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which
|
||||
/// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600)
|
||||
/// so playback of arbitrarily long video never grows unbounded.
|
||||
frame_cache: LruCache<(Uuid, i64, u32, u32), Arc<VideoFrame>>,
|
||||
frame_cache: LruCache<(Uuid, i64, u32, u32, bool), Arc<VideoFrame>>,
|
||||
/// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len),
|
||||
/// kept in sync on insert/evict/remove so eviction is O(1) per frame.
|
||||
frame_cache_bytes: usize,
|
||||
|
|
@ -588,6 +771,15 @@ pub struct VideoManager {
|
|||
|
||||
/// Maximum number of frames to cache per decoder
|
||||
cache_size: usize,
|
||||
|
||||
/// Hardware (VAAPI) decode, injected by the editor once the shared device is up. When set, each
|
||||
/// decoder attaches the VAAPI device and imports frames as GPU textures via `hw_importer`.
|
||||
hw_device: Option<HwDeviceHandle>,
|
||||
hw_importer: Option<Arc<dyn HwVideoImporter>>,
|
||||
/// Whether the current render pass can consume GPU textures (preview = true; export = false,
|
||||
/// since it composites on a different device → a hardware decoder downloads to CPU instead).
|
||||
/// Set by the render caller before each pass.
|
||||
render_hardware_ok: bool,
|
||||
}
|
||||
|
||||
/// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames).
|
||||
|
|
@ -610,9 +802,34 @@ impl VideoManager {
|
|||
thumbnail_cache: HashMap::new(),
|
||||
thumbnails_complete: std::collections::HashSet::new(),
|
||||
cache_size,
|
||||
hw_device: None,
|
||||
hw_importer: None,
|
||||
render_hardware_ok: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set whether the upcoming render pass can consume GPU video textures (preview = true; export =
|
||||
/// false). Call before `render_document_for_compositing`.
|
||||
pub fn set_render_hardware_ok(&mut self, ok: bool) {
|
||||
self.render_hardware_ok = ok;
|
||||
}
|
||||
|
||||
/// Enable hardware (VAAPI) decode for all clips. Injected by the editor once the shared wgpu
|
||||
/// device is active; `hw_device` is the FFmpeg VAAPI device and `importer` imports decoded
|
||||
/// surfaces as GPU textures on that device. Applies to existing and future decoders. Clears the
|
||||
/// frame cache (cached CPU frames would otherwise hide the new GPU frames).
|
||||
pub fn set_hardware_decode(&mut self, hw_device: HwDeviceHandle, importer: Arc<dyn HwVideoImporter>) {
|
||||
self.hw_device = Some(hw_device);
|
||||
self.hw_importer = Some(Arc::clone(&importer));
|
||||
for dec in self.decoders.values() {
|
||||
if let Ok(mut d) = dec.lock() {
|
||||
d.set_hardware(hw_device, Arc::clone(&importer));
|
||||
}
|
||||
}
|
||||
self.frame_cache.clear();
|
||||
self.frame_cache_bytes = 0;
|
||||
}
|
||||
|
||||
/// Load a video file and create a decoder for it
|
||||
///
|
||||
/// `target_width` and `target_height` specify the maximum dimensions
|
||||
|
|
@ -632,7 +849,7 @@ impl VideoManager {
|
|||
let metadata = probe_video(&source)?;
|
||||
|
||||
// Create decoder with target dimensions, without building keyframe index
|
||||
let decoder = VideoDecoder::new(
|
||||
let mut decoder = VideoDecoder::new(
|
||||
source,
|
||||
self.cache_size,
|
||||
Some(target_width),
|
||||
|
|
@ -640,6 +857,11 @@ impl VideoManager {
|
|||
false, // Don't build keyframe index synchronously
|
||||
)?;
|
||||
|
||||
// Inherit hardware decode if the manager has it configured.
|
||||
if let (Some(hw), Some(imp)) = (self.hw_device, &self.hw_importer) {
|
||||
decoder.set_hardware(hw, Arc::clone(imp));
|
||||
}
|
||||
|
||||
// Store decoder in pool
|
||||
self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder)));
|
||||
|
||||
|
|
@ -648,14 +870,16 @@ impl VideoManager {
|
|||
|
||||
/// Get a decoded frame for a specific clip at a specific timestamp
|
||||
///
|
||||
/// Returns None if the clip is not loaded or decoding fails.
|
||||
/// Frames are cached for performance.
|
||||
/// Returns None if the clip is not loaded or decoding fails. Frames are cached.
|
||||
/// Whether a hardware decoder returns a GPU texture or downloads to CPU RGBA depends on
|
||||
/// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the
|
||||
/// preview, false for export, which composites on a different device).
|
||||
pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> {
|
||||
// Convert timestamp to milliseconds for cache key. The target size is part of the key: the
|
||||
// canvas (preview res) and an in-progress export (export res) request the same clip/time at
|
||||
// different sizes, and must not collide.
|
||||
let hardware_ok = self.render_hardware_ok;
|
||||
// The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export
|
||||
// (CPU, export res) request the same clip/time and must not collide or cross representation.
|
||||
let timestamp_ms = (timestamp * 1000.0) as i64;
|
||||
let cache_key = (*clip_id, timestamp_ms, target_w, target_h);
|
||||
let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok);
|
||||
|
||||
// Check frame cache first
|
||||
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
||||
|
|
@ -668,15 +892,25 @@ impl VideoManager {
|
|||
let mut decoder = decoder_arc.lock().ok()?;
|
||||
|
||||
// Decode the frame at the requested target (capped to native by the decoder).
|
||||
let (rgba_data, width, height) = decoder.get_frame(timestamp, target_w, target_h).ok()?;
|
||||
let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?;
|
||||
drop(decoder); // release the lock before touching `self`
|
||||
|
||||
// Create VideoFrame and cache it
|
||||
let frame = Arc::new(VideoFrame {
|
||||
width,
|
||||
height,
|
||||
rgba_data: Arc::new(rgba_data),
|
||||
timestamp,
|
||||
// Create VideoFrame and cache it.
|
||||
let frame = Arc::new(match decoded {
|
||||
DecodedFrame::Cpu { rgba, width, height } => VideoFrame {
|
||||
width,
|
||||
height,
|
||||
rgba_data: Arc::new(rgba),
|
||||
gpu: None,
|
||||
timestamp,
|
||||
},
|
||||
DecodedFrame::Gpu(gpu) => VideoFrame {
|
||||
width: gpu.width,
|
||||
height: gpu.height,
|
||||
rgba_data: Arc::new(Vec::new()),
|
||||
gpu: Some(gpu),
|
||||
timestamp,
|
||||
},
|
||||
});
|
||||
|
||||
self.cache_frame(cache_key, Arc::clone(&frame));
|
||||
|
|
@ -686,16 +920,16 @@ impl VideoManager {
|
|||
|
||||
/// Insert a frame into the byte-budgeted cache, evicting least-recently-used
|
||||
/// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`].
|
||||
fn cache_frame(&mut self, key: (Uuid, i64, u32, u32), frame: Arc<VideoFrame>) {
|
||||
let bytes = frame.rgba_data.len();
|
||||
fn cache_frame(&mut self, key: (Uuid, i64, u32, u32, bool), frame: Arc<VideoFrame>) {
|
||||
let bytes = frame_cache_bytes(&frame);
|
||||
if let Some(old) = self.frame_cache.put(key, frame) {
|
||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len());
|
||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&old));
|
||||
}
|
||||
self.frame_cache_bytes += bytes;
|
||||
// Keep at least one frame resident even if it alone exceeds the budget.
|
||||
while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 {
|
||||
if let Some((_, evicted)) = self.frame_cache.pop_lru() {
|
||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(evicted.rgba_data.len());
|
||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&evicted));
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
|
@ -802,15 +1036,15 @@ impl VideoManager {
|
|||
|
||||
// Remove all cached frames for this clip (LruCache has no retain; collect
|
||||
// matching keys, then pop each, keeping the byte total in sync).
|
||||
let keys: Vec<(Uuid, i64, u32, u32)> = self
|
||||
let keys: Vec<(Uuid, i64, u32, u32, bool)> = self
|
||||
.frame_cache
|
||||
.iter()
|
||||
.filter(|((id, _, _, _), _)| id == clip_id)
|
||||
.filter(|((id, _, _, _, _), _)| id == clip_id)
|
||||
.map(|(k, _)| *k)
|
||||
.collect();
|
||||
for key in keys {
|
||||
if let Some(frame) = self.frame_cache.pop(&key) {
|
||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame.rgba_data.len());
|
||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&frame));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue