core: hardware video decode engine in VideoDecoder (Stage 3b)
Extend the existing VideoDecoder with an optional hardware path, reusing its
demux/seek/keyframe/blob engine (no duplication):
- GpuVideoFrame (NV12 plane wgpu textures) + HwVideoImporter trait (editor
implements the DMA-BUF import; the AVFrame crosses as an opaque pointer so
core needn't reference the GPU crate) + HwDeviceHandle (opaque AVBufferRef).
- VideoManager::set_hardware_decode injects the VAAPI device + importer; each
decoder attaches hw_device_ctx + a get_format(VAAPI) callback before opening,
decodes into VAAPI surfaces, and imports them (no CPU copy).
- get_frame returns DecodedFrame::{Cpu,Gpu}; VideoFrame/VideoRenderInstance gain
an optional `gpu`. The frame cache budgets GPU frames as ~w*h*3/2 and keys on
whether the consumer wanted GPU.
- A hardware decoder serving a CPU consumer (export, render_hardware_ok=false)
downloads the surface via av_hwframe_transfer_data then swscales — so export
stays software/correct and only the preview goes GPU-resident. HW init or
import failure falls back to software per clip.
Dormant until the editor injects an importer (next): no importer => software,
unchanged.
This commit is contained in:
parent
7909b51df1
commit
863edc80fc
|
|
@ -221,8 +221,11 @@ fn decode_image_brush(data: &[u8]) -> Option<ImageBrush> {
|
||||||
|
|
||||||
/// A single decoded video frame ready for GPU upload, with its document-space transform.
|
/// A single decoded video frame ready for GPU upload, with its document-space transform.
|
||||||
pub struct VideoRenderInstance {
|
pub struct VideoRenderInstance {
|
||||||
/// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg).
|
/// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). Empty when `gpu` is `Some`.
|
||||||
pub rgba_data: Arc<Vec<u8>>,
|
pub rgba_data: Arc<Vec<u8>>,
|
||||||
|
/// Hardware-decoded frame as GPU NV12 textures (preview path). When `Some`, the compositor
|
||||||
|
/// samples it directly (NV12→RGB) instead of uploading `rgba_data`.
|
||||||
|
pub gpu: Option<crate::video::GpuVideoFrame>,
|
||||||
pub width: u32,
|
pub width: u32,
|
||||||
pub height: u32,
|
pub height: u32,
|
||||||
/// Affine transform that maps from video-pixel space to document space.
|
/// Affine transform that maps from video-pixel space to document space.
|
||||||
|
|
@ -609,6 +612,7 @@ pub fn render_layer_isolated(
|
||||||
};
|
};
|
||||||
instances.push(VideoRenderInstance {
|
instances.push(VideoRenderInstance {
|
||||||
rgba_data: frame.rgba_data.clone(),
|
rgba_data: frame.rgba_data.clone(),
|
||||||
|
gpu: frame.gpu.clone(),
|
||||||
width: frame.width,
|
width: frame.width,
|
||||||
height: frame.height,
|
height: frame.height,
|
||||||
transform: base_transform * clip_transform * frame_to_clip,
|
transform: base_transform * clip_transform * frame_to_clip,
|
||||||
|
|
@ -629,6 +633,7 @@ pub fn render_layer_isolated(
|
||||||
* Affine::scale(scale);
|
* Affine::scale(scale);
|
||||||
instances.push(VideoRenderInstance {
|
instances.push(VideoRenderInstance {
|
||||||
rgba_data: frame.rgba_data.clone(),
|
rgba_data: frame.rgba_data.clone(),
|
||||||
|
gpu: None, // webcam frames are always CPU
|
||||||
width: frame.width,
|
width: frame.width,
|
||||||
height: frame.height,
|
height: frame.height,
|
||||||
transform: cam_transform,
|
transform: cam_transform,
|
||||||
|
|
@ -1260,6 +1265,7 @@ fn render_video_layer(
|
||||||
if let Some(ex) = extract.as_deref_mut() {
|
if let Some(ex) = extract.as_deref_mut() {
|
||||||
ex.instances.push(VideoRenderInstance {
|
ex.instances.push(VideoRenderInstance {
|
||||||
rgba_data: frame.rgba_data.clone(),
|
rgba_data: frame.rgba_data.clone(),
|
||||||
|
gpu: frame.gpu.clone(),
|
||||||
width: frame.width,
|
width: frame.width,
|
||||||
height: frame.height,
|
height: frame.height,
|
||||||
transform: instance_transform * brush_transform,
|
transform: instance_transform * brush_transform,
|
||||||
|
|
@ -1314,6 +1320,7 @@ fn render_video_layer(
|
||||||
if let Some(ex) = extract.as_deref_mut() {
|
if let Some(ex) = extract.as_deref_mut() {
|
||||||
ex.instances.push(VideoRenderInstance {
|
ex.instances.push(VideoRenderInstance {
|
||||||
rgba_data: frame.rgba_data.clone(),
|
rgba_data: frame.rgba_data.clone(),
|
||||||
|
gpu: None, // webcam frames are always CPU
|
||||||
width: frame.width,
|
width: frame.width,
|
||||||
height: frame.height,
|
height: frame.height,
|
||||||
transform: preview_transform,
|
transform: preview_transform,
|
||||||
|
|
|
||||||
|
|
@ -101,6 +101,34 @@ pub struct VideoDecoder {
|
||||||
/// Building an swscale context isn't free; a stream's frames share one input format/size and a
|
/// Building an swscale context isn't free; a stream's frames share one input format/size and a
|
||||||
/// consumer keeps one output size, so it's built once and rebuilt only when either changes.
|
/// consumer keeps one output size, so it's built once and rebuilt only when either changes.
|
||||||
scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>,
|
scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>,
|
||||||
|
/// When set (and `hw_failed` is false), decode on the GPU: attach `hw_device` as the decoder's
|
||||||
|
/// `hw_device_ctx`, decode into VAAPI surfaces, and hand each surface to `importer` to import as
|
||||||
|
/// wgpu NV12 textures (no CPU copy). `None`/failure → the software swscale path.
|
||||||
|
hw_device: Option<HwDeviceHandle>,
|
||||||
|
importer: Option<Arc<dyn HwVideoImporter>>,
|
||||||
|
/// Set if hardware decode init failed for this clip — fall back to software permanently.
|
||||||
|
hw_failed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A decoded frame: CPU RGBA (software) or GPU NV12 textures (hardware).
|
||||||
|
enum DecodedFrame {
|
||||||
|
Cpu { rgba: Vec<u8>, width: u32, height: u32 },
|
||||||
|
Gpu(GpuVideoFrame),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `get_format` callback for hardware decode: select VAAPI surfaces. With `hw_device_ctx` set,
|
||||||
|
/// FFmpeg auto-allocates the frames context.
|
||||||
|
unsafe extern "C" fn get_vaapi_format(
|
||||||
|
_ctx: *mut ffmpeg::ffi::AVCodecContext,
|
||||||
|
mut fmts: *const ffmpeg::ffi::AVPixelFormat,
|
||||||
|
) -> ffmpeg::ffi::AVPixelFormat {
|
||||||
|
while *fmts != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE {
|
||||||
|
if *fmts == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI {
|
||||||
|
return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI;
|
||||||
|
}
|
||||||
|
fmts = fmts.add(1);
|
||||||
|
}
|
||||||
|
ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only
|
/// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only
|
||||||
|
|
@ -197,9 +225,28 @@ impl VideoDecoder {
|
||||||
last_decoded_ts: -1,
|
last_decoded_ts: -1,
|
||||||
keyframe_positions,
|
keyframe_positions,
|
||||||
scaler: None,
|
scaler: None,
|
||||||
|
hw_device: None,
|
||||||
|
importer: None,
|
||||||
|
hw_failed: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Configure hardware (VAAPI) decode for this clip. The next decoder open attaches `hw_device`
|
||||||
|
/// and decodes into VAAPI surfaces imported by `importer`. Resets any prior decoder so the new
|
||||||
|
/// mode takes effect on the next `get_frame`.
|
||||||
|
fn set_hardware(&mut self, hw_device: HwDeviceHandle, importer: Arc<dyn HwVideoImporter>) {
|
||||||
|
self.hw_device = Some(hw_device);
|
||||||
|
self.importer = Some(importer);
|
||||||
|
self.hw_failed = false;
|
||||||
|
self.decoder = None; // force a rebuild with hw_device_ctx
|
||||||
|
self.input = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether this decoder will hardware-decode (configured + not failed).
|
||||||
|
fn hw_active(&self) -> bool {
|
||||||
|
self.hw_device.is_some() && self.importer.is_some() && !self.hw_failed
|
||||||
|
}
|
||||||
|
|
||||||
/// The source this decoder reads from (file path or packed container blob).
|
/// The source this decoder reads from (file path or packed container blob).
|
||||||
pub fn source(&self) -> VideoSource {
|
pub fn source(&self) -> VideoSource {
|
||||||
self.source.clone()
|
self.source.clone()
|
||||||
|
|
@ -226,7 +273,11 @@ impl VideoDecoder {
|
||||||
|
|
||||||
/// Decode a frame at the specified timestamp, at native resolution (public wrapper).
|
/// Decode a frame at the specified timestamp, at native resolution (public wrapper).
|
||||||
pub fn decode_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> {
|
pub fn decode_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> {
|
||||||
self.get_frame(timestamp, self.native_width, self.native_height).map(|(d, _, _)| d)
|
// Software-only helper; request CPU output.
|
||||||
|
match self.get_frame(timestamp, self.native_width, self.native_height, false)? {
|
||||||
|
DecodedFrame::Cpu { rgba, .. } => Ok(rgba),
|
||||||
|
DecodedFrame::Gpu(_) => Err("decode_frame: unexpected GPU frame".into()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build an index of all keyframe positions in the video by scanning packets
|
/// Build an index of all keyframe positions in the video by scanning packets
|
||||||
|
|
@ -268,13 +319,19 @@ impl VideoDecoder {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get a decoded frame at the specified timestamp
|
/// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns GPU
|
||||||
/// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns the
|
/// NV12 textures when hardware-decoding and `want_gpu` (the consumer is on the shared device,
|
||||||
/// RGBA bytes and the actual output dimensions.
|
/// i.e. the preview); otherwise CPU RGBA. A hardware decoder serving a CPU consumer (export)
|
||||||
fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32) -> Result<(Vec<u8>, u32, u32), String> {
|
/// downloads the surface via `av_hwframe_transfer_data` then swscales. The `VideoManager` caches
|
||||||
|
/// the result, so the inner RGBA cache here is for CPU output only.
|
||||||
|
fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Result<DecodedFrame, String> {
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
let t_start = Instant::now();
|
let t_start = Instant::now();
|
||||||
|
|
||||||
|
// `hw` = decoder is opened in hardware mode (produces VAAPI surfaces).
|
||||||
|
// `gpu_out` = return GPU textures (hw + the consumer can use them).
|
||||||
|
let hw = self.hw_active();
|
||||||
|
let gpu_out = hw && want_gpu;
|
||||||
let (out_w, out_h) = self.capped_output(target_w, target_h);
|
let (out_w, out_h) = self.capped_output(target_w, target_h);
|
||||||
|
|
||||||
// Round timestamp to nearest frame boundary to improve cache hits
|
// Round timestamp to nearest frame boundary to improve cache hits
|
||||||
|
|
@ -286,12 +343,14 @@ impl VideoDecoder {
|
||||||
let frame_ts = (rounded_timestamp / self.time_base) as i64;
|
let frame_ts = (rounded_timestamp / self.time_base) as i64;
|
||||||
let cache_key = (frame_ts, out_w, out_h);
|
let cache_key = (frame_ts, out_w, out_h);
|
||||||
|
|
||||||
// Check cache
|
// Check the inner RGBA cache (CPU output only; GPU frames are cached by VideoManager).
|
||||||
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
if !gpu_out {
|
||||||
if video_debug() {
|
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
||||||
eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis());
|
if video_debug() {
|
||||||
|
eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis());
|
||||||
|
}
|
||||||
|
return Ok(DecodedFrame::Cpu { rgba: cached_frame.clone(), width: out_w, height: out_h });
|
||||||
}
|
}
|
||||||
return Ok((cached_frame.clone(), out_w, out_h));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine if we need to seek
|
// Determine if we need to seek
|
||||||
|
|
@ -336,9 +395,29 @@ impl VideoDecoder {
|
||||||
input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters()
|
input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters()
|
||||||
).map_err(|e| e.to_string())?;
|
).map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
let decoder = context_decoder.decoder().video()
|
let mut dec_ctx = context_decoder.decoder();
|
||||||
.map_err(|e| e.to_string())?;
|
if hw {
|
||||||
self.decoder = Some(decoder);
|
// Attach the VAAPI device + format selector before opening so the decoder
|
||||||
|
// produces hardware surfaces.
|
||||||
|
unsafe {
|
||||||
|
let ctx = dec_ctx.as_mut_ptr();
|
||||||
|
let hwdev = self.hw_device.unwrap().0 as *mut ffmpeg::ffi::AVBufferRef;
|
||||||
|
(*ctx).hw_device_ctx = ffmpeg::ffi::av_buffer_ref(hwdev);
|
||||||
|
(*ctx).get_format = Some(get_vaapi_format);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match dec_ctx.video() {
|
||||||
|
Ok(decoder) => self.decoder = Some(decoder),
|
||||||
|
Err(e) if hw => {
|
||||||
|
// Hardware decode unavailable for this clip — fall back to software. This
|
||||||
|
// frame fails; the next call rebuilds a software decoder.
|
||||||
|
eprintln!("[Video] hardware decode unavailable ({e}); falling back to software");
|
||||||
|
self.hw_failed = true;
|
||||||
|
self.decoder = None;
|
||||||
|
return Err(format!("hw decode init failed: {e}"));
|
||||||
|
}
|
||||||
|
Err(e) => return Err(e.to_string()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
self.input = Some(owned);
|
self.input = Some(owned);
|
||||||
// Set last_decoded_ts to just before the seek target so forward playback works
|
// Set last_decoded_ts to just before the seek target so forward playback works
|
||||||
|
|
@ -351,12 +430,14 @@ impl VideoDecoder {
|
||||||
|
|
||||||
// Decode frames until we find the one closest to our target timestamp
|
// Decode frames until we find the one closest to our target timestamp
|
||||||
let mut best_frame_data: Option<Vec<u8>> = None;
|
let mut best_frame_data: Option<Vec<u8>> = None;
|
||||||
|
let mut best_gpu: Option<GpuVideoFrame> = None;
|
||||||
let mut best_frame_ts: Option<i64> = None;
|
let mut best_frame_ts: Option<i64> = None;
|
||||||
let t_decode_start = Instant::now();
|
let t_decode_start = Instant::now();
|
||||||
let mut decode_count = 0;
|
let mut decode_count = 0;
|
||||||
let mut scale_time_ms = 0u128;
|
let mut scale_time_ms = 0u128;
|
||||||
|
let mut hw_import_failed = false;
|
||||||
|
|
||||||
for (stream, packet) in input.packets() {
|
'decode: for (stream, packet) in input.packets() {
|
||||||
if stream.index() == self.stream_index {
|
if stream.index() == self.stream_index {
|
||||||
decoder.send_packet(&packet)
|
decoder.send_packet(&packet)
|
||||||
.map_err(|e| e.to_string())?;
|
.map_err(|e| e.to_string())?;
|
||||||
|
|
@ -376,73 +457,127 @@ impl VideoDecoder {
|
||||||
};
|
};
|
||||||
|
|
||||||
if is_better {
|
if is_better {
|
||||||
let t_scale_start = Instant::now();
|
if gpu_out {
|
||||||
|
// Hardware + GPU consumer: import the VAAPI surface as wgpu NV12 textures
|
||||||
// Reuse the RGBA scaler across frames; rebuild only if the input
|
// (no CPU copy).
|
||||||
// format/size or the requested output size changes.
|
let importer = self.importer.as_ref().unwrap();
|
||||||
let need_new = match &self.scaler {
|
match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } {
|
||||||
Some((fmt, w, h, ow, oh, _)) => {
|
Some(gpu) => {
|
||||||
*fmt != frame.format() || *w != frame.width() || *h != frame.height()
|
best_gpu = Some(gpu);
|
||||||
|| *ow != out_w || *oh != out_h
|
best_frame_ts = Some(current_frame_ts);
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
// Import failed → fall back to software for this clip.
|
||||||
|
self.hw_failed = true;
|
||||||
|
hw_import_failed = true;
|
||||||
|
break 'decode;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
None => true,
|
} else {
|
||||||
};
|
let t_scale_start = Instant::now();
|
||||||
if need_new {
|
|
||||||
let ctx = ffmpeg::software::scaling::context::Context::get(
|
// A hardware decoder produces VAAPI surfaces; a CPU consumer (export)
|
||||||
frame.format(),
|
// downloads to system memory first, then swscales like the software path.
|
||||||
frame.width(),
|
let downloaded;
|
||||||
frame.height(),
|
let src: &ffmpeg::util::frame::Video = if hw {
|
||||||
ffmpeg::format::Pixel::RGBA,
|
let mut dl = ffmpeg::util::frame::Video::empty();
|
||||||
out_w,
|
let r = unsafe {
|
||||||
out_h,
|
ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0)
|
||||||
ffmpeg::software::scaling::flag::Flags::BILINEAR,
|
};
|
||||||
).map_err(|e| e.to_string())?;
|
if r < 0 {
|
||||||
self.scaler = Some((frame.format(), frame.width(), frame.height(), out_w, out_h, SendScaler(ctx)));
|
return Err(format!("av_hwframe_transfer_data failed: {r}"));
|
||||||
|
}
|
||||||
|
downloaded = dl;
|
||||||
|
&downloaded
|
||||||
|
} else {
|
||||||
|
&frame
|
||||||
|
};
|
||||||
|
|
||||||
|
// Reuse the RGBA scaler across frames; rebuild only if the input
|
||||||
|
// format/size or the requested output size changes.
|
||||||
|
let need_new = match &self.scaler {
|
||||||
|
Some((fmt, w, h, ow, oh, _)) => {
|
||||||
|
*fmt != src.format() || *w != src.width() || *h != src.height()
|
||||||
|
|| *ow != out_w || *oh != out_h
|
||||||
|
}
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
if need_new {
|
||||||
|
let ctx = ffmpeg::software::scaling::context::Context::get(
|
||||||
|
src.format(),
|
||||||
|
src.width(),
|
||||||
|
src.height(),
|
||||||
|
ffmpeg::format::Pixel::RGBA,
|
||||||
|
out_w,
|
||||||
|
out_h,
|
||||||
|
ffmpeg::software::scaling::flag::Flags::BILINEAR,
|
||||||
|
).map_err(|e| e.to_string())?;
|
||||||
|
self.scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx)));
|
||||||
|
}
|
||||||
|
let scaler = &mut self.scaler.as_mut().unwrap().5.0;
|
||||||
|
|
||||||
|
let mut rgb_frame = ffmpeg::util::frame::Video::empty();
|
||||||
|
scaler.run(src, &mut rgb_frame)
|
||||||
|
.map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
|
// Remove stride padding to create tightly packed RGBA data
|
||||||
|
let width = out_w as usize;
|
||||||
|
let height = out_h as usize;
|
||||||
|
let stride = rgb_frame.stride(0);
|
||||||
|
let row_size = width * 4; // RGBA = 4 bytes per pixel
|
||||||
|
let source_data = rgb_frame.data(0);
|
||||||
|
|
||||||
|
let mut packed_data = Vec::with_capacity(row_size * height);
|
||||||
|
for y in 0..height {
|
||||||
|
let row_start = y * stride;
|
||||||
|
let row_end = row_start + row_size;
|
||||||
|
packed_data.extend_from_slice(&source_data[row_start..row_end]);
|
||||||
|
}
|
||||||
|
|
||||||
|
scale_time_ms += t_scale_start.elapsed().as_millis();
|
||||||
|
best_frame_data = Some(packed_data);
|
||||||
|
best_frame_ts = Some(current_frame_ts);
|
||||||
}
|
}
|
||||||
let scaler = &mut self.scaler.as_mut().unwrap().5.0;
|
|
||||||
|
|
||||||
let mut rgb_frame = ffmpeg::util::frame::Video::empty();
|
|
||||||
scaler.run(&frame, &mut rgb_frame)
|
|
||||||
.map_err(|e| e.to_string())?;
|
|
||||||
|
|
||||||
// Remove stride padding to create tightly packed RGBA data
|
|
||||||
let width = out_w as usize;
|
|
||||||
let height = out_h as usize;
|
|
||||||
let stride = rgb_frame.stride(0);
|
|
||||||
let row_size = width * 4; // RGBA = 4 bytes per pixel
|
|
||||||
let source_data = rgb_frame.data(0);
|
|
||||||
|
|
||||||
let mut packed_data = Vec::with_capacity(row_size * height);
|
|
||||||
for y in 0..height {
|
|
||||||
let row_start = y * stride;
|
|
||||||
let row_end = row_start + row_size;
|
|
||||||
packed_data.extend_from_slice(&source_data[row_start..row_end]);
|
|
||||||
}
|
|
||||||
|
|
||||||
scale_time_ms += t_scale_start.elapsed().as_millis();
|
|
||||||
best_frame_data = Some(packed_data);
|
|
||||||
best_frame_ts = Some(current_frame_ts);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we've reached or passed the target timestamp, we can stop
|
// If we've reached or passed the target timestamp, we can stop
|
||||||
if current_frame_ts >= frame_ts {
|
if current_frame_ts >= frame_ts {
|
||||||
// Found our frame, cache and return it
|
if video_debug() {
|
||||||
if let Some(data) = best_frame_data {
|
let total_time = t_start.elapsed().as_millis();
|
||||||
if video_debug() {
|
let decode_time = t_decode_start.elapsed().as_millis();
|
||||||
let total_time = t_start.elapsed().as_millis();
|
eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}",
|
||||||
let decode_time = t_decode_start.elapsed().as_millis();
|
timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" });
|
||||||
eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms",
|
|
||||||
timestamp, decode_count, decode_time, scale_time_ms, total_time);
|
|
||||||
}
|
|
||||||
self.frame_cache.put(cache_key, data.clone());
|
|
||||||
return Ok((data, out_w, out_h));
|
|
||||||
}
|
}
|
||||||
break;
|
if gpu_out {
|
||||||
|
if let Some(gpu) = best_gpu.take() {
|
||||||
|
return Ok(DecodedFrame::Gpu(gpu));
|
||||||
|
}
|
||||||
|
} else if let Some(data) = best_frame_data {
|
||||||
|
self.frame_cache.put(cache_key, data.clone());
|
||||||
|
return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h });
|
||||||
|
}
|
||||||
|
break 'decode;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Reached EOF without hitting the target, or HW import failed mid-stream.
|
||||||
|
if hw_import_failed {
|
||||||
|
self.decoder = None; // force a software rebuild next call (decoder borrow ended here)
|
||||||
|
self.input = None;
|
||||||
|
return Err("hardware frame import failed; retrying software".to_string());
|
||||||
|
}
|
||||||
|
// EOF: return the closest frame we found, if any.
|
||||||
|
if gpu_out {
|
||||||
|
if let Some(gpu) = best_gpu.take() {
|
||||||
|
return Ok(DecodedFrame::Gpu(gpu));
|
||||||
|
}
|
||||||
|
} else if let Some(data) = best_frame_data {
|
||||||
|
self.frame_cache.put(cache_key, data.clone());
|
||||||
|
return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h });
|
||||||
|
}
|
||||||
|
|
||||||
eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp);
|
eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp);
|
||||||
Err("Failed to decode frame".to_string())
|
Err("Failed to decode frame".to_string())
|
||||||
}
|
}
|
||||||
|
|
@ -494,7 +629,8 @@ pub fn generate_keyframe_thumbnails(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Decode at the thumbnail width (large height so width is the constraint), capped to native.
|
// Decode at the thumbnail width (large height so width is the constraint), capped to native.
|
||||||
if let Ok((rgba, _, _)) = decoder.get_frame(ks, thumb_width, 100_000) {
|
// Thumbnail decoders are always software (no hardware importer).
|
||||||
|
if let Ok(DecodedFrame::Cpu { rgba, .. }) = decoder.get_frame(ks, thumb_width, 100_000, false) {
|
||||||
on_thumb(ks, Arc::new(rgba));
|
on_thumb(ks, Arc::new(rgba));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -559,10 +695,57 @@ pub fn probe_video(source: &VideoSource) -> Result<VideoMetadata, String> {
|
||||||
pub struct VideoFrame {
|
pub struct VideoFrame {
|
||||||
pub width: u32,
|
pub width: u32,
|
||||||
pub height: u32,
|
pub height: u32,
|
||||||
|
/// CPU-decoded sRGB RGBA8 (software path). Empty when `gpu` is `Some`.
|
||||||
pub rgba_data: Arc<Vec<u8>>,
|
pub rgba_data: Arc<Vec<u8>>,
|
||||||
|
/// Hardware-decoded frame living on the GPU (NV12 plane textures). When `Some`, the compositor
|
||||||
|
/// samples it directly and `rgba_data` is empty.
|
||||||
|
pub gpu: Option<GpuVideoFrame>,
|
||||||
pub timestamp: f64,
|
pub timestamp: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A hardware-decoded video frame on the GPU: two NV12 plane textures (Y = R8, UV = RG8) imported
|
||||||
|
/// from a VAAPI DMA-BUF on the editor's shared wgpu device. The compositor samples these directly
|
||||||
|
/// (NV12→RGB), no CPU copy.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct GpuVideoFrame {
|
||||||
|
pub y: Arc<wgpu::Texture>,
|
||||||
|
pub uv: Arc<wgpu::Texture>,
|
||||||
|
pub width: u32,
|
||||||
|
pub height: u32,
|
||||||
|
/// Source YUV range: true = full/PC (0–255), false = limited/TV (16–235). Drives the NV12→RGB
|
||||||
|
/// offset/scale in the compositor.
|
||||||
|
pub full_range: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't
|
||||||
|
/// reference the GPU crate's ffmpeg-sys types) into [`GpuVideoFrame`] textures on the shared device.
|
||||||
|
/// Implemented by the editor; `gpu-video-encoder` does the actual DMA-BUF import.
|
||||||
|
pub trait HwVideoImporter: Send + Sync {
|
||||||
|
/// # Safety
|
||||||
|
/// `av_frame` must be a valid `*mut ffmpeg_sys_next::AVFrame` holding a VAAPI surface.
|
||||||
|
unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option<GpuVideoFrame>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opaque handle to the FFmpeg VAAPI hardware device (`*mut AVBufferRef`), created by the editor and
|
||||||
|
/// handed to core so decoders can attach it as `hw_device_ctx`. Core never frees it (the editor owns
|
||||||
|
/// it for the app's lifetime).
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
pub struct HwDeviceHandle(pub *mut std::ffi::c_void);
|
||||||
|
// SAFETY: the pointer is an AVBufferRef whose refcount is managed by FFmpeg; we only `av_buffer_ref`
|
||||||
|
// it (atomic) and never free it, so sharing the handle across threads is sound.
|
||||||
|
unsafe impl Send for HwDeviceHandle {}
|
||||||
|
unsafe impl Sync for HwDeviceHandle {}
|
||||||
|
|
||||||
|
/// Approximate resident bytes of a cached frame for the byte budget: the CPU RGBA buffer, or for a
|
||||||
|
/// GPU (NV12) frame ~`w*h*3/2` of VRAM, so GPU frames stay bounded too.
|
||||||
|
fn frame_cache_bytes(frame: &VideoFrame) -> usize {
|
||||||
|
if frame.gpu.is_some() {
|
||||||
|
(frame.width as usize * frame.height as usize * 3) / 2
|
||||||
|
} else {
|
||||||
|
frame.rgba_data.len()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Manages video decoders and frame caching for multiple video clips
|
/// Manages video decoders and frame caching for multiple video clips
|
||||||
pub struct VideoManager {
|
pub struct VideoManager {
|
||||||
/// Pool of video decoders, one per clip
|
/// Pool of video decoders, one per clip
|
||||||
|
|
@ -572,7 +755,7 @@ pub struct VideoManager {
|
||||||
/// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which
|
/// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which
|
||||||
/// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600)
|
/// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600)
|
||||||
/// so playback of arbitrarily long video never grows unbounded.
|
/// so playback of arbitrarily long video never grows unbounded.
|
||||||
frame_cache: LruCache<(Uuid, i64, u32, u32), Arc<VideoFrame>>,
|
frame_cache: LruCache<(Uuid, i64, u32, u32, bool), Arc<VideoFrame>>,
|
||||||
/// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len),
|
/// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len),
|
||||||
/// kept in sync on insert/evict/remove so eviction is O(1) per frame.
|
/// kept in sync on insert/evict/remove so eviction is O(1) per frame.
|
||||||
frame_cache_bytes: usize,
|
frame_cache_bytes: usize,
|
||||||
|
|
@ -588,6 +771,15 @@ pub struct VideoManager {
|
||||||
|
|
||||||
/// Maximum number of frames to cache per decoder
|
/// Maximum number of frames to cache per decoder
|
||||||
cache_size: usize,
|
cache_size: usize,
|
||||||
|
|
||||||
|
/// Hardware (VAAPI) decode, injected by the editor once the shared device is up. When set, each
|
||||||
|
/// decoder attaches the VAAPI device and imports frames as GPU textures via `hw_importer`.
|
||||||
|
hw_device: Option<HwDeviceHandle>,
|
||||||
|
hw_importer: Option<Arc<dyn HwVideoImporter>>,
|
||||||
|
/// Whether the current render pass can consume GPU textures (preview = true; export = false,
|
||||||
|
/// since it composites on a different device → a hardware decoder downloads to CPU instead).
|
||||||
|
/// Set by the render caller before each pass.
|
||||||
|
render_hardware_ok: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames).
|
/// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames).
|
||||||
|
|
@ -610,9 +802,34 @@ impl VideoManager {
|
||||||
thumbnail_cache: HashMap::new(),
|
thumbnail_cache: HashMap::new(),
|
||||||
thumbnails_complete: std::collections::HashSet::new(),
|
thumbnails_complete: std::collections::HashSet::new(),
|
||||||
cache_size,
|
cache_size,
|
||||||
|
hw_device: None,
|
||||||
|
hw_importer: None,
|
||||||
|
render_hardware_ok: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Set whether the upcoming render pass can consume GPU video textures (preview = true; export =
|
||||||
|
/// false). Call before `render_document_for_compositing`.
|
||||||
|
pub fn set_render_hardware_ok(&mut self, ok: bool) {
|
||||||
|
self.render_hardware_ok = ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Enable hardware (VAAPI) decode for all clips. Injected by the editor once the shared wgpu
|
||||||
|
/// device is active; `hw_device` is the FFmpeg VAAPI device and `importer` imports decoded
|
||||||
|
/// surfaces as GPU textures on that device. Applies to existing and future decoders. Clears the
|
||||||
|
/// frame cache (cached CPU frames would otherwise hide the new GPU frames).
|
||||||
|
pub fn set_hardware_decode(&mut self, hw_device: HwDeviceHandle, importer: Arc<dyn HwVideoImporter>) {
|
||||||
|
self.hw_device = Some(hw_device);
|
||||||
|
self.hw_importer = Some(Arc::clone(&importer));
|
||||||
|
for dec in self.decoders.values() {
|
||||||
|
if let Ok(mut d) = dec.lock() {
|
||||||
|
d.set_hardware(hw_device, Arc::clone(&importer));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.frame_cache.clear();
|
||||||
|
self.frame_cache_bytes = 0;
|
||||||
|
}
|
||||||
|
|
||||||
/// Load a video file and create a decoder for it
|
/// Load a video file and create a decoder for it
|
||||||
///
|
///
|
||||||
/// `target_width` and `target_height` specify the maximum dimensions
|
/// `target_width` and `target_height` specify the maximum dimensions
|
||||||
|
|
@ -632,7 +849,7 @@ impl VideoManager {
|
||||||
let metadata = probe_video(&source)?;
|
let metadata = probe_video(&source)?;
|
||||||
|
|
||||||
// Create decoder with target dimensions, without building keyframe index
|
// Create decoder with target dimensions, without building keyframe index
|
||||||
let decoder = VideoDecoder::new(
|
let mut decoder = VideoDecoder::new(
|
||||||
source,
|
source,
|
||||||
self.cache_size,
|
self.cache_size,
|
||||||
Some(target_width),
|
Some(target_width),
|
||||||
|
|
@ -640,6 +857,11 @@ impl VideoManager {
|
||||||
false, // Don't build keyframe index synchronously
|
false, // Don't build keyframe index synchronously
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
// Inherit hardware decode if the manager has it configured.
|
||||||
|
if let (Some(hw), Some(imp)) = (self.hw_device, &self.hw_importer) {
|
||||||
|
decoder.set_hardware(hw, Arc::clone(imp));
|
||||||
|
}
|
||||||
|
|
||||||
// Store decoder in pool
|
// Store decoder in pool
|
||||||
self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder)));
|
self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder)));
|
||||||
|
|
||||||
|
|
@ -648,14 +870,16 @@ impl VideoManager {
|
||||||
|
|
||||||
/// Get a decoded frame for a specific clip at a specific timestamp
|
/// Get a decoded frame for a specific clip at a specific timestamp
|
||||||
///
|
///
|
||||||
/// Returns None if the clip is not loaded or decoding fails.
|
/// Returns None if the clip is not loaded or decoding fails. Frames are cached.
|
||||||
/// Frames are cached for performance.
|
/// Whether a hardware decoder returns a GPU texture or downloads to CPU RGBA depends on
|
||||||
|
/// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the
|
||||||
|
/// preview, false for export, which composites on a different device).
|
||||||
pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> {
|
pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> {
|
||||||
// Convert timestamp to milliseconds for cache key. The target size is part of the key: the
|
let hardware_ok = self.render_hardware_ok;
|
||||||
// canvas (preview res) and an in-progress export (export res) request the same clip/time at
|
// The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export
|
||||||
// different sizes, and must not collide.
|
// (CPU, export res) request the same clip/time and must not collide or cross representation.
|
||||||
let timestamp_ms = (timestamp * 1000.0) as i64;
|
let timestamp_ms = (timestamp * 1000.0) as i64;
|
||||||
let cache_key = (*clip_id, timestamp_ms, target_w, target_h);
|
let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok);
|
||||||
|
|
||||||
// Check frame cache first
|
// Check frame cache first
|
||||||
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
||||||
|
|
@ -668,15 +892,25 @@ impl VideoManager {
|
||||||
let mut decoder = decoder_arc.lock().ok()?;
|
let mut decoder = decoder_arc.lock().ok()?;
|
||||||
|
|
||||||
// Decode the frame at the requested target (capped to native by the decoder).
|
// Decode the frame at the requested target (capped to native by the decoder).
|
||||||
let (rgba_data, width, height) = decoder.get_frame(timestamp, target_w, target_h).ok()?;
|
let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?;
|
||||||
drop(decoder); // release the lock before touching `self`
|
drop(decoder); // release the lock before touching `self`
|
||||||
|
|
||||||
// Create VideoFrame and cache it
|
// Create VideoFrame and cache it.
|
||||||
let frame = Arc::new(VideoFrame {
|
let frame = Arc::new(match decoded {
|
||||||
width,
|
DecodedFrame::Cpu { rgba, width, height } => VideoFrame {
|
||||||
height,
|
width,
|
||||||
rgba_data: Arc::new(rgba_data),
|
height,
|
||||||
timestamp,
|
rgba_data: Arc::new(rgba),
|
||||||
|
gpu: None,
|
||||||
|
timestamp,
|
||||||
|
},
|
||||||
|
DecodedFrame::Gpu(gpu) => VideoFrame {
|
||||||
|
width: gpu.width,
|
||||||
|
height: gpu.height,
|
||||||
|
rgba_data: Arc::new(Vec::new()),
|
||||||
|
gpu: Some(gpu),
|
||||||
|
timestamp,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
self.cache_frame(cache_key, Arc::clone(&frame));
|
self.cache_frame(cache_key, Arc::clone(&frame));
|
||||||
|
|
@ -686,16 +920,16 @@ impl VideoManager {
|
||||||
|
|
||||||
/// Insert a frame into the byte-budgeted cache, evicting least-recently-used
|
/// Insert a frame into the byte-budgeted cache, evicting least-recently-used
|
||||||
/// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`].
|
/// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`].
|
||||||
fn cache_frame(&mut self, key: (Uuid, i64, u32, u32), frame: Arc<VideoFrame>) {
|
fn cache_frame(&mut self, key: (Uuid, i64, u32, u32, bool), frame: Arc<VideoFrame>) {
|
||||||
let bytes = frame.rgba_data.len();
|
let bytes = frame_cache_bytes(&frame);
|
||||||
if let Some(old) = self.frame_cache.put(key, frame) {
|
if let Some(old) = self.frame_cache.put(key, frame) {
|
||||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len());
|
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&old));
|
||||||
}
|
}
|
||||||
self.frame_cache_bytes += bytes;
|
self.frame_cache_bytes += bytes;
|
||||||
// Keep at least one frame resident even if it alone exceeds the budget.
|
// Keep at least one frame resident even if it alone exceeds the budget.
|
||||||
while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 {
|
while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 {
|
||||||
if let Some((_, evicted)) = self.frame_cache.pop_lru() {
|
if let Some((_, evicted)) = self.frame_cache.pop_lru() {
|
||||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(evicted.rgba_data.len());
|
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&evicted));
|
||||||
} else {
|
} else {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
@ -802,15 +1036,15 @@ impl VideoManager {
|
||||||
|
|
||||||
// Remove all cached frames for this clip (LruCache has no retain; collect
|
// Remove all cached frames for this clip (LruCache has no retain; collect
|
||||||
// matching keys, then pop each, keeping the byte total in sync).
|
// matching keys, then pop each, keeping the byte total in sync).
|
||||||
let keys: Vec<(Uuid, i64, u32, u32)> = self
|
let keys: Vec<(Uuid, i64, u32, u32, bool)> = self
|
||||||
.frame_cache
|
.frame_cache
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|((id, _, _, _), _)| id == clip_id)
|
.filter(|((id, _, _, _, _), _)| id == clip_id)
|
||||||
.map(|(k, _)| *k)
|
.map(|(k, _)| *k)
|
||||||
.collect();
|
.collect();
|
||||||
for key in keys {
|
for key in keys {
|
||||||
if let Some(frame) = self.frame_cache.pop(&key) {
|
if let Some(frame) = self.frame_cache.pop(&key) {
|
||||||
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame.rgba_data.len());
|
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&frame));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue