diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index c367a15..f838b23 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -221,8 +221,11 @@ fn decode_image_brush(data: &[u8]) -> Option { /// A single decoded video frame ready for GPU upload, with its document-space transform. pub struct VideoRenderInstance { - /// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). + /// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). Empty when `gpu` is `Some`. pub rgba_data: Arc>, + /// Hardware-decoded frame as GPU NV12 textures (preview path). When `Some`, the compositor + /// samples it directly (NV12→RGB) instead of uploading `rgba_data`. + pub gpu: Option, pub width: u32, pub height: u32, /// Affine transform that maps from video-pixel space to document space. @@ -609,6 +612,7 @@ pub fn render_layer_isolated( }; instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: frame.gpu.clone(), width: frame.width, height: frame.height, transform: base_transform * clip_transform * frame_to_clip, @@ -629,6 +633,7 @@ pub fn render_layer_isolated( * Affine::scale(scale); instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: None, // webcam frames are always CPU width: frame.width, height: frame.height, transform: cam_transform, @@ -1260,6 +1265,7 @@ fn render_video_layer( if let Some(ex) = extract.as_deref_mut() { ex.instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: frame.gpu.clone(), width: frame.width, height: frame.height, transform: instance_transform * brush_transform, @@ -1314,6 +1320,7 @@ fn render_video_layer( if let Some(ex) = extract.as_deref_mut() { ex.instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: None, // webcam frames are always CPU width: frame.width, height: frame.height, transform: preview_transform, diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index e2beb5e..0ddd083 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -101,6 +101,34 @@ pub struct VideoDecoder { /// Building an swscale context isn't free; a stream's frames share one input format/size and a /// consumer keeps one output size, so it's built once and rebuilt only when either changes. scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>, + /// When set (and `hw_failed` is false), decode on the GPU: attach `hw_device` as the decoder's + /// `hw_device_ctx`, decode into VAAPI surfaces, and hand each surface to `importer` to import as + /// wgpu NV12 textures (no CPU copy). `None`/failure → the software swscale path. + hw_device: Option, + importer: Option>, + /// Set if hardware decode init failed for this clip — fall back to software permanently. + hw_failed: bool, +} + +/// A decoded frame: CPU RGBA (software) or GPU NV12 textures (hardware). +enum DecodedFrame { + Cpu { rgba: Vec, width: u32, height: u32 }, + Gpu(GpuVideoFrame), +} + +/// `get_format` callback for hardware decode: select VAAPI surfaces. With `hw_device_ctx` set, +/// FFmpeg auto-allocates the frames context. +unsafe extern "C" fn get_vaapi_format( + _ctx: *mut ffmpeg::ffi::AVCodecContext, + mut fmts: *const ffmpeg::ffi::AVPixelFormat, +) -> ffmpeg::ffi::AVPixelFormat { + while *fmts != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE { + if *fmts == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI { + return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI; + } + fmts = fmts.add(1); + } + ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE } /// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only @@ -197,9 +225,28 @@ impl VideoDecoder { last_decoded_ts: -1, keyframe_positions, scaler: None, + hw_device: None, + importer: None, + hw_failed: false, }) } + /// Configure hardware (VAAPI) decode for this clip. The next decoder open attaches `hw_device` + /// and decodes into VAAPI surfaces imported by `importer`. Resets any prior decoder so the new + /// mode takes effect on the next `get_frame`. + fn set_hardware(&mut self, hw_device: HwDeviceHandle, importer: Arc) { + self.hw_device = Some(hw_device); + self.importer = Some(importer); + self.hw_failed = false; + self.decoder = None; // force a rebuild with hw_device_ctx + self.input = None; + } + + /// Whether this decoder will hardware-decode (configured + not failed). + fn hw_active(&self) -> bool { + self.hw_device.is_some() && self.importer.is_some() && !self.hw_failed + } + /// The source this decoder reads from (file path or packed container blob). pub fn source(&self) -> VideoSource { self.source.clone() @@ -226,7 +273,11 @@ impl VideoDecoder { /// Decode a frame at the specified timestamp, at native resolution (public wrapper). pub fn decode_frame(&mut self, timestamp: f64) -> Result, String> { - self.get_frame(timestamp, self.native_width, self.native_height).map(|(d, _, _)| d) + // Software-only helper; request CPU output. + match self.get_frame(timestamp, self.native_width, self.native_height, false)? { + DecodedFrame::Cpu { rgba, .. } => Ok(rgba), + DecodedFrame::Gpu(_) => Err("decode_frame: unexpected GPU frame".into()), + } } /// Build an index of all keyframe positions in the video by scanning packets @@ -268,13 +319,19 @@ impl VideoDecoder { } } - /// Get a decoded frame at the specified timestamp - /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns the - /// RGBA bytes and the actual output dimensions. - fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32) -> Result<(Vec, u32, u32), String> { + /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns GPU + /// NV12 textures when hardware-decoding and `want_gpu` (the consumer is on the shared device, + /// i.e. the preview); otherwise CPU RGBA. A hardware decoder serving a CPU consumer (export) + /// downloads the surface via `av_hwframe_transfer_data` then swscales. The `VideoManager` caches + /// the result, so the inner RGBA cache here is for CPU output only. + fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Result { use std::time::Instant; let t_start = Instant::now(); + // `hw` = decoder is opened in hardware mode (produces VAAPI surfaces). + // `gpu_out` = return GPU textures (hw + the consumer can use them). + let hw = self.hw_active(); + let gpu_out = hw && want_gpu; let (out_w, out_h) = self.capped_output(target_w, target_h); // Round timestamp to nearest frame boundary to improve cache hits @@ -286,12 +343,14 @@ impl VideoDecoder { let frame_ts = (rounded_timestamp / self.time_base) as i64; let cache_key = (frame_ts, out_w, out_h); - // Check cache - if let Some(cached_frame) = self.frame_cache.get(&cache_key) { - if video_debug() { - eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + // Check the inner RGBA cache (CPU output only; GPU frames are cached by VideoManager). + if !gpu_out { + if let Some(cached_frame) = self.frame_cache.get(&cache_key) { + if video_debug() { + eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + } + return Ok(DecodedFrame::Cpu { rgba: cached_frame.clone(), width: out_w, height: out_h }); } - return Ok((cached_frame.clone(), out_w, out_h)); } // Determine if we need to seek @@ -336,9 +395,29 @@ impl VideoDecoder { input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() ).map_err(|e| e.to_string())?; - let decoder = context_decoder.decoder().video() - .map_err(|e| e.to_string())?; - self.decoder = Some(decoder); + let mut dec_ctx = context_decoder.decoder(); + if hw { + // Attach the VAAPI device + format selector before opening so the decoder + // produces hardware surfaces. + unsafe { + let ctx = dec_ctx.as_mut_ptr(); + let hwdev = self.hw_device.unwrap().0 as *mut ffmpeg::ffi::AVBufferRef; + (*ctx).hw_device_ctx = ffmpeg::ffi::av_buffer_ref(hwdev); + (*ctx).get_format = Some(get_vaapi_format); + } + } + match dec_ctx.video() { + Ok(decoder) => self.decoder = Some(decoder), + Err(e) if hw => { + // Hardware decode unavailable for this clip — fall back to software. This + // frame fails; the next call rebuilds a software decoder. + eprintln!("[Video] hardware decode unavailable ({e}); falling back to software"); + self.hw_failed = true; + self.decoder = None; + return Err(format!("hw decode init failed: {e}")); + } + Err(e) => return Err(e.to_string()), + } } self.input = Some(owned); // Set last_decoded_ts to just before the seek target so forward playback works @@ -351,12 +430,14 @@ impl VideoDecoder { // Decode frames until we find the one closest to our target timestamp let mut best_frame_data: Option> = None; + let mut best_gpu: Option = None; let mut best_frame_ts: Option = None; let t_decode_start = Instant::now(); let mut decode_count = 0; let mut scale_time_ms = 0u128; + let mut hw_import_failed = false; - for (stream, packet) in input.packets() { + 'decode: for (stream, packet) in input.packets() { if stream.index() == self.stream_index { decoder.send_packet(&packet) .map_err(|e| e.to_string())?; @@ -376,73 +457,127 @@ impl VideoDecoder { }; if is_better { - let t_scale_start = Instant::now(); - - // Reuse the RGBA scaler across frames; rebuild only if the input - // format/size or the requested output size changes. - let need_new = match &self.scaler { - Some((fmt, w, h, ow, oh, _)) => { - *fmt != frame.format() || *w != frame.width() || *h != frame.height() - || *ow != out_w || *oh != out_h + if gpu_out { + // Hardware + GPU consumer: import the VAAPI surface as wgpu NV12 textures + // (no CPU copy). + let importer = self.importer.as_ref().unwrap(); + match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } { + Some(gpu) => { + best_gpu = Some(gpu); + best_frame_ts = Some(current_frame_ts); + } + None => { + // Import failed → fall back to software for this clip. + self.hw_failed = true; + hw_import_failed = true; + break 'decode; + } } - None => true, - }; - if need_new { - let ctx = ffmpeg::software::scaling::context::Context::get( - frame.format(), - frame.width(), - frame.height(), - ffmpeg::format::Pixel::RGBA, - out_w, - out_h, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ).map_err(|e| e.to_string())?; - self.scaler = Some((frame.format(), frame.width(), frame.height(), out_w, out_h, SendScaler(ctx))); + } else { + let t_scale_start = Instant::now(); + + // A hardware decoder produces VAAPI surfaces; a CPU consumer (export) + // downloads to system memory first, then swscales like the software path. + let downloaded; + let src: &ffmpeg::util::frame::Video = if hw { + let mut dl = ffmpeg::util::frame::Video::empty(); + let r = unsafe { + ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0) + }; + if r < 0 { + return Err(format!("av_hwframe_transfer_data failed: {r}")); + } + downloaded = dl; + &downloaded + } else { + &frame + }; + + // Reuse the RGBA scaler across frames; rebuild only if the input + // format/size or the requested output size changes. + let need_new = match &self.scaler { + Some((fmt, w, h, ow, oh, _)) => { + *fmt != src.format() || *w != src.width() || *h != src.height() + || *ow != out_w || *oh != out_h + } + None => true, + }; + if need_new { + let ctx = ffmpeg::software::scaling::context::Context::get( + src.format(), + src.width(), + src.height(), + ffmpeg::format::Pixel::RGBA, + out_w, + out_h, + ffmpeg::software::scaling::flag::Flags::BILINEAR, + ).map_err(|e| e.to_string())?; + self.scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx))); + } + let scaler = &mut self.scaler.as_mut().unwrap().5.0; + + let mut rgb_frame = ffmpeg::util::frame::Video::empty(); + scaler.run(src, &mut rgb_frame) + .map_err(|e| e.to_string())?; + + // Remove stride padding to create tightly packed RGBA data + let width = out_w as usize; + let height = out_h as usize; + let stride = rgb_frame.stride(0); + let row_size = width * 4; // RGBA = 4 bytes per pixel + let source_data = rgb_frame.data(0); + + let mut packed_data = Vec::with_capacity(row_size * height); + for y in 0..height { + let row_start = y * stride; + let row_end = row_start + row_size; + packed_data.extend_from_slice(&source_data[row_start..row_end]); + } + + scale_time_ms += t_scale_start.elapsed().as_millis(); + best_frame_data = Some(packed_data); + best_frame_ts = Some(current_frame_ts); } - let scaler = &mut self.scaler.as_mut().unwrap().5.0; - - let mut rgb_frame = ffmpeg::util::frame::Video::empty(); - scaler.run(&frame, &mut rgb_frame) - .map_err(|e| e.to_string())?; - - // Remove stride padding to create tightly packed RGBA data - let width = out_w as usize; - let height = out_h as usize; - let stride = rgb_frame.stride(0); - let row_size = width * 4; // RGBA = 4 bytes per pixel - let source_data = rgb_frame.data(0); - - let mut packed_data = Vec::with_capacity(row_size * height); - for y in 0..height { - let row_start = y * stride; - let row_end = row_start + row_size; - packed_data.extend_from_slice(&source_data[row_start..row_end]); - } - - scale_time_ms += t_scale_start.elapsed().as_millis(); - best_frame_data = Some(packed_data); - best_frame_ts = Some(current_frame_ts); } // If we've reached or passed the target timestamp, we can stop if current_frame_ts >= frame_ts { - // Found our frame, cache and return it - if let Some(data) = best_frame_data { - if video_debug() { - let total_time = t_start.elapsed().as_millis(); - let decode_time = t_decode_start.elapsed().as_millis(); - eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", - timestamp, decode_count, decode_time, scale_time_ms, total_time); - } - self.frame_cache.put(cache_key, data.clone()); - return Ok((data, out_w, out_h)); + if video_debug() { + let total_time = t_start.elapsed().as_millis(); + let decode_time = t_decode_start.elapsed().as_millis(); + eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}", + timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" }); } - break; + if gpu_out { + if let Some(gpu) = best_gpu.take() { + return Ok(DecodedFrame::Gpu(gpu)); + } + } else if let Some(data) = best_frame_data { + self.frame_cache.put(cache_key, data.clone()); + return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h }); + } + break 'decode; } } } } + // Reached EOF without hitting the target, or HW import failed mid-stream. + if hw_import_failed { + self.decoder = None; // force a software rebuild next call (decoder borrow ended here) + self.input = None; + return Err("hardware frame import failed; retrying software".to_string()); + } + // EOF: return the closest frame we found, if any. + if gpu_out { + if let Some(gpu) = best_gpu.take() { + return Ok(DecodedFrame::Gpu(gpu)); + } + } else if let Some(data) = best_frame_data { + self.frame_cache.put(cache_key, data.clone()); + return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h }); + } + eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp); Err("Failed to decode frame".to_string()) } @@ -494,7 +629,8 @@ pub fn generate_keyframe_thumbnails( continue; } // Decode at the thumbnail width (large height so width is the constraint), capped to native. - if let Ok((rgba, _, _)) = decoder.get_frame(ks, thumb_width, 100_000) { + // Thumbnail decoders are always software (no hardware importer). + if let Ok(DecodedFrame::Cpu { rgba, .. }) = decoder.get_frame(ks, thumb_width, 100_000, false) { on_thumb(ks, Arc::new(rgba)); } } @@ -559,10 +695,57 @@ pub fn probe_video(source: &VideoSource) -> Result { pub struct VideoFrame { pub width: u32, pub height: u32, + /// CPU-decoded sRGB RGBA8 (software path). Empty when `gpu` is `Some`. pub rgba_data: Arc>, + /// Hardware-decoded frame living on the GPU (NV12 plane textures). When `Some`, the compositor + /// samples it directly and `rgba_data` is empty. + pub gpu: Option, pub timestamp: f64, } +/// A hardware-decoded video frame on the GPU: two NV12 plane textures (Y = R8, UV = RG8) imported +/// from a VAAPI DMA-BUF on the editor's shared wgpu device. The compositor samples these directly +/// (NV12→RGB), no CPU copy. +#[derive(Clone, Debug)] +pub struct GpuVideoFrame { + pub y: Arc, + pub uv: Arc, + pub width: u32, + pub height: u32, + /// Source YUV range: true = full/PC (0–255), false = limited/TV (16–235). Drives the NV12→RGB + /// offset/scale in the compositor. + pub full_range: bool, +} + +/// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't +/// reference the GPU crate's ffmpeg-sys types) into [`GpuVideoFrame`] textures on the shared device. +/// Implemented by the editor; `gpu-video-encoder` does the actual DMA-BUF import. +pub trait HwVideoImporter: Send + Sync { + /// # Safety + /// `av_frame` must be a valid `*mut ffmpeg_sys_next::AVFrame` holding a VAAPI surface. + unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option; +} + +/// Opaque handle to the FFmpeg VAAPI hardware device (`*mut AVBufferRef`), created by the editor and +/// handed to core so decoders can attach it as `hw_device_ctx`. Core never frees it (the editor owns +/// it for the app's lifetime). +#[derive(Clone, Copy)] +pub struct HwDeviceHandle(pub *mut std::ffi::c_void); +// SAFETY: the pointer is an AVBufferRef whose refcount is managed by FFmpeg; we only `av_buffer_ref` +// it (atomic) and never free it, so sharing the handle across threads is sound. +unsafe impl Send for HwDeviceHandle {} +unsafe impl Sync for HwDeviceHandle {} + +/// Approximate resident bytes of a cached frame for the byte budget: the CPU RGBA buffer, or for a +/// GPU (NV12) frame ~`w*h*3/2` of VRAM, so GPU frames stay bounded too. +fn frame_cache_bytes(frame: &VideoFrame) -> usize { + if frame.gpu.is_some() { + (frame.width as usize * frame.height as usize * 3) / 2 + } else { + frame.rgba_data.len() + } +} + /// Manages video decoders and frame caching for multiple video clips pub struct VideoManager { /// Pool of video decoders, one per clip @@ -572,7 +755,7 @@ pub struct VideoManager { /// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which /// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600) /// so playback of arbitrarily long video never grows unbounded. - frame_cache: LruCache<(Uuid, i64, u32, u32), Arc>, + frame_cache: LruCache<(Uuid, i64, u32, u32, bool), Arc>, /// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len), /// kept in sync on insert/evict/remove so eviction is O(1) per frame. frame_cache_bytes: usize, @@ -588,6 +771,15 @@ pub struct VideoManager { /// Maximum number of frames to cache per decoder cache_size: usize, + + /// Hardware (VAAPI) decode, injected by the editor once the shared device is up. When set, each + /// decoder attaches the VAAPI device and imports frames as GPU textures via `hw_importer`. + hw_device: Option, + hw_importer: Option>, + /// Whether the current render pass can consume GPU textures (preview = true; export = false, + /// since it composites on a different device → a hardware decoder downloads to CPU instead). + /// Set by the render caller before each pass. + render_hardware_ok: bool, } /// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames). @@ -610,9 +802,34 @@ impl VideoManager { thumbnail_cache: HashMap::new(), thumbnails_complete: std::collections::HashSet::new(), cache_size, + hw_device: None, + hw_importer: None, + render_hardware_ok: true, } } + /// Set whether the upcoming render pass can consume GPU video textures (preview = true; export = + /// false). Call before `render_document_for_compositing`. + pub fn set_render_hardware_ok(&mut self, ok: bool) { + self.render_hardware_ok = ok; + } + + /// Enable hardware (VAAPI) decode for all clips. Injected by the editor once the shared wgpu + /// device is active; `hw_device` is the FFmpeg VAAPI device and `importer` imports decoded + /// surfaces as GPU textures on that device. Applies to existing and future decoders. Clears the + /// frame cache (cached CPU frames would otherwise hide the new GPU frames). + pub fn set_hardware_decode(&mut self, hw_device: HwDeviceHandle, importer: Arc) { + self.hw_device = Some(hw_device); + self.hw_importer = Some(Arc::clone(&importer)); + for dec in self.decoders.values() { + if let Ok(mut d) = dec.lock() { + d.set_hardware(hw_device, Arc::clone(&importer)); + } + } + self.frame_cache.clear(); + self.frame_cache_bytes = 0; + } + /// Load a video file and create a decoder for it /// /// `target_width` and `target_height` specify the maximum dimensions @@ -632,7 +849,7 @@ impl VideoManager { let metadata = probe_video(&source)?; // Create decoder with target dimensions, without building keyframe index - let decoder = VideoDecoder::new( + let mut decoder = VideoDecoder::new( source, self.cache_size, Some(target_width), @@ -640,6 +857,11 @@ impl VideoManager { false, // Don't build keyframe index synchronously )?; + // Inherit hardware decode if the manager has it configured. + if let (Some(hw), Some(imp)) = (self.hw_device, &self.hw_importer) { + decoder.set_hardware(hw, Arc::clone(imp)); + } + // Store decoder in pool self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder))); @@ -648,14 +870,16 @@ impl VideoManager { /// Get a decoded frame for a specific clip at a specific timestamp /// - /// Returns None if the clip is not loaded or decoding fails. - /// Frames are cached for performance. + /// Returns None if the clip is not loaded or decoding fails. Frames are cached. + /// Whether a hardware decoder returns a GPU texture or downloads to CPU RGBA depends on + /// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the + /// preview, false for export, which composites on a different device). pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { - // Convert timestamp to milliseconds for cache key. The target size is part of the key: the - // canvas (preview res) and an in-progress export (export res) request the same clip/time at - // different sizes, and must not collide. + let hardware_ok = self.render_hardware_ok; + // The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export + // (CPU, export res) request the same clip/time and must not collide or cross representation. let timestamp_ms = (timestamp * 1000.0) as i64; - let cache_key = (*clip_id, timestamp_ms, target_w, target_h); + let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok); // Check frame cache first if let Some(cached_frame) = self.frame_cache.get(&cache_key) { @@ -668,15 +892,25 @@ impl VideoManager { let mut decoder = decoder_arc.lock().ok()?; // Decode the frame at the requested target (capped to native by the decoder). - let (rgba_data, width, height) = decoder.get_frame(timestamp, target_w, target_h).ok()?; + let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?; drop(decoder); // release the lock before touching `self` - // Create VideoFrame and cache it - let frame = Arc::new(VideoFrame { - width, - height, - rgba_data: Arc::new(rgba_data), - timestamp, + // Create VideoFrame and cache it. + let frame = Arc::new(match decoded { + DecodedFrame::Cpu { rgba, width, height } => VideoFrame { + width, + height, + rgba_data: Arc::new(rgba), + gpu: None, + timestamp, + }, + DecodedFrame::Gpu(gpu) => VideoFrame { + width: gpu.width, + height: gpu.height, + rgba_data: Arc::new(Vec::new()), + gpu: Some(gpu), + timestamp, + }, }); self.cache_frame(cache_key, Arc::clone(&frame)); @@ -686,16 +920,16 @@ impl VideoManager { /// Insert a frame into the byte-budgeted cache, evicting least-recently-used /// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`]. - fn cache_frame(&mut self, key: (Uuid, i64, u32, u32), frame: Arc) { - let bytes = frame.rgba_data.len(); + fn cache_frame(&mut self, key: (Uuid, i64, u32, u32, bool), frame: Arc) { + let bytes = frame_cache_bytes(&frame); if let Some(old) = self.frame_cache.put(key, frame) { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&old)); } self.frame_cache_bytes += bytes; // Keep at least one frame resident even if it alone exceeds the budget. while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 { if let Some((_, evicted)) = self.frame_cache.pop_lru() { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(evicted.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&evicted)); } else { break; } @@ -802,15 +1036,15 @@ impl VideoManager { // Remove all cached frames for this clip (LruCache has no retain; collect // matching keys, then pop each, keeping the byte total in sync). - let keys: Vec<(Uuid, i64, u32, u32)> = self + let keys: Vec<(Uuid, i64, u32, u32, bool)> = self .frame_cache .iter() - .filter(|((id, _, _, _), _)| id == clip_id) + .filter(|((id, _, _, _, _), _)| id == clip_id) .map(|(k, _)| *k) .collect(); for key in keys { if let Some(frame) = self.frame_cache.pop(&key) { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&frame)); } }