skyler
/
Lightningbeam
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

video: decode at the consumer's target resolution (Stage 1) 

The decoder's output size was frozen to the document size at import, and export
reused that decoder — so exporting above document res upscaled the video (real
source detail discarded) and a document resize never re-targeted the decode.

Decode size is now chosen per get_frame call: VideoDecoder::get_frame and
VideoManager::get_frame take a target (w, h), capped to native (never upscale),
with the swscale context and frame caches keyed on the output size so preview
(preview res) and an in-progress export (export res) don't collide. The renderer
derives the target from the document->output base_transform, so export decodes
at export res (full detail) and the canvas at preview res. Thumbnails/asset
library pass small targets.
This commit is contained in:
Skyler Lehmkuhl 2026-06-26 00:36:15 -04:00
parent aa7d3a3bf4
commit f1fba186c1
3 changed files with 79 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lightningbeam-ui/lightningbeam-core/src/renderer.rs
View File

@ -366,6 +366,20 @@ pub struct CompositeRenderResult {
/// and effects in the GPU compositor. /// and effects in the GPU compositor.
/// ///
/// Layers are returned in bottom-to-top order for compositing. /// Layers are returned in bottom-to-top order for compositing.
/// Decode-target resolution for video clips = the output (export/preview) resolution, derived from
/// the document→output `base_transform` scale. The decoder caps this to the source's native size,
/// so it means "decode at the size we'll actually display, never upscaling": full detail when
/// exporting above document res (instead of upscaling a document-res frame), and cheap small frames
/// for the canvas. Stable per render pass, so it doesn't thrash the decoder's scaler/cache.
fn video_decode_target(document: &Document, base_transform: Affine) -> (u32, u32) {
let c = base_transform.as_coeffs(); // [a, b, c, d, e, f]
let sx = (c[0] * c[0] + c[1] * c[1]).sqrt();
let sy = (c[2] * c[2] + c[3] * c[3]).sqrt();
let w = (document.width * sx).ceil().max(1.0) as u32;
let h = (document.height * sy).ceil().max(1.0) as u32;
(w, h)
}
pub fn render_document_for_compositing( pub fn render_document_for_compositing(
document: &Document, document: &Document,
base_transform: Affine, base_transform: Affine,
@ -545,12 +559,13 @@ pub fn render_layer_isolated(
let layer_opacity = layer.opacity(); let layer_opacity = layer.opacity();
let mut video_mgr = video_manager.lock().unwrap(); let mut video_mgr = video_manager.lock().unwrap();
let mut instances = Vec::new(); let mut instances = Vec::new();
let (target_w, target_h) = video_decode_target(document, base_transform);
let tempo_map = document.tempo_map(); let tempo_map = document.tempo_map();
for clip_instance in &video_layer.clip_instances { for clip_instance in &video_layer.clip_instances {
let Some(video_clip) = document.video_clips.get(&clip_instance.clip_id) else { continue }; let Some(video_clip) = document.video_clips.get(&clip_instance.clip_id) else { continue };
let Some(clip_time) = clip_instance.remap_time(time, video_clip.duration, tempo_map) else { continue }; let Some(clip_time) = clip_instance.remap_time(time, video_clip.duration, tempo_map) else { continue };
let Some(frame) = video_mgr.get_frame(&clip_instance.clip_id, clip_time) else { continue }; let Some(frame) = video_mgr.get_frame(&clip_instance.clip_id, clip_time, target_w, target_h) else { continue };
// Evaluate animated transform properties. // Evaluate animated transform properties.
let anim = &video_layer.layer.animation_data; let anim = &video_layer.layer.animation_data;
@ -1095,8 +1110,9 @@ fn render_video_layer(
continue; // Clip instance not active at this time continue; // Clip instance not active at this time
}; };
// Get video frame from VideoManager // Get video frame from VideoManager at the output (export/preview) resolution.
let Some(frame) = video_manager.get_frame(&clip_instance.clip_id, clip_time) else { let (target_w, target_h) = video_decode_target(document, base_transform);
let Some(frame) = video_manager.get_frame(&clip_instance.clip_id, clip_time, target_w, target_h) else {
continue; // Frame not available continue; // Frame not available
}; };

113
lightningbeam-ui/lightningbeam-core/src/video.rs
View File

@ -84,22 +84,23 @@ pub struct VideoMetadata {
/// Video decoder with LRU frame caching /// Video decoder with LRU frame caching
pub struct VideoDecoder { pub struct VideoDecoder {
source: VideoSource, source: VideoSource,
_width: u32, // Original video width native_width: u32, // Original (decoded) video width
_height: u32, // Original video height native_height: u32, // Original (decoded) video height
output_width: u32, // Scaled output width
output_height: u32, // Scaled output height
fps: f64, fps: f64,
_duration: f64, _duration: f64,
time_base: f64, time_base: f64,
stream_index: usize, stream_index: usize,
frame_cache: LruCache<i64, Vec<u8>>, // timestamp -> RGBA data // Decoded RGBA keyed by (frame timestamp, output width, output height): the same source
// frame may be requested at different sizes (preview res vs export res).
frame_cache: LruCache<(i64, u32, u32), Vec<u8>>,
input: Option<OwnedInput>, input: Option<OwnedInput>,
decoder: Option<ffmpeg::decoder::Video>, decoder: Option<ffmpeg::decoder::Video>,
last_decoded_ts: i64, // Track the last decoded frame timestamp last_decoded_ts: i64, // Track the last decoded frame timestamp
keyframe_positions: Vec<i64>, // Index of keyframe timestamps for fast seeking keyframe_positions: Vec<i64>, // Index of keyframe timestamps for fast seeking
/// Reused RGBA scaler, keyed by the input (format, width, height). Building an swscale /// Reused RGBA scaler, keyed by `(input format, input w, input h, output w, output h)`.
/// context is not free; a stream's frames share one format/size, so build it once. /// Building an swscale context isn't free; a stream's frames share one input format/size and a
scaler: Option<(ffmpeg::format::Pixel, u32, u32, SendScaler)>, /// consumer keeps one output size, so it's built once and rebuilt only when either changes.
scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>,
} }
/// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only /// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only
@ -145,14 +146,12 @@ impl VideoDecoder {
let height = decoder.height(); let height = decoder.height();
let time_base = f64::from(video_stream.time_base()); let time_base = f64::from(video_stream.time_base());
// Calculate output dimensions (scale down if larger than max) // Output dimensions are now chosen per `get_frame` call (the caller's target res, capped to
let (output_width, output_height) = if let (Some(max_w), Some(max_h)) = (max_width, max_height) { // native) rather than frozen here — so the same clip can be decoded at preview res for the
// Calculate scale to fit within max dimensions while preserving aspect ratio // canvas and at full export res, and exporting above document res no longer upscales.
let scale = (max_w as f32 / width as f32).min(max_h as f32 / height as f32).min(1.0); // `max_width`/`max_height` are retained as an upper bound for callers that want a fixed cap
((width as f32 * scale) as u32, (height as f32 * scale) as u32) // (e.g. thumbnails pass their thumb width per call instead).
} else { let _ = (max_width, max_height);
(width, height)
};
// Try to get duration from stream, fallback to container // Try to get duration from stream, fallback to container
let duration = if video_stream.duration() > 0 { let duration = if video_stream.duration() > 0 {
@ -184,10 +183,8 @@ impl VideoDecoder {
Ok(Self { Ok(Self {
source, source,
_width: width, native_width: width,
_height: height, native_height: height,
output_width,
output_height,
fps, fps,
_duration: duration, _duration: duration,
time_base, time_base,
@ -218,19 +215,18 @@ impl VideoDecoder {
self.keyframe_positions = positions; self.keyframe_positions = positions;
} }
/// Get the output width (scaled dimensions) /// The output size for a requested target: the target capped to native resolution, preserving
pub fn get_output_width(&self) -> u32 { /// aspect ratio (never upscale beyond native — there's no detail to invent).
self.output_width fn capped_output(&self, target_w: u32, target_h: u32) -> (u32, u32) {
let (nw, nh) = (self.native_width as f32, self.native_height as f32);
if nw <= 0.0 || nh <= 0.0 { return (self.native_width.max(1), self.native_height.max(1)); }
let scale = (target_w as f32 / nw).min(target_h as f32 / nh).min(1.0);
(((nw * scale) as u32).max(1), ((nh * scale) as u32).max(1))
} }
/// Get the output height (scaled dimensions) /// Decode a frame at the specified timestamp, at native resolution (public wrapper).
pub fn get_output_height(&self) -> u32 {
self.output_height
}
/// Decode a frame at the specified timestamp (public wrapper)
pub fn decode_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> { pub fn decode_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> {
self.get_frame(timestamp) self.get_frame(timestamp, self.native_width, self.native_height).map(|(d, _, _)| d)
} }
/// Build an index of all keyframe positions in the video by scanning packets /// Build an index of all keyframe positions in the video by scanning packets
@ -273,10 +269,14 @@ impl VideoDecoder {
} }
/// Get a decoded frame at the specified timestamp /// Get a decoded frame at the specified timestamp
fn get_frame(&mut self, timestamp: f64) -> Result<Vec<u8>, String> { /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns the
/// RGBA bytes and the actual output dimensions.
fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32) -> Result<(Vec<u8>, u32, u32), String> {
use std::time::Instant; use std::time::Instant;
let t_start = Instant::now(); let t_start = Instant::now();
let (out_w, out_h) = self.capped_output(target_w, target_h);
// Round timestamp to nearest frame boundary to improve cache hits // Round timestamp to nearest frame boundary to improve cache hits
// This ensures that timestamps like 1.0001s and 0.9999s both map to frame 1.0s // This ensures that timestamps like 1.0001s and 0.9999s both map to frame 1.0s
let frame_duration = 1.0 / self.fps; let frame_duration = 1.0 / self.fps;
@ -284,13 +284,14 @@ impl VideoDecoder {
// Convert timestamp to frame timestamp // Convert timestamp to frame timestamp
let frame_ts = (rounded_timestamp / self.time_base) as i64; let frame_ts = (rounded_timestamp / self.time_base) as i64;
let cache_key = (frame_ts, out_w, out_h);
// Check cache // Check cache
if let Some(cached_frame) = self.frame_cache.get(&frame_ts) { if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
if video_debug() { if video_debug() {
eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis());
} }
return Ok(cached_frame.clone()); return Ok((cached_frame.clone(), out_w, out_h));
} }
// Determine if we need to seek // Determine if we need to seek
@ -378,10 +379,11 @@ impl VideoDecoder {
let t_scale_start = Instant::now(); let t_scale_start = Instant::now();
// Reuse the RGBA scaler across frames; rebuild only if the input // Reuse the RGBA scaler across frames; rebuild only if the input
// format/size changes (it won't within a stream). // format/size or the requested output size changes.
let need_new = match &self.scaler { let need_new = match &self.scaler {
Some((fmt, w, h, _)) => { Some((fmt, w, h, ow, oh, _)) => {
*fmt != frame.format() || *w != frame.width() || *h != frame.height() *fmt != frame.format() || *w != frame.width() || *h != frame.height()
|| *ow != out_w || *oh != out_h
} }
None => true, None => true,
}; };
@ -391,21 +393,21 @@ impl VideoDecoder {
frame.width(), frame.width(),
frame.height(), frame.height(),
ffmpeg::format::Pixel::RGBA, ffmpeg::format::Pixel::RGBA,
self.output_width, out_w,
self.output_height, out_h,
ffmpeg::software::scaling::flag::Flags::BILINEAR, ffmpeg::software::scaling::flag::Flags::BILINEAR,
).map_err(|e| e.to_string())?; ).map_err(|e| e.to_string())?;
self.scaler = Some((frame.format(), frame.width(), frame.height(), SendScaler(ctx))); self.scaler = Some((frame.format(), frame.width(), frame.height(), out_w, out_h, SendScaler(ctx)));
} }
let scaler = &mut self.scaler.as_mut().unwrap().3.0; let scaler = &mut self.scaler.as_mut().unwrap().5.0;
let mut rgb_frame = ffmpeg::util::frame::Video::empty(); let mut rgb_frame = ffmpeg::util::frame::Video::empty();
scaler.run(&frame, &mut rgb_frame) scaler.run(&frame, &mut rgb_frame)
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
// Remove stride padding to create tightly packed RGBA data // Remove stride padding to create tightly packed RGBA data
let width = self.output_width as usize; let width = out_w as usize;
let height = self.output_height as usize; let height = out_h as usize;
let stride = rgb_frame.stride(0); let stride = rgb_frame.stride(0);
let row_size = width * 4; // RGBA = 4 bytes per pixel let row_size = width * 4; // RGBA = 4 bytes per pixel
let source_data = rgb_frame.data(0); let source_data = rgb_frame.data(0);
@ -432,8 +434,8 @@ impl VideoDecoder {
eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms",
timestamp, decode_count, decode_time, scale_time_ms, total_time); timestamp, decode_count, decode_time, scale_time_ms, total_time);
} }
self.frame_cache.put(frame_ts, data.clone()); self.frame_cache.put(cache_key, data.clone());
return Ok(data); return Ok((data, out_w, out_h));
} }
break; break;
} }
@ -491,7 +493,8 @@ pub fn generate_keyframe_thumbnails(
if should_skip(ks) { if should_skip(ks) {
continue; continue;
} }
if let Ok(rgba) = decoder.get_frame(ks) { // Decode at the thumbnail width (large height so width is the constraint), capped to native.
if let Ok((rgba, _, _)) = decoder.get_frame(ks, thumb_width, 100_000) {
on_thumb(ks, Arc::new(rgba)); on_thumb(ks, Arc::new(rgba));
} }
} }
@ -569,7 +572,7 @@ pub struct VideoManager {
/// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which /// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which
/// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600) /// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600)
/// so playback of arbitrarily long video never grows unbounded. /// so playback of arbitrarily long video never grows unbounded.
frame_cache: LruCache<(Uuid, i64), Arc<VideoFrame>>, frame_cache: LruCache<(Uuid, i64, u32, u32), Arc<VideoFrame>>,
/// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len), /// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len),
/// kept in sync on insert/evict/remove so eviction is O(1) per frame. /// kept in sync on insert/evict/remove so eviction is O(1) per frame.
frame_cache_bytes: usize, frame_cache_bytes: usize,
@ -647,10 +650,12 @@ impl VideoManager {
/// ///
/// Returns None if the clip is not loaded or decoding fails. /// Returns None if the clip is not loaded or decoding fails.
/// Frames are cached for performance. /// Frames are cached for performance.
pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64) -> Option<Arc<VideoFrame>> { pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> {
// Convert timestamp to milliseconds for cache key // Convert timestamp to milliseconds for cache key. The target size is part of the key: the
// canvas (preview res) and an in-progress export (export res) request the same clip/time at
// different sizes, and must not collide.
let timestamp_ms = (timestamp * 1000.0) as i64; let timestamp_ms = (timestamp * 1000.0) as i64;
let cache_key = (*clip_id, timestamp_ms); let cache_key = (*clip_id, timestamp_ms, target_w, target_h);
// Check frame cache first // Check frame cache first
if let Some(cached_frame) = self.frame_cache.get(&cache_key) { if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
@ -662,10 +667,8 @@ impl VideoManager {
let decoder_arc = Arc::clone(self.decoders.get(clip_id)?); let decoder_arc = Arc::clone(self.decoders.get(clip_id)?);
let mut decoder = decoder_arc.lock().ok()?; let mut decoder = decoder_arc.lock().ok()?;
// Decode the frame // Decode the frame at the requested target (capped to native by the decoder).
let rgba_data = decoder.get_frame(timestamp).ok()?; let (rgba_data, width, height) = decoder.get_frame(timestamp, target_w, target_h).ok()?;
let width = decoder.output_width;
let height = decoder.output_height;
drop(decoder); // release the lock before touching `self` drop(decoder); // release the lock before touching `self`
// Create VideoFrame and cache it // Create VideoFrame and cache it
@ -683,7 +686,7 @@ impl VideoManager {
/// Insert a frame into the byte-budgeted cache, evicting least-recently-used /// Insert a frame into the byte-budgeted cache, evicting least-recently-used
/// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`]. /// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`].
fn cache_frame(&mut self, key: (Uuid, i64), frame: Arc<VideoFrame>) { fn cache_frame(&mut self, key: (Uuid, i64, u32, u32), frame: Arc<VideoFrame>) {
let bytes = frame.rgba_data.len(); let bytes = frame.rgba_data.len();
if let Some(old) = self.frame_cache.put(key, frame) { if let Some(old) = self.frame_cache.put(key, frame) {
self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len()); self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len());
@ -799,10 +802,10 @@ impl VideoManager {
// Remove all cached frames for this clip (LruCache has no retain; collect // Remove all cached frames for this clip (LruCache has no retain; collect
// matching keys, then pop each, keeping the byte total in sync). // matching keys, then pop each, keeping the byte total in sync).
let keys: Vec<(Uuid, i64)> = self let keys: Vec<(Uuid, i64, u32, u32)> = self
.frame_cache .frame_cache
.iter() .iter()
.filter(|((id, _), _)| id == clip_id) .filter(|((id, _, _, _), _)| id == clip_id)
.map(|(k, _)| *k) .map(|(k, _)| *k)
.collect(); .collect();
for key in keys { for key in keys {

3
lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs
View File

@ -316,7 +316,8 @@ fn generate_video_thumbnail(
let frame = { let frame = {
let mut video_mgr = video_manager.lock().ok()?; let mut video_mgr = video_manager.lock().ok()?;
video_mgr.get_frame(clip_id, timestamp)? // Small frame for the asset thumbnail (capped to native, aspect preserved).
video_mgr.get_frame(clip_id, timestamp, THUMBNAIL_SIZE, THUMBNAIL_SIZE)?
}; };
let src_width = frame.width as usize; let src_width = frame.width as usize;