From 097345be76af0e67f2d8024acc2af95bb4f6249c Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Wed, 17 Jun 2026 15:36:56 -0400 Subject: [PATCH] Fix video thumbnail strip bugs + persist thumbnails (resumable) Thumbnail rendering fixes: - Strip now tiles from each clip's true (unclamped) origin and draws only the tiles intersecting the visible rect, so it scrolls correctly and shows the right frames when a clip is scrolled partly off the left. Both render sites (collapsed group + expanded track) share one draw_video_thumbnail_strip helper. - On-clip strip no longer freezes on the first thumbnail: get_thumbnail_at now returns the actual thumbnail timestamp and the GPU texture cache keys on it, so tiles refresh as closer thumbnails finish generating. - Hover preview derives content time from the clip's true origin too (matches the strip when scrolled off-screen). - insert_thumbnail keeps the cache sorted + deduped (fixes a latent unsorted binary_search bug, and makes concurrent restore + resume race-safe). Thumbnail persistence (mirrors waveform persistence): - MediaKind::Thumbnail rows, keyed by thumbnail_media_id(clip_id) (clip id XOR a sentinel). Each clip's thumbnails PNG-encoded into one opaque LBTN blob (editor owns the format), snapshotted cheaply (Arc clones) and encoded off the UI thread. - Save writes the packs (kept in place on re-save); load reads them into LoadedProject.thumbnail_blobs; the editor decodes + inserts them on a background thread, so reload shows thumbnails instantly with no re-decode (even if the source video file is missing). - Partial sets are persisted with a complete flag and RESUMED on load: generate_keyframe_thumbnails takes a should_skip predicate so a save made mid-generation continues from where it left off instead of redoing the work. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lightningbeam-core/src/beam_archive.rs | 4 + .../lightningbeam-core/src/file_io.rs | 49 +++- .../lightningbeam-core/src/video.rs | 77 ++++- .../lightningbeam-editor/src/main.rs | 161 +++++++++- .../src/panes/timeline.rs | 274 +++++++++--------- 5 files changed, 407 insertions(+), 158 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs index 7f27258..ee01208 100644 --- a/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs +++ b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs @@ -55,6 +55,9 @@ pub enum MediaKind { /// A precomputed waveform LOD pyramid blob for an audio item (keyed by the /// same id as the audio it describes). See `daw_backend::audio::waveform_pyramid`. Waveform = 4, + /// A pack of precomputed video thumbnails for a video clip (keyed by a + /// sentinel-derived id from the clip id). Opaque blob; format owned by the editor. + Thumbnail = 5, } impl MediaKind { @@ -65,6 +68,7 @@ impl MediaKind { 2 => Some(Self::Raster), 3 => Some(Self::ImageAsset), 4 => Some(Self::Waveform), + 5 => Some(Self::Thumbnail), _ => None, } } diff --git a/lightningbeam-ui/lightningbeam-core/src/file_io.rs b/lightningbeam-ui/lightningbeam-core/src/file_io.rs index 71fce50..6d08d93 100644 --- a/lightningbeam-ui/lightningbeam-core/src/file_io.rs +++ b/lightningbeam-ui/lightningbeam-core/src/file_io.rs @@ -127,6 +127,11 @@ pub struct LoadedProject { /// Loaded audio pool entries pub audio_pool_entries: Vec, + /// Persisted video-thumbnail packs by clip id (opaque LBTN blobs; decoded and + /// inserted into the VideoManager by the editor). Clips present here don't need + /// their thumbnails regenerated on load. + pub thumbnail_blobs: std::collections::HashMap>, + /// List of files that couldn't be found pub missing_files: Vec, } @@ -239,12 +244,23 @@ fn waveform_media_id(pool_index: usize) -> Uuid { Uuid::from_u128(SENTINEL | (pool_index as u128)) } +/// Deterministic id for the thumbnail-pack media row of a video clip. Derived from +/// the clip id by XOR with a fixed constant — bijective and stable across saves, so +/// it reuses the row in place, and it can't (in practice) collide with the random +/// v4 ids used for other media of a different kind. +fn thumbnail_media_id(clip_id: Uuid) -> Uuid { + // "LBTN" repeated — moves clip ids into a distinct region of the id space. + const SENTINEL: u128 = 0x4C42_544E_4C42_544E_4C42_544E_4C42_544E; + Uuid::from_u128(clip_id.as_u128() ^ SENTINEL) +} + pub fn save_beam( path: &Path, document: &Document, audio_project: &mut AudioProject, audio_pool_entries: Vec, layer_to_track_map: &std::collections::HashMap, + thumbnail_blobs: &std::collections::HashMap>, _settings: &SaveSettings, ) -> Result<(), String> { let fn_start = std::time::Instant::now(); @@ -400,6 +416,19 @@ pub fn save_beam( } } + // --- video thumbnail packs -> media rows (opaque LBTN blob), keyed by a + // sentinel-derived id from the video clip id --- + for clip_id in document.video_clips.keys() { + let tn_id = thumbnail_media_id(*clip_id); + if let Some(blob) = thumbnail_blobs.get(clip_id) { + txn.put_media_packed(tn_id, MediaKind::Thumbnail, "lbtn", blob, MediaMeta::default())?; + live_media.insert(tn_id); + } else if txn.media_exists(tn_id)? { + // Not regenerated this session — keep the stored pack. + live_media.insert(tn_id); + } + } + // --- orphan cleanup: drop media for removed clips/keyframes --- let removed = txn.retain_media(&live_media)?; @@ -576,10 +605,26 @@ fn load_beam_sqlite(path: &Path) -> Result { }) .collect(); + // Persisted video thumbnail packs (opaque LBTN blobs), keyed by clip id. The + // editor decodes + inserts them and skips regeneration for these clips. + let mut thumbnail_blobs = std::collections::HashMap::new(); + for clip_id in document.video_clips.keys() { + let tn_id = thumbnail_media_id(*clip_id); + if let Ok(Some(info)) = archive.media_info(tn_id) { + if info.kind == MediaKind::Thumbnail { + match archive.read_media_full(tn_id) { + Ok(bytes) => { thumbnail_blobs.insert(*clip_id, bytes); } + Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to read thumbnails for {}: {}", clip_id, e), + } + } + } + } + eprintln!( - "📊 [LOAD_BEAM] ✅ Loaded {} audio entries, {} raster frames in {:.2}ms", + "📊 [LOAD_BEAM] ✅ Loaded {} audio entries, {} raster frames, {} thumbnail packs in {:.2}ms", restored_entries.len(), raster_load_count, + thumbnail_blobs.len(), fn_start.elapsed().as_secs_f64() * 1000.0 ); @@ -588,6 +633,7 @@ fn load_beam_sqlite(path: &Path) -> Result { audio_project, layer_to_track_map, audio_pool_entries: restored_entries, + thumbnail_blobs, missing_files, }) } @@ -815,6 +861,7 @@ fn load_beam_zip_legacy(path: &Path) -> Result { audio_project, layer_to_track_map, audio_pool_entries: restored_entries, + thumbnail_blobs: std::collections::HashMap::new(), // legacy ZIP has no thumbnail packs missing_files, }) } diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index df1469c..6182df7 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -359,6 +359,7 @@ pub fn generate_keyframe_thumbnails( path: &str, interval_secs: f64, thumb_width: u32, + mut should_skip: impl FnMut(f64) -> bool, mut on_thumb: impl FnMut(f64, Arc>), ) -> Result<(), String> { // Own decoder at thumbnail resolution; builds its own keyframe index. The @@ -383,8 +384,14 @@ pub fn generate_keyframe_thumbnails( if ks - last_emitted < interval_secs { continue; } + // This keyframe is a target slot; advance regardless of skip so the chosen + // slots are deterministic (lets a resumed pass target the same timestamps). + last_emitted = ks; + // Skip slots already covered (resume after a partial save / dedup). + if should_skip(ks) { + continue; + } if let Ok(rgba) = decoder.get_frame(ks) { - last_emitted = ks; on_thumb(ks, Arc::new(rgba)); } } @@ -471,6 +478,11 @@ pub struct VideoManager { /// Low-resolution (64px width) thumbnails for scrubbing thumbnail_cache: HashMap>)>>, + /// Clips whose thumbnail generation finished. Only complete sets are worth + /// persisting — a partial set (saved mid-generation) is dropped so the load + /// regenerates it fully rather than leaving it permanently incomplete. + thumbnails_complete: std::collections::HashSet, + /// Maximum number of frames to cache per decoder cache_size: usize, } @@ -493,6 +505,7 @@ impl VideoManager { frame_cache: LruCache::unbounded(), frame_cache_bytes: 0, thumbnail_cache: HashMap::new(), + thumbnails_complete: std::collections::HashSet::new(), cache_size, } } @@ -592,18 +605,57 @@ impl VideoManager { self.decoders.get(clip_id).cloned() } - /// Insert a thumbnail into the cache (for external thumbnail generation) - pub fn insert_thumbnail(&mut self, clip_id: &Uuid, timestamp: f64, data: Arc>) { - self.thumbnail_cache - .entry(*clip_id) - .or_insert_with(Vec::new) - .push((timestamp, data)); + /// Snapshot all cached thumbnails for persistence (clip id -> sorted + /// (timestamp, rgba) pairs). Cheap: clones the `Arc`s, not the pixel data. + /// Partial sets are persisted too — pair with [`complete_thumbnail_clips`] so + /// the load knows which clips still need generation resumed. + pub fn snapshot_all_thumbnails(&self) -> HashMap>)>> { + self.thumbnail_cache.clone() } - /// Get the thumbnail closest to the specified timestamp + /// The set of clips whose thumbnail generation has finished (a full keyframe + /// pass). A persisted set flagged incomplete is resumed on load. + pub fn complete_thumbnail_clips(&self) -> std::collections::HashSet { + self.thumbnails_complete.clone() + } + + /// Mark a clip's thumbnail generation as complete (called when the background + /// generator finishes the full keyframe pass). + pub fn mark_thumbnails_complete(&mut self, clip_id: &Uuid) { + self.thumbnails_complete.insert(*clip_id); + } + + /// Whether the clip already has a thumbnail within `tol` seconds of `ts`. + /// Lets the generator skip keyframes already covered (resume / dedup). + pub fn has_thumbnail_near(&self, clip_id: &Uuid, ts: f64, tol: f64) -> bool { + self.thumbnail_cache + .get(clip_id) + .map_or(false, |v| v.iter().any(|(t, _)| (t - ts).abs() < tol)) + } + + /// Insert a thumbnail into the cache, keeping it **sorted by timestamp** and + /// **deduped** (an existing entry at the same timestamp is replaced). Sorted + /// order is required by `get_thumbnail_at`'s binary search, and dedup makes + /// concurrent restore + resumed generation idempotent (no double inserts). + pub fn insert_thumbnail(&mut self, clip_id: &Uuid, timestamp: f64, data: Arc>) { + let vec = self.thumbnail_cache.entry(*clip_id).or_default(); + match vec.binary_search_by(|(t, _)| { + t.partial_cmp(×tamp).unwrap_or(std::cmp::Ordering::Equal) + }) { + Ok(i) => vec[i] = (timestamp, data), + Err(i) => vec.insert(i, (timestamp, data)), + } + } + + /// Get the thumbnail closest to the specified timestamp. /// + /// Returns `(actual_timestamp, width, height, data)` — `actual_timestamp` is + /// the time of the thumbnail actually chosen (which may differ from the + /// requested `timestamp`, and changes as closer thumbnails finish generating). + /// Callers key their GPU texture cache on it so the on-clip strip refreshes as + /// better thumbnails load instead of freezing on the first one. /// Returns None if no thumbnails have been generated for this clip. - pub fn get_thumbnail_at(&self, clip_id: &Uuid, timestamp: f64) -> Option<(u32, u32, Arc>)> { + pub fn get_thumbnail_at(&self, clip_id: &Uuid, timestamp: f64) -> Option<(f64, u32, u32, Arc>)> { let thumbnails = self.thumbnail_cache.get(clip_id)?; if thumbnails.is_empty() { @@ -631,14 +683,14 @@ impl VideoManager { } }); - let (_, rgba_data) = &thumbnails[idx]; + let (actual_ts, rgba_data) = &thumbnails[idx]; - // Return (width, height, data) + // Return (actual_timestamp, width, height, data) // Thumbnails are always 128px width let thumb_width = 128; let thumb_height = (rgba_data.len() / (thumb_width * 4)) as u32; - Some((thumb_width as u32, thumb_height, Arc::clone(rgba_data))) + Some((*actual_ts, thumb_width as u32, thumb_height, Arc::clone(rgba_data))) } /// Remove a video clip and its cached data @@ -661,6 +713,7 @@ impl VideoManager { // Remove thumbnails self.thumbnail_cache.remove(clip_id); + self.thumbnails_complete.remove(clip_id); } /// Clear all frame caches (useful for memory management) diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 1f56046..6623f7e 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -486,6 +486,79 @@ impl FocusIconCache { } } +/// Serialize a clip's thumbnails to an `LBTN` blob (version 2): a `complete` flag +/// then each frame PNG-encoded and packed with its timestamp. Thumbnails are 128px +/// wide; height is implied by the PNG. `complete` records whether the generating +/// pass finished — a partial pack is persisted and resumed on load. Frames that +/// fail to encode are skipped (the count reflects what's written). +fn encode_thumbnail_blob(thumbs: &[(f64, std::sync::Arc>)], complete: bool) -> Vec { + let mut entries: Vec<(f64, Vec)> = Vec::with_capacity(thumbs.len()); + for (ts, rgba) in thumbs { + let width = 128u32; + let height = (rgba.len() / (width as usize * 4)) as u32; + if height == 0 { + continue; + } + let img = lightningbeam_core::brush_engine::image_from_raw((**rgba).clone(), width, height); + if let Ok(png) = lightningbeam_core::brush_engine::encode_png(&img) { + entries.push((*ts, png)); + } + } + + let mut out = Vec::new(); + out.extend_from_slice(b"LBTN"); + out.extend_from_slice(&2u32.to_le_bytes()); // version + out.push(if complete { 1 } else { 0 }); + out.extend_from_slice(&(entries.len() as u32).to_le_bytes()); + for (ts, png) in &entries { + out.extend_from_slice(&ts.to_le_bytes()); + out.extend_from_slice(&(png.len() as u32).to_le_bytes()); + out.extend_from_slice(png); + } + out +} + +/// Read just the `complete` flag from an `LBTN` blob (cheap header read, no PNG +/// decode). Unknown/old versions are treated as incomplete (→ regenerate). +fn thumbnail_blob_is_complete(blob: &[u8]) -> bool { + blob.len() >= 9 + && &blob[0..4] == b"LBTN" + && u32::from_le_bytes(blob[4..8].try_into().unwrap()) == 2 + && blob[8] == 1 +} + +/// Decode an `LBTN` blob into `(timestamp, rgba)` thumbnails (128px wide). Returns +/// empty for unknown/old versions. +fn decode_thumbnail_blob(blob: &[u8]) -> Vec<(f64, std::sync::Arc>)> { + let mut out = Vec::new(); + if blob.len() < 13 || &blob[0..4] != b"LBTN" { + return out; + } + if u32::from_le_bytes(blob[4..8].try_into().unwrap()) != 2 { + return out; + } + // blob[8] = complete flag (read separately via thumbnail_blob_is_complete) + let count = u32::from_le_bytes(blob[9..13].try_into().unwrap()) as usize; + let mut pos = 13; + for _ in 0..count { + if pos + 12 > blob.len() { + break; + } + let ts = f64::from_le_bytes(blob[pos..pos + 8].try_into().unwrap()); + pos += 8; + let png_len = u32::from_le_bytes(blob[pos..pos + 4].try_into().unwrap()) as usize; + pos += 4; + if pos + png_len > blob.len() { + break; + } + if let Ok(img) = lightningbeam_core::brush_engine::decode_png(&blob[pos..pos + png_len]) { + out.push((ts, std::sync::Arc::new(img.into_raw()))); + } + pos += png_len; + } + out +} + /// Command sent to file operations worker thread enum FileCommand { Save { @@ -497,6 +570,12 @@ enum FileCommand { /// Serialized waveform-pyramid blobs per audio pool index, persisted into /// the container so a later load needn't re-decode the source media. waveform_blobs: std::collections::HashMap>, + /// Raw video thumbnails by clip id (cheap Arc-clone snapshot). The worker + /// PNG-encodes them off the UI thread, then persists them in the container. + thumbnail_snapshot: std::collections::HashMap>)>>, + /// Clips whose thumbnail generation has finished — partial sets are still + /// persisted (and resumed on load), but flagged incomplete. + complete_thumbnail_clips: std::collections::HashSet, progress_tx: std::sync::mpsc::Sender, }, Load { @@ -571,8 +650,8 @@ impl FileOperationsWorker { fn run(self) { while let Ok(command) = self.command_rx.recv() { match command { - FileCommand::Save { path, document, layer_to_track_map, large_media_mode, waveform_blobs, progress_tx } => { - self.handle_save(path, document, &layer_to_track_map, large_media_mode, waveform_blobs, progress_tx); + FileCommand::Save { path, document, layer_to_track_map, large_media_mode, waveform_blobs, thumbnail_snapshot, complete_thumbnail_clips, progress_tx } => { + self.handle_save(path, document, &layer_to_track_map, large_media_mode, waveform_blobs, thumbnail_snapshot, complete_thumbnail_clips, progress_tx); } FileCommand::Load { path, progress_tx } => { self.handle_load(path, progress_tx); @@ -589,10 +668,23 @@ impl FileOperationsWorker { layer_to_track_map: &std::collections::HashMap, large_media_mode: lightningbeam_core::file_io::LargeMediaMode, waveform_blobs: std::collections::HashMap>, + thumbnail_snapshot: std::collections::HashMap>)>>, + complete_thumbnail_clips: std::collections::HashSet, progress_tx: std::sync::mpsc::Sender, ) { use lightningbeam_core::file_io::{save_beam, SaveSettings}; + // PNG-encode the thumbnail snapshot into per-clip LBTN blobs (off the UI + // thread), flagging each pack complete-or-partial so the load can resume. + let thumbnail_blobs: std::collections::HashMap> = thumbnail_snapshot + .iter() + .filter(|(_, thumbs)| !thumbs.is_empty()) + .map(|(clip_id, thumbs)| { + let complete = complete_thumbnail_clips.contains(clip_id); + (*clip_id, encode_thumbnail_blob(thumbs, complete)) + }) + .collect(); + let save_start = std::time::Instant::now(); eprintln!("📊 [SAVE] Starting save operation..."); @@ -641,7 +733,7 @@ impl FileOperationsWorker { large_media_mode, ..SaveSettings::default() }; - match save_beam(&path, &document, &mut audio_project, audio_pool_entries, layer_to_track_map, &settings) { + match save_beam(&path, &document, &mut audio_project, audio_pool_entries, layer_to_track_map, &thumbnail_blobs, &settings) { Ok(()) => { eprintln!("📊 [SAVE] Step 3: save_beam() took {:.2}ms", step3_start.elapsed().as_secs_f64() * 1000.0); eprintln!("📊 [SAVE] ✅ Total save time: {:.2}ms", save_start.elapsed().as_secs_f64() * 1000.0); @@ -3716,6 +3808,14 @@ impl EditorApp { .map(|(&idx, blob)| (idx, blob.as_ref().clone())) .collect(); + // Snapshot all video thumbnails (cheap Arc-clone) + which clips finished + // generating; the worker PNG-encodes them with a complete/partial flag so a + // save mid-generation persists progress and resumes on load. + let (thumbnail_snapshot, complete_thumbnail_clips) = { + let vm = self.video_manager.lock().unwrap(); + (vm.snapshot_all_thumbnails(), vm.complete_thumbnail_clips()) + }; + // Send save command to worker thread let command = FileCommand::Save { path: path.clone(), @@ -3723,6 +3823,8 @@ impl EditorApp { layer_to_track_map: self.layer_to_track_map.clone(), large_media_mode: self.config.large_media_default, waveform_blobs, + thumbnail_snapshot, + complete_thumbnail_clips, progress_tx, }; @@ -3808,7 +3910,7 @@ impl EditorApp { } /// Apply loaded project data (called after successful load in background) - fn apply_loaded_project(&mut self, loaded_project: lightningbeam_core::file_io::LoadedProject, path: std::path::PathBuf) { + fn apply_loaded_project(&mut self, mut loaded_project: lightningbeam_core::file_io::LoadedProject, path: std::path::PathBuf) { use lightningbeam_core::action::ActionExecutor; let apply_start = std::time::Instant::now(); @@ -4059,11 +4161,39 @@ impl EditorApp { } eprintln!("📊 [APPLY] Step 8: Rebuilt MIDI event cache for {} clips in {:.2}ms", midi_fetched, step8_start.elapsed().as_secs_f64() * 1000.0); + // Restore persisted video thumbnails (decode + insert off-thread). Only + // *complete* packs are skipped from regeneration; *partial* packs are + // restored AND have generation resumed below (it skips the keyframes already + // restored), so a save mid-generation continues from where it left off. + let skip_thumbnail_clips: std::collections::HashSet = loaded_project + .thumbnail_blobs + .iter() + .filter(|(_, blob)| thumbnail_blob_is_complete(blob)) + .map(|(id, _)| *id) + .collect(); + if !loaded_project.thumbnail_blobs.is_empty() { + let blobs = std::mem::take(&mut loaded_project.thumbnail_blobs); + let video_manager = Arc::clone(&self.video_manager); + std::thread::spawn(move || { + for (clip_id, blob) in blobs { + let complete = thumbnail_blob_is_complete(&blob); + let thumbs = decode_thumbnail_blob(&blob); + let mut vm = video_manager.lock().unwrap(); + for (ts, rgba) in thumbs { + vm.insert_thumbnail(&clip_id, ts, rgba); + } + if complete { + vm.mark_thumbnails_complete(&clip_id); + } + } + }); + } + // Re-register video clips with the VideoManager (decoder + keyframe index + // thumbnails). Restored video_clips carry only metadata + file_path; without // this they render black with no thumbnails. let step9_start = std::time::Instant::now(); - self.register_loaded_videos(); + self.register_loaded_videos(&skip_thumbnail_clips); eprintln!("📊 [APPLY] Step 9: Registered loaded videos in {:.2}ms", step9_start.elapsed().as_secs_f64() * 1000.0); // Reset playback state @@ -4088,7 +4218,7 @@ impl EditorApp { /// so it can decode and display. Mirrors the import path's setup (decoder + /// background keyframe index + thumbnails) but does NOT re-extract audio — /// the extracted audio is already restored via the audio pool. - fn register_loaded_videos(&mut self) { + fn register_loaded_videos(&mut self, skip_thumbnail_clips: &std::collections::HashSet) { let doc_width = self.action_executor.document().width as u32; let doc_height = self.action_executor.document().height as u32; @@ -4117,7 +4247,11 @@ impl EditorApp { } self.spawn_keyframe_index(clip_id); - self.spawn_thumbnail_generation(clip_id); + // Skip regeneration for clips whose thumbnails were restored from the + // container — they're being decoded + inserted in the background. + if !skip_thumbnail_clips.contains(&clip_id) { + self.spawn_thumbnail_generation(clip_id); + } } } @@ -4174,17 +4308,26 @@ impl EditorApp { }; let vm_insert = Arc::clone(&video_manager); + let vm_skip = Arc::clone(&video_manager); let result = lightningbeam_core::video::generate_keyframe_thumbnails( &path, 5.0, 128, + // Resume: skip keyframes already covered (e.g. restored from a + // partial persisted pack), so generation only fills the gaps. + |ts| vm_skip.lock().unwrap().has_thumbnail_near(&clip_id, ts, 2.5), |ts, rgba| { let mut vm = vm_insert.lock().unwrap(); vm.insert_thumbnail(&clip_id, ts, rgba); }, ); - if let Err(e) = result { - eprintln!("Thumbnail generation failed for {}: {}", clip_id, e); + match result { + Ok(()) => { + // Full keyframe pass finished — mark complete so this set is + // worth persisting (a partial set is left unmarked → not saved). + vm_insert.lock().unwrap().mark_thumbnails_complete(&clip_id); + } + Err(e) => eprintln!("Thumbnail generation failed for {}: {}", clip_id, e), } }); } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index aae69ae..882c40b 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -99,6 +99,94 @@ fn clip_instance_y_bounds(row: usize, total_rows: usize) -> (f32, f32) { } } +/// Draw the strip of video thumbnails across a clip. +/// +/// Thumbnails are tiled from the clip's **true** (unclamped) screen origin +/// `clip_origin_x` over its full width `clip_true_width_px`, and only the tiles +/// intersecting `visible_rect` are drawn. This makes the strip scroll naturally +/// with the timeline and show the correct content when the clip is partly off the +/// left edge (a tile's content time is derived from its offset from the true +/// origin, not from the clamped visible edge). +/// +/// Each tile's GPU texture is cached by the **actual** thumbnail timestamp (from +/// [`VideoManager::get_thumbnail_at`]), so as closer thumbnails finish generating +/// the strip refreshes instead of freezing on whatever loaded first. +#[allow(clippy::too_many_arguments)] +fn draw_video_thumbnail_strip( + painter: &egui::Painter, + ui: &egui::Ui, + video_mgr: &lightningbeam_core::video::VideoManager, + textures: &mut std::collections::HashMap<(uuid::Uuid, i64), egui::TextureHandle>, + clip_id: uuid::Uuid, + trim_start: f64, + clip_origin_x: f32, + clip_true_width_px: f32, + visible_rect: egui::Rect, + thumb_top_y: f32, + thumb_height: f32, + pixels_per_second: f64, +) { + if clip_true_width_px <= 0.0 || thumb_height <= 0.0 { + return; + } + // Aspect ratio from any available thumbnail; sets the per-tile width. + let aspect = match video_mgr.get_thumbnail_at(&clip_id, 0.0) { + Some((_, tw, th, _)) if th > 0 => tw as f32 / th as f32, + _ => return, + }; + let step = (thumb_height * aspect).max(1.0); + let num_thumbs = ((clip_true_width_px / step).ceil() as usize).max(1); + + // Iterate only the tiles that intersect the visible rect. + let first = (((visible_rect.min.x - clip_origin_x) / step).floor() as i64).max(0) as usize; + let last = (((visible_rect.max.x - clip_origin_x) / step).ceil() as i64).max(0) as usize; + let last = last.min(num_thumbs); + + for i in first..last { + let thumb_x = clip_origin_x + i as f32 * step; + // Content time at the tile centre, measured from the TRUE clip origin so + // it's stable as the clip scrolls. + let content_time = + trim_start + (i as f64 * step as f64 + step as f64 * 0.5) / pixels_per_second; + + let Some((actual_ts, tw, th, rgba_data)) = video_mgr.get_thumbnail_at(&clip_id, content_time) + else { + continue; + }; + let ts_key = (actual_ts * 1000.0) as i64; + let texture = textures.entry((clip_id, ts_key)).or_insert_with(|| { + let image = egui::ColorImage::from_rgba_unmultiplied([tw as usize, th as usize], &rgba_data); + ui.ctx().load_texture( + format!("vthumb_{}_{}", clip_id, ts_key), + image, + egui::TextureOptions::LINEAR, + ) + }); + + let full_rect = egui::Rect::from_min_size( + egui::pos2(thumb_x, thumb_top_y), + egui::vec2(step, thumb_height), + ); + let thumb_rect = full_rect.intersect(visible_rect); + if thumb_rect.width() > 2.0 && thumb_rect.height() > 2.0 { + let uv_min = egui::pos2( + (thumb_rect.min.x - full_rect.min.x) / full_rect.width(), + (thumb_rect.min.y - full_rect.min.y) / full_rect.height(), + ); + let uv_max = egui::pos2( + (thumb_rect.max.x - full_rect.min.x) / full_rect.width(), + (thumb_rect.max.y - full_rect.min.y) / full_rect.height(), + ); + painter.image( + texture.id(), + thumb_rect, + egui::Rect::from_min_max(uv_min, uv_max), + egui::Color32::WHITE, + ); + } + } +} + /// Get the effective clip duration for a clip instance on a given layer. /// For groups on vector layers, the duration spans all consecutive keyframes /// where the group is present. For regular clips, returns the clip's internal duration. @@ -2639,12 +2727,12 @@ impl TimelinePane { video_manager: &std::sync::Arc>, audio_cache: &HashMap>, playback_time: f64, - ) -> (Vec<(egui::Rect, uuid::Uuid, f64, f64)>, Vec) { + ) -> (Vec<(egui::Rect, uuid::Uuid, f64, f32)>, Vec) { let painter = ui.painter().clone(); let mut pending_lane_renders: Vec = Vec::new(); // Collect video clip rects for hover detection (to avoid borrow conflicts) - let mut video_clip_hovers: Vec<(egui::Rect, uuid::Uuid, f64, f64)> = Vec::new(); + let mut video_clip_hovers: Vec<(egui::Rect, uuid::Uuid, f64, f32)> = Vec::new(); // Track visible video clip IDs for texture cache cleanup let mut visible_video_clip_ids: std::collections::HashSet = std::collections::HashSet::new(); @@ -2959,69 +3047,29 @@ impl TimelinePane { egui::pos2(ci_rect.min.x, span_y_min), egui::pos2(ci_rect.max.x, span_y_max), ); - video_clip_hovers.push((hover_rect, ci.clip_id, ci.trim_start, ci_start)); + // 4th elem = clip's TRUE (unclamped) origin x, for correct + // hover content time when scrolled partly off the left. + video_clip_hovers.push((hover_rect, ci.clip_id, ci.trim_start, rect.min.x + sx)); let thumb_display_height = (thumb_y_max - span_y_min) - 4.0; if thumb_display_height > 8.0 { let video_mgr = video_manager.lock().unwrap(); - if let Some((tw, th, _)) = video_mgr.get_thumbnail_at(&ci.clip_id, 0.0) { - let aspect = tw as f32 / th as f32; - let thumb_display_width = thumb_display_height * aspect; - let ci_width = ci_rect.width(); - let num_thumbs = ((ci_width / thumb_display_width).ceil() as usize).max(1); - - for ti in 0..num_thumbs { - let x_offset = ti as f32 * thumb_display_width; - if x_offset >= ci_width { break; } - - let time_offset = (x_offset as f64 + thumb_display_width as f64 * 0.5) - / self.pixels_per_second as f64; - let content_time = ci.trim_start + time_offset; - - if let Some((tw, th, rgba_data)) = video_mgr.get_thumbnail_at(&ci.clip_id, content_time) { - let ts_key = (content_time * 1000.0) as i64; - let cache_key = (ci.clip_id, ts_key); - - let texture = self.video_thumbnail_textures - .entry(cache_key) - .or_insert_with(|| { - let image = egui::ColorImage::from_rgba_unmultiplied( - [tw as usize, th as usize], - &rgba_data, - ); - ui.ctx().load_texture( - format!("vthumb_{}_{}", ci.clip_id, ts_key), - image, - egui::TextureOptions::LINEAR, - ) - }); - - let full_rect = egui::Rect::from_min_size( - egui::pos2(ci_rect.min.x + x_offset, ci_rect.min.y + 2.0), - egui::vec2(thumb_display_width, thumb_display_height), - ); - let thumb_rect = full_rect.intersect(ci_rect); - - if thumb_rect.width() > 2.0 && thumb_rect.height() > 2.0 { - let uv_min = egui::pos2( - (thumb_rect.min.x - full_rect.min.x) / full_rect.width(), - (thumb_rect.min.y - full_rect.min.y) / full_rect.height(), - ); - let uv_max = egui::pos2( - (thumb_rect.max.x - full_rect.min.x) / full_rect.width(), - (thumb_rect.max.y - full_rect.min.y) / full_rect.height(), - ); - - painter.image( - texture.id(), - thumb_rect, - egui::Rect::from_min_max(uv_min, uv_max), - egui::Color32::WHITE, - ); - } - } - } - } + // Tile from the clip's true origin (sx unclamped; ci_rect is + // the clamped visible rect) — scrolls correctly off-screen. + draw_video_thumbnail_strip( + &painter, + ui, + &video_mgr, + &mut self.video_thumbnail_textures, + ci.clip_id, + ci.trim_start, + rect.min.x + sx, + ex - sx, + ci_rect, + ci_rect.min.y + 2.0, + thumb_display_height, + self.pixels_per_second as f64, + ); } } } @@ -3818,75 +3866,31 @@ impl TimelinePane { let thumb_display_height = clip_rect.height() - 4.0; if thumb_display_height > 8.0 { let video_mgr = video_manager.lock().unwrap(); - if let Some((tw, th, _)) = video_mgr.get_thumbnail_at(&clip_instance.clip_id, 0.0) { - let aspect = tw as f32 / th as f32; - let thumb_display_width = thumb_display_height * aspect; - let thumb_step_px = thumb_display_width; - - let clip_width = clip_rect.width(); - let num_thumbs = ((clip_width / thumb_step_px).ceil() as usize).max(1); - - for i in 0..num_thumbs { - let x_offset = i as f32 * thumb_step_px; - if x_offset >= clip_width { break; } - - // Map pixel position to content time - let time_offset = (x_offset as f64 + thumb_display_width as f64 * 0.5) - / self.pixels_per_second as f64; - let content_time = clip_instance.trim_start + time_offset; - - if let Some((tw, th, rgba_data)) = video_mgr.get_thumbnail_at( - &clip_instance.clip_id, content_time - ) { - let ts_key = (content_time * 1000.0) as i64; - let cache_key = (clip_instance.clip_id, ts_key); - - let texture = self.video_thumbnail_textures - .entry(cache_key) - .or_insert_with(|| { - let image = egui::ColorImage::from_rgba_unmultiplied( - [tw as usize, th as usize], - &rgba_data, - ); - ui.ctx().load_texture( - format!("vthumb_{}_{}", clip_instance.clip_id, ts_key), - image, - egui::TextureOptions::LINEAR, - ) - }); - - let full_rect = egui::Rect::from_min_size( - egui::pos2(clip_rect.min.x + x_offset, clip_rect.min.y + 2.0), - egui::vec2(thumb_display_width, thumb_display_height), - ); - let thumb_rect = full_rect.intersect(clip_rect); - - if thumb_rect.width() > 2.0 && thumb_rect.height() > 2.0 { - let uv_min = egui::pos2( - (thumb_rect.min.x - full_rect.min.x) / full_rect.width(), - (thumb_rect.min.y - full_rect.min.y) / full_rect.height(), - ); - let uv_max = egui::pos2( - (thumb_rect.max.x - full_rect.min.x) / full_rect.width(), - (thumb_rect.max.y - full_rect.min.y) / full_rect.height(), - ); - - painter.image( - texture.id(), - thumb_rect, - egui::Rect::from_min_max(uv_min, uv_max), - egui::Color32::WHITE, - ); - } - } - } - } + // Tile from the clip's true origin (start_x is unclamped; + // clip_rect is the clamped visible rect) so the strip scrolls + // correctly and shows the right content when partly off-screen. + draw_video_thumbnail_strip( + &painter, + ui, + &video_mgr, + &mut self.video_thumbnail_textures, + clip_instance.clip_id, + clip_instance.trim_start, + rect.min.x + start_x, + end_x - start_x, + clip_rect, + clip_rect.min.y + 2.0, + thumb_display_height, + self.pixels_per_second as f64, + ); } } - // VIDEO PREVIEW: Collect clip rect for hover detection + // VIDEO PREVIEW: Collect clip rect for hover detection. Store the + // clip's TRUE (unclamped) origin x so the hover content time is + // correct even when the clip is scrolled partly off the left. if let lightningbeam_core::layer::AnyLayer::Video(_) = layer { - video_clip_hovers.push((clip_rect, clip_instance.clip_id, clip_instance.trim_start, instance_start)); + video_clip_hovers.push((clip_rect, clip_instance.clip_id, clip_instance.trim_start, rect.min.x + start_x)); } // Draw border per segment (per loop iteration for looping clips) @@ -5918,26 +5922,24 @@ impl PaneRenderer for TimelinePane { // VIDEO HOVER DETECTION: Handle video clip hover tooltips AFTER input handling // This ensures hover events aren't consumed by the main input handler - for (clip_rect, clip_id, trim_start, instance_start) in video_clip_hovers { + for (clip_rect, clip_id, trim_start, clip_origin_x) in video_clip_hovers { let hover_response = ui.allocate_rect(clip_rect, egui::Sense::hover()); if hover_response.hovered() { if let Some(hover_pos) = hover_response.hover_pos() { - // Calculate timestamp at hover position - let hover_offset_pixels = hover_pos.x - clip_rect.min.x; - let hover_offset_time = (hover_offset_pixels as f64) / (self.pixels_per_second as f64); - let hover_timestamp = instance_start + hover_offset_time; - - // Remap to clip content time accounting for trim - let clip_content_time = trim_start + (hover_timestamp - instance_start); + // Content time from the clip's TRUE origin. `clip_rect` is clamped + // to the viewport, so using its left edge mislabels the frame when + // the clip is scrolled partly off the left (same bug the strip had). + let hover_offset_pixels = hover_pos.x - clip_origin_x; + let clip_content_time = trim_start + (hover_offset_pixels as f64) / (self.pixels_per_second as f64); // Try to get thumbnail from video manager - let thumbnail_data: Option<(u32, u32, std::sync::Arc>)> = { + let thumbnail_data: Option<(f64, u32, u32, std::sync::Arc>)> = { let video_mgr = shared.video_manager.lock().unwrap(); video_mgr.get_thumbnail_at(&clip_id, clip_content_time) }; - if let Some((thumb_width, thumb_height, ref thumb_data)) = thumbnail_data { + if let Some((_, thumb_width, thumb_height, ref thumb_data)) = thumbnail_data { // Create texture from thumbnail let color_image = egui::ColorImage::from_rgba_unmultiplied( [thumb_width as usize, thumb_height as usize],