Fix video thumbnail strip bugs + persist thumbnails (resumable)
Thumbnail rendering fixes: - Strip now tiles from each clip's true (unclamped) origin and draws only the tiles intersecting the visible rect, so it scrolls correctly and shows the right frames when a clip is scrolled partly off the left. Both render sites (collapsed group + expanded track) share one draw_video_thumbnail_strip helper. - On-clip strip no longer freezes on the first thumbnail: get_thumbnail_at now returns the actual thumbnail timestamp and the GPU texture cache keys on it, so tiles refresh as closer thumbnails finish generating. - Hover preview derives content time from the clip's true origin too (matches the strip when scrolled off-screen). - insert_thumbnail keeps the cache sorted + deduped (fixes a latent unsorted binary_search bug, and makes concurrent restore + resume race-safe). Thumbnail persistence (mirrors waveform persistence): - MediaKind::Thumbnail rows, keyed by thumbnail_media_id(clip_id) (clip id XOR a sentinel). Each clip's thumbnails PNG-encoded into one opaque LBTN blob (editor owns the format), snapshotted cheaply (Arc clones) and encoded off the UI thread. - Save writes the packs (kept in place on re-save); load reads them into LoadedProject.thumbnail_blobs; the editor decodes + inserts them on a background thread, so reload shows thumbnails instantly with no re-decode (even if the source video file is missing). - Partial sets are persisted with a complete flag and RESUMED on load: generate_keyframe_thumbnails takes a should_skip predicate so a save made mid-generation continues from where it left off instead of redoing the work. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Skyler Lehmkuhl
parent
c784816615
commit
097345be76
|
|
@ -55,6 +55,9 @@ pub enum MediaKind {
|
|||
/// A precomputed waveform LOD pyramid blob for an audio item (keyed by the
|
||||
/// same id as the audio it describes). See `daw_backend::audio::waveform_pyramid`.
|
||||
Waveform = 4,
|
||||
/// A pack of precomputed video thumbnails for a video clip (keyed by a
|
||||
/// sentinel-derived id from the clip id). Opaque blob; format owned by the editor.
|
||||
Thumbnail = 5,
|
||||
}
|
||||
|
||||
impl MediaKind {
|
||||
|
|
@ -65,6 +68,7 @@ impl MediaKind {
|
|||
2 => Some(Self::Raster),
|
||||
3 => Some(Self::ImageAsset),
|
||||
4 => Some(Self::Waveform),
|
||||
5 => Some(Self::Thumbnail),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -127,6 +127,11 @@ pub struct LoadedProject {
|
|||
/// Loaded audio pool entries
|
||||
pub audio_pool_entries: Vec<AudioPoolEntry>,
|
||||
|
||||
/// Persisted video-thumbnail packs by clip id (opaque LBTN blobs; decoded and
|
||||
/// inserted into the VideoManager by the editor). Clips present here don't need
|
||||
/// their thumbnails regenerated on load.
|
||||
pub thumbnail_blobs: std::collections::HashMap<uuid::Uuid, Vec<u8>>,
|
||||
|
||||
/// List of files that couldn't be found
|
||||
pub missing_files: Vec<MissingFileInfo>,
|
||||
}
|
||||
|
|
@ -239,12 +244,23 @@ fn waveform_media_id(pool_index: usize) -> Uuid {
|
|||
Uuid::from_u128(SENTINEL | (pool_index as u128))
|
||||
}
|
||||
|
||||
/// Deterministic id for the thumbnail-pack media row of a video clip. Derived from
|
||||
/// the clip id by XOR with a fixed constant — bijective and stable across saves, so
|
||||
/// it reuses the row in place, and it can't (in practice) collide with the random
|
||||
/// v4 ids used for other media of a different kind.
|
||||
fn thumbnail_media_id(clip_id: Uuid) -> Uuid {
|
||||
// "LBTN" repeated — moves clip ids into a distinct region of the id space.
|
||||
const SENTINEL: u128 = 0x4C42_544E_4C42_544E_4C42_544E_4C42_544E;
|
||||
Uuid::from_u128(clip_id.as_u128() ^ SENTINEL)
|
||||
}
|
||||
|
||||
pub fn save_beam(
|
||||
path: &Path,
|
||||
document: &Document,
|
||||
audio_project: &mut AudioProject,
|
||||
audio_pool_entries: Vec<AudioPoolEntry>,
|
||||
layer_to_track_map: &std::collections::HashMap<uuid::Uuid, u32>,
|
||||
thumbnail_blobs: &std::collections::HashMap<uuid::Uuid, Vec<u8>>,
|
||||
_settings: &SaveSettings,
|
||||
) -> Result<(), String> {
|
||||
let fn_start = std::time::Instant::now();
|
||||
|
|
@ -400,6 +416,19 @@ pub fn save_beam(
|
|||
}
|
||||
}
|
||||
|
||||
// --- video thumbnail packs -> media rows (opaque LBTN blob), keyed by a
|
||||
// sentinel-derived id from the video clip id ---
|
||||
for clip_id in document.video_clips.keys() {
|
||||
let tn_id = thumbnail_media_id(*clip_id);
|
||||
if let Some(blob) = thumbnail_blobs.get(clip_id) {
|
||||
txn.put_media_packed(tn_id, MediaKind::Thumbnail, "lbtn", blob, MediaMeta::default())?;
|
||||
live_media.insert(tn_id);
|
||||
} else if txn.media_exists(tn_id)? {
|
||||
// Not regenerated this session — keep the stored pack.
|
||||
live_media.insert(tn_id);
|
||||
}
|
||||
}
|
||||
|
||||
// --- orphan cleanup: drop media for removed clips/keyframes ---
|
||||
let removed = txn.retain_media(&live_media)?;
|
||||
|
||||
|
|
@ -576,10 +605,26 @@ fn load_beam_sqlite(path: &Path) -> Result<LoadedProject, String> {
|
|||
})
|
||||
.collect();
|
||||
|
||||
// Persisted video thumbnail packs (opaque LBTN blobs), keyed by clip id. The
|
||||
// editor decodes + inserts them and skips regeneration for these clips.
|
||||
let mut thumbnail_blobs = std::collections::HashMap::new();
|
||||
for clip_id in document.video_clips.keys() {
|
||||
let tn_id = thumbnail_media_id(*clip_id);
|
||||
if let Ok(Some(info)) = archive.media_info(tn_id) {
|
||||
if info.kind == MediaKind::Thumbnail {
|
||||
match archive.read_media_full(tn_id) {
|
||||
Ok(bytes) => { thumbnail_blobs.insert(*clip_id, bytes); }
|
||||
Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to read thumbnails for {}: {}", clip_id, e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!(
|
||||
"📊 [LOAD_BEAM] ✅ Loaded {} audio entries, {} raster frames in {:.2}ms",
|
||||
"📊 [LOAD_BEAM] ✅ Loaded {} audio entries, {} raster frames, {} thumbnail packs in {:.2}ms",
|
||||
restored_entries.len(),
|
||||
raster_load_count,
|
||||
thumbnail_blobs.len(),
|
||||
fn_start.elapsed().as_secs_f64() * 1000.0
|
||||
);
|
||||
|
||||
|
|
@ -588,6 +633,7 @@ fn load_beam_sqlite(path: &Path) -> Result<LoadedProject, String> {
|
|||
audio_project,
|
||||
layer_to_track_map,
|
||||
audio_pool_entries: restored_entries,
|
||||
thumbnail_blobs,
|
||||
missing_files,
|
||||
})
|
||||
}
|
||||
|
|
@ -815,6 +861,7 @@ fn load_beam_zip_legacy(path: &Path) -> Result<LoadedProject, String> {
|
|||
audio_project,
|
||||
layer_to_track_map,
|
||||
audio_pool_entries: restored_entries,
|
||||
thumbnail_blobs: std::collections::HashMap::new(), // legacy ZIP has no thumbnail packs
|
||||
missing_files,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -359,6 +359,7 @@ pub fn generate_keyframe_thumbnails(
|
|||
path: &str,
|
||||
interval_secs: f64,
|
||||
thumb_width: u32,
|
||||
mut should_skip: impl FnMut(f64) -> bool,
|
||||
mut on_thumb: impl FnMut(f64, Arc<Vec<u8>>),
|
||||
) -> Result<(), String> {
|
||||
// Own decoder at thumbnail resolution; builds its own keyframe index. The
|
||||
|
|
@ -383,8 +384,14 @@ pub fn generate_keyframe_thumbnails(
|
|||
if ks - last_emitted < interval_secs {
|
||||
continue;
|
||||
}
|
||||
// This keyframe is a target slot; advance regardless of skip so the chosen
|
||||
// slots are deterministic (lets a resumed pass target the same timestamps).
|
||||
last_emitted = ks;
|
||||
// Skip slots already covered (resume after a partial save / dedup).
|
||||
if should_skip(ks) {
|
||||
continue;
|
||||
}
|
||||
if let Ok(rgba) = decoder.get_frame(ks) {
|
||||
last_emitted = ks;
|
||||
on_thumb(ks, Arc::new(rgba));
|
||||
}
|
||||
}
|
||||
|
|
@ -471,6 +478,11 @@ pub struct VideoManager {
|
|||
/// Low-resolution (64px width) thumbnails for scrubbing
|
||||
thumbnail_cache: HashMap<Uuid, Vec<(f64, Arc<Vec<u8>>)>>,
|
||||
|
||||
/// Clips whose thumbnail generation finished. Only complete sets are worth
|
||||
/// persisting — a partial set (saved mid-generation) is dropped so the load
|
||||
/// regenerates it fully rather than leaving it permanently incomplete.
|
||||
thumbnails_complete: std::collections::HashSet<Uuid>,
|
||||
|
||||
/// Maximum number of frames to cache per decoder
|
||||
cache_size: usize,
|
||||
}
|
||||
|
|
@ -493,6 +505,7 @@ impl VideoManager {
|
|||
frame_cache: LruCache::unbounded(),
|
||||
frame_cache_bytes: 0,
|
||||
thumbnail_cache: HashMap::new(),
|
||||
thumbnails_complete: std::collections::HashSet::new(),
|
||||
cache_size,
|
||||
}
|
||||
}
|
||||
|
|
@ -592,18 +605,57 @@ impl VideoManager {
|
|||
self.decoders.get(clip_id).cloned()
|
||||
}
|
||||
|
||||
/// Insert a thumbnail into the cache (for external thumbnail generation)
|
||||
pub fn insert_thumbnail(&mut self, clip_id: &Uuid, timestamp: f64, data: Arc<Vec<u8>>) {
|
||||
self.thumbnail_cache
|
||||
.entry(*clip_id)
|
||||
.or_insert_with(Vec::new)
|
||||
.push((timestamp, data));
|
||||
/// Snapshot all cached thumbnails for persistence (clip id -> sorted
|
||||
/// (timestamp, rgba) pairs). Cheap: clones the `Arc`s, not the pixel data.
|
||||
/// Partial sets are persisted too — pair with [`complete_thumbnail_clips`] so
|
||||
/// the load knows which clips still need generation resumed.
|
||||
pub fn snapshot_all_thumbnails(&self) -> HashMap<Uuid, Vec<(f64, Arc<Vec<u8>>)>> {
|
||||
self.thumbnail_cache.clone()
|
||||
}
|
||||
|
||||
/// Get the thumbnail closest to the specified timestamp
|
||||
/// The set of clips whose thumbnail generation has finished (a full keyframe
|
||||
/// pass). A persisted set flagged incomplete is resumed on load.
|
||||
pub fn complete_thumbnail_clips(&self) -> std::collections::HashSet<Uuid> {
|
||||
self.thumbnails_complete.clone()
|
||||
}
|
||||
|
||||
/// Mark a clip's thumbnail generation as complete (called when the background
|
||||
/// generator finishes the full keyframe pass).
|
||||
pub fn mark_thumbnails_complete(&mut self, clip_id: &Uuid) {
|
||||
self.thumbnails_complete.insert(*clip_id);
|
||||
}
|
||||
|
||||
/// Whether the clip already has a thumbnail within `tol` seconds of `ts`.
|
||||
/// Lets the generator skip keyframes already covered (resume / dedup).
|
||||
pub fn has_thumbnail_near(&self, clip_id: &Uuid, ts: f64, tol: f64) -> bool {
|
||||
self.thumbnail_cache
|
||||
.get(clip_id)
|
||||
.map_or(false, |v| v.iter().any(|(t, _)| (t - ts).abs() < tol))
|
||||
}
|
||||
|
||||
/// Insert a thumbnail into the cache, keeping it **sorted by timestamp** and
|
||||
/// **deduped** (an existing entry at the same timestamp is replaced). Sorted
|
||||
/// order is required by `get_thumbnail_at`'s binary search, and dedup makes
|
||||
/// concurrent restore + resumed generation idempotent (no double inserts).
|
||||
pub fn insert_thumbnail(&mut self, clip_id: &Uuid, timestamp: f64, data: Arc<Vec<u8>>) {
|
||||
let vec = self.thumbnail_cache.entry(*clip_id).or_default();
|
||||
match vec.binary_search_by(|(t, _)| {
|
||||
t.partial_cmp(×tamp).unwrap_or(std::cmp::Ordering::Equal)
|
||||
}) {
|
||||
Ok(i) => vec[i] = (timestamp, data),
|
||||
Err(i) => vec.insert(i, (timestamp, data)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the thumbnail closest to the specified timestamp.
|
||||
///
|
||||
/// Returns `(actual_timestamp, width, height, data)` — `actual_timestamp` is
|
||||
/// the time of the thumbnail actually chosen (which may differ from the
|
||||
/// requested `timestamp`, and changes as closer thumbnails finish generating).
|
||||
/// Callers key their GPU texture cache on it so the on-clip strip refreshes as
|
||||
/// better thumbnails load instead of freezing on the first one.
|
||||
/// Returns None if no thumbnails have been generated for this clip.
|
||||
pub fn get_thumbnail_at(&self, clip_id: &Uuid, timestamp: f64) -> Option<(u32, u32, Arc<Vec<u8>>)> {
|
||||
pub fn get_thumbnail_at(&self, clip_id: &Uuid, timestamp: f64) -> Option<(f64, u32, u32, Arc<Vec<u8>>)> {
|
||||
let thumbnails = self.thumbnail_cache.get(clip_id)?;
|
||||
|
||||
if thumbnails.is_empty() {
|
||||
|
|
@ -631,14 +683,14 @@ impl VideoManager {
|
|||
}
|
||||
});
|
||||
|
||||
let (_, rgba_data) = &thumbnails[idx];
|
||||
let (actual_ts, rgba_data) = &thumbnails[idx];
|
||||
|
||||
// Return (width, height, data)
|
||||
// Return (actual_timestamp, width, height, data)
|
||||
// Thumbnails are always 128px width
|
||||
let thumb_width = 128;
|
||||
let thumb_height = (rgba_data.len() / (thumb_width * 4)) as u32;
|
||||
|
||||
Some((thumb_width as u32, thumb_height, Arc::clone(rgba_data)))
|
||||
Some((*actual_ts, thumb_width as u32, thumb_height, Arc::clone(rgba_data)))
|
||||
}
|
||||
|
||||
/// Remove a video clip and its cached data
|
||||
|
|
@ -661,6 +713,7 @@ impl VideoManager {
|
|||
|
||||
// Remove thumbnails
|
||||
self.thumbnail_cache.remove(clip_id);
|
||||
self.thumbnails_complete.remove(clip_id);
|
||||
}
|
||||
|
||||
/// Clear all frame caches (useful for memory management)
|
||||
|
|
|
|||
|
|
@ -486,6 +486,79 @@ impl FocusIconCache {
|
|||
}
|
||||
}
|
||||
|
||||
/// Serialize a clip's thumbnails to an `LBTN` blob (version 2): a `complete` flag
|
||||
/// then each frame PNG-encoded and packed with its timestamp. Thumbnails are 128px
|
||||
/// wide; height is implied by the PNG. `complete` records whether the generating
|
||||
/// pass finished — a partial pack is persisted and resumed on load. Frames that
|
||||
/// fail to encode are skipped (the count reflects what's written).
|
||||
fn encode_thumbnail_blob(thumbs: &[(f64, std::sync::Arc<Vec<u8>>)], complete: bool) -> Vec<u8> {
|
||||
let mut entries: Vec<(f64, Vec<u8>)> = Vec::with_capacity(thumbs.len());
|
||||
for (ts, rgba) in thumbs {
|
||||
let width = 128u32;
|
||||
let height = (rgba.len() / (width as usize * 4)) as u32;
|
||||
if height == 0 {
|
||||
continue;
|
||||
}
|
||||
let img = lightningbeam_core::brush_engine::image_from_raw((**rgba).clone(), width, height);
|
||||
if let Ok(png) = lightningbeam_core::brush_engine::encode_png(&img) {
|
||||
entries.push((*ts, png));
|
||||
}
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(b"LBTN");
|
||||
out.extend_from_slice(&2u32.to_le_bytes()); // version
|
||||
out.push(if complete { 1 } else { 0 });
|
||||
out.extend_from_slice(&(entries.len() as u32).to_le_bytes());
|
||||
for (ts, png) in &entries {
|
||||
out.extend_from_slice(&ts.to_le_bytes());
|
||||
out.extend_from_slice(&(png.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(png);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Read just the `complete` flag from an `LBTN` blob (cheap header read, no PNG
|
||||
/// decode). Unknown/old versions are treated as incomplete (→ regenerate).
|
||||
fn thumbnail_blob_is_complete(blob: &[u8]) -> bool {
|
||||
blob.len() >= 9
|
||||
&& &blob[0..4] == b"LBTN"
|
||||
&& u32::from_le_bytes(blob[4..8].try_into().unwrap()) == 2
|
||||
&& blob[8] == 1
|
||||
}
|
||||
|
||||
/// Decode an `LBTN` blob into `(timestamp, rgba)` thumbnails (128px wide). Returns
|
||||
/// empty for unknown/old versions.
|
||||
fn decode_thumbnail_blob(blob: &[u8]) -> Vec<(f64, std::sync::Arc<Vec<u8>>)> {
|
||||
let mut out = Vec::new();
|
||||
if blob.len() < 13 || &blob[0..4] != b"LBTN" {
|
||||
return out;
|
||||
}
|
||||
if u32::from_le_bytes(blob[4..8].try_into().unwrap()) != 2 {
|
||||
return out;
|
||||
}
|
||||
// blob[8] = complete flag (read separately via thumbnail_blob_is_complete)
|
||||
let count = u32::from_le_bytes(blob[9..13].try_into().unwrap()) as usize;
|
||||
let mut pos = 13;
|
||||
for _ in 0..count {
|
||||
if pos + 12 > blob.len() {
|
||||
break;
|
||||
}
|
||||
let ts = f64::from_le_bytes(blob[pos..pos + 8].try_into().unwrap());
|
||||
pos += 8;
|
||||
let png_len = u32::from_le_bytes(blob[pos..pos + 4].try_into().unwrap()) as usize;
|
||||
pos += 4;
|
||||
if pos + png_len > blob.len() {
|
||||
break;
|
||||
}
|
||||
if let Ok(img) = lightningbeam_core::brush_engine::decode_png(&blob[pos..pos + png_len]) {
|
||||
out.push((ts, std::sync::Arc::new(img.into_raw())));
|
||||
}
|
||||
pos += png_len;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Command sent to file operations worker thread
|
||||
enum FileCommand {
|
||||
Save {
|
||||
|
|
@ -497,6 +570,12 @@ enum FileCommand {
|
|||
/// Serialized waveform-pyramid blobs per audio pool index, persisted into
|
||||
/// the container so a later load needn't re-decode the source media.
|
||||
waveform_blobs: std::collections::HashMap<usize, Vec<u8>>,
|
||||
/// Raw video thumbnails by clip id (cheap Arc-clone snapshot). The worker
|
||||
/// PNG-encodes them off the UI thread, then persists them in the container.
|
||||
thumbnail_snapshot: std::collections::HashMap<uuid::Uuid, Vec<(f64, std::sync::Arc<Vec<u8>>)>>,
|
||||
/// Clips whose thumbnail generation has finished — partial sets are still
|
||||
/// persisted (and resumed on load), but flagged incomplete.
|
||||
complete_thumbnail_clips: std::collections::HashSet<uuid::Uuid>,
|
||||
progress_tx: std::sync::mpsc::Sender<FileProgress>,
|
||||
},
|
||||
Load {
|
||||
|
|
@ -571,8 +650,8 @@ impl FileOperationsWorker {
|
|||
fn run(self) {
|
||||
while let Ok(command) = self.command_rx.recv() {
|
||||
match command {
|
||||
FileCommand::Save { path, document, layer_to_track_map, large_media_mode, waveform_blobs, progress_tx } => {
|
||||
self.handle_save(path, document, &layer_to_track_map, large_media_mode, waveform_blobs, progress_tx);
|
||||
FileCommand::Save { path, document, layer_to_track_map, large_media_mode, waveform_blobs, thumbnail_snapshot, complete_thumbnail_clips, progress_tx } => {
|
||||
self.handle_save(path, document, &layer_to_track_map, large_media_mode, waveform_blobs, thumbnail_snapshot, complete_thumbnail_clips, progress_tx);
|
||||
}
|
||||
FileCommand::Load { path, progress_tx } => {
|
||||
self.handle_load(path, progress_tx);
|
||||
|
|
@ -589,10 +668,23 @@ impl FileOperationsWorker {
|
|||
layer_to_track_map: &std::collections::HashMap<uuid::Uuid, u32>,
|
||||
large_media_mode: lightningbeam_core::file_io::LargeMediaMode,
|
||||
waveform_blobs: std::collections::HashMap<usize, Vec<u8>>,
|
||||
thumbnail_snapshot: std::collections::HashMap<uuid::Uuid, Vec<(f64, std::sync::Arc<Vec<u8>>)>>,
|
||||
complete_thumbnail_clips: std::collections::HashSet<uuid::Uuid>,
|
||||
progress_tx: std::sync::mpsc::Sender<FileProgress>,
|
||||
) {
|
||||
use lightningbeam_core::file_io::{save_beam, SaveSettings};
|
||||
|
||||
// PNG-encode the thumbnail snapshot into per-clip LBTN blobs (off the UI
|
||||
// thread), flagging each pack complete-or-partial so the load can resume.
|
||||
let thumbnail_blobs: std::collections::HashMap<uuid::Uuid, Vec<u8>> = thumbnail_snapshot
|
||||
.iter()
|
||||
.filter(|(_, thumbs)| !thumbs.is_empty())
|
||||
.map(|(clip_id, thumbs)| {
|
||||
let complete = complete_thumbnail_clips.contains(clip_id);
|
||||
(*clip_id, encode_thumbnail_blob(thumbs, complete))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let save_start = std::time::Instant::now();
|
||||
eprintln!("📊 [SAVE] Starting save operation...");
|
||||
|
||||
|
|
@ -641,7 +733,7 @@ impl FileOperationsWorker {
|
|||
large_media_mode,
|
||||
..SaveSettings::default()
|
||||
};
|
||||
match save_beam(&path, &document, &mut audio_project, audio_pool_entries, layer_to_track_map, &settings) {
|
||||
match save_beam(&path, &document, &mut audio_project, audio_pool_entries, layer_to_track_map, &thumbnail_blobs, &settings) {
|
||||
Ok(()) => {
|
||||
eprintln!("📊 [SAVE] Step 3: save_beam() took {:.2}ms", step3_start.elapsed().as_secs_f64() * 1000.0);
|
||||
eprintln!("📊 [SAVE] ✅ Total save time: {:.2}ms", save_start.elapsed().as_secs_f64() * 1000.0);
|
||||
|
|
@ -3716,6 +3808,14 @@ impl EditorApp {
|
|||
.map(|(&idx, blob)| (idx, blob.as_ref().clone()))
|
||||
.collect();
|
||||
|
||||
// Snapshot all video thumbnails (cheap Arc-clone) + which clips finished
|
||||
// generating; the worker PNG-encodes them with a complete/partial flag so a
|
||||
// save mid-generation persists progress and resumes on load.
|
||||
let (thumbnail_snapshot, complete_thumbnail_clips) = {
|
||||
let vm = self.video_manager.lock().unwrap();
|
||||
(vm.snapshot_all_thumbnails(), vm.complete_thumbnail_clips())
|
||||
};
|
||||
|
||||
// Send save command to worker thread
|
||||
let command = FileCommand::Save {
|
||||
path: path.clone(),
|
||||
|
|
@ -3723,6 +3823,8 @@ impl EditorApp {
|
|||
layer_to_track_map: self.layer_to_track_map.clone(),
|
||||
large_media_mode: self.config.large_media_default,
|
||||
waveform_blobs,
|
||||
thumbnail_snapshot,
|
||||
complete_thumbnail_clips,
|
||||
progress_tx,
|
||||
};
|
||||
|
||||
|
|
@ -3808,7 +3910,7 @@ impl EditorApp {
|
|||
}
|
||||
|
||||
/// Apply loaded project data (called after successful load in background)
|
||||
fn apply_loaded_project(&mut self, loaded_project: lightningbeam_core::file_io::LoadedProject, path: std::path::PathBuf) {
|
||||
fn apply_loaded_project(&mut self, mut loaded_project: lightningbeam_core::file_io::LoadedProject, path: std::path::PathBuf) {
|
||||
use lightningbeam_core::action::ActionExecutor;
|
||||
|
||||
let apply_start = std::time::Instant::now();
|
||||
|
|
@ -4059,11 +4161,39 @@ impl EditorApp {
|
|||
}
|
||||
eprintln!("📊 [APPLY] Step 8: Rebuilt MIDI event cache for {} clips in {:.2}ms", midi_fetched, step8_start.elapsed().as_secs_f64() * 1000.0);
|
||||
|
||||
// Restore persisted video thumbnails (decode + insert off-thread). Only
|
||||
// *complete* packs are skipped from regeneration; *partial* packs are
|
||||
// restored AND have generation resumed below (it skips the keyframes already
|
||||
// restored), so a save mid-generation continues from where it left off.
|
||||
let skip_thumbnail_clips: std::collections::HashSet<uuid::Uuid> = loaded_project
|
||||
.thumbnail_blobs
|
||||
.iter()
|
||||
.filter(|(_, blob)| thumbnail_blob_is_complete(blob))
|
||||
.map(|(id, _)| *id)
|
||||
.collect();
|
||||
if !loaded_project.thumbnail_blobs.is_empty() {
|
||||
let blobs = std::mem::take(&mut loaded_project.thumbnail_blobs);
|
||||
let video_manager = Arc::clone(&self.video_manager);
|
||||
std::thread::spawn(move || {
|
||||
for (clip_id, blob) in blobs {
|
||||
let complete = thumbnail_blob_is_complete(&blob);
|
||||
let thumbs = decode_thumbnail_blob(&blob);
|
||||
let mut vm = video_manager.lock().unwrap();
|
||||
for (ts, rgba) in thumbs {
|
||||
vm.insert_thumbnail(&clip_id, ts, rgba);
|
||||
}
|
||||
if complete {
|
||||
vm.mark_thumbnails_complete(&clip_id);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Re-register video clips with the VideoManager (decoder + keyframe index +
|
||||
// thumbnails). Restored video_clips carry only metadata + file_path; without
|
||||
// this they render black with no thumbnails.
|
||||
let step9_start = std::time::Instant::now();
|
||||
self.register_loaded_videos();
|
||||
self.register_loaded_videos(&skip_thumbnail_clips);
|
||||
eprintln!("📊 [APPLY] Step 9: Registered loaded videos in {:.2}ms", step9_start.elapsed().as_secs_f64() * 1000.0);
|
||||
|
||||
// Reset playback state
|
||||
|
|
@ -4088,7 +4218,7 @@ impl EditorApp {
|
|||
/// so it can decode and display. Mirrors the import path's setup (decoder +
|
||||
/// background keyframe index + thumbnails) but does NOT re-extract audio —
|
||||
/// the extracted audio is already restored via the audio pool.
|
||||
fn register_loaded_videos(&mut self) {
|
||||
fn register_loaded_videos(&mut self, skip_thumbnail_clips: &std::collections::HashSet<uuid::Uuid>) {
|
||||
let doc_width = self.action_executor.document().width as u32;
|
||||
let doc_height = self.action_executor.document().height as u32;
|
||||
|
||||
|
|
@ -4117,7 +4247,11 @@ impl EditorApp {
|
|||
}
|
||||
|
||||
self.spawn_keyframe_index(clip_id);
|
||||
self.spawn_thumbnail_generation(clip_id);
|
||||
// Skip regeneration for clips whose thumbnails were restored from the
|
||||
// container — they're being decoded + inserted in the background.
|
||||
if !skip_thumbnail_clips.contains(&clip_id) {
|
||||
self.spawn_thumbnail_generation(clip_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4174,17 +4308,26 @@ impl EditorApp {
|
|||
};
|
||||
|
||||
let vm_insert = Arc::clone(&video_manager);
|
||||
let vm_skip = Arc::clone(&video_manager);
|
||||
let result = lightningbeam_core::video::generate_keyframe_thumbnails(
|
||||
&path,
|
||||
5.0,
|
||||
128,
|
||||
// Resume: skip keyframes already covered (e.g. restored from a
|
||||
// partial persisted pack), so generation only fills the gaps.
|
||||
|ts| vm_skip.lock().unwrap().has_thumbnail_near(&clip_id, ts, 2.5),
|
||||
|ts, rgba| {
|
||||
let mut vm = vm_insert.lock().unwrap();
|
||||
vm.insert_thumbnail(&clip_id, ts, rgba);
|
||||
},
|
||||
);
|
||||
if let Err(e) = result {
|
||||
eprintln!("Thumbnail generation failed for {}: {}", clip_id, e);
|
||||
match result {
|
||||
Ok(()) => {
|
||||
// Full keyframe pass finished — mark complete so this set is
|
||||
// worth persisting (a partial set is left unmarked → not saved).
|
||||
vm_insert.lock().unwrap().mark_thumbnails_complete(&clip_id);
|
||||
}
|
||||
Err(e) => eprintln!("Thumbnail generation failed for {}: {}", clip_id, e),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,6 +99,94 @@ fn clip_instance_y_bounds(row: usize, total_rows: usize) -> (f32, f32) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Draw the strip of video thumbnails across a clip.
|
||||
///
|
||||
/// Thumbnails are tiled from the clip's **true** (unclamped) screen origin
|
||||
/// `clip_origin_x` over its full width `clip_true_width_px`, and only the tiles
|
||||
/// intersecting `visible_rect` are drawn. This makes the strip scroll naturally
|
||||
/// with the timeline and show the correct content when the clip is partly off the
|
||||
/// left edge (a tile's content time is derived from its offset from the true
|
||||
/// origin, not from the clamped visible edge).
|
||||
///
|
||||
/// Each tile's GPU texture is cached by the **actual** thumbnail timestamp (from
|
||||
/// [`VideoManager::get_thumbnail_at`]), so as closer thumbnails finish generating
|
||||
/// the strip refreshes instead of freezing on whatever loaded first.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn draw_video_thumbnail_strip(
|
||||
painter: &egui::Painter,
|
||||
ui: &egui::Ui,
|
||||
video_mgr: &lightningbeam_core::video::VideoManager,
|
||||
textures: &mut std::collections::HashMap<(uuid::Uuid, i64), egui::TextureHandle>,
|
||||
clip_id: uuid::Uuid,
|
||||
trim_start: f64,
|
||||
clip_origin_x: f32,
|
||||
clip_true_width_px: f32,
|
||||
visible_rect: egui::Rect,
|
||||
thumb_top_y: f32,
|
||||
thumb_height: f32,
|
||||
pixels_per_second: f64,
|
||||
) {
|
||||
if clip_true_width_px <= 0.0 || thumb_height <= 0.0 {
|
||||
return;
|
||||
}
|
||||
// Aspect ratio from any available thumbnail; sets the per-tile width.
|
||||
let aspect = match video_mgr.get_thumbnail_at(&clip_id, 0.0) {
|
||||
Some((_, tw, th, _)) if th > 0 => tw as f32 / th as f32,
|
||||
_ => return,
|
||||
};
|
||||
let step = (thumb_height * aspect).max(1.0);
|
||||
let num_thumbs = ((clip_true_width_px / step).ceil() as usize).max(1);
|
||||
|
||||
// Iterate only the tiles that intersect the visible rect.
|
||||
let first = (((visible_rect.min.x - clip_origin_x) / step).floor() as i64).max(0) as usize;
|
||||
let last = (((visible_rect.max.x - clip_origin_x) / step).ceil() as i64).max(0) as usize;
|
||||
let last = last.min(num_thumbs);
|
||||
|
||||
for i in first..last {
|
||||
let thumb_x = clip_origin_x + i as f32 * step;
|
||||
// Content time at the tile centre, measured from the TRUE clip origin so
|
||||
// it's stable as the clip scrolls.
|
||||
let content_time =
|
||||
trim_start + (i as f64 * step as f64 + step as f64 * 0.5) / pixels_per_second;
|
||||
|
||||
let Some((actual_ts, tw, th, rgba_data)) = video_mgr.get_thumbnail_at(&clip_id, content_time)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let ts_key = (actual_ts * 1000.0) as i64;
|
||||
let texture = textures.entry((clip_id, ts_key)).or_insert_with(|| {
|
||||
let image = egui::ColorImage::from_rgba_unmultiplied([tw as usize, th as usize], &rgba_data);
|
||||
ui.ctx().load_texture(
|
||||
format!("vthumb_{}_{}", clip_id, ts_key),
|
||||
image,
|
||||
egui::TextureOptions::LINEAR,
|
||||
)
|
||||
});
|
||||
|
||||
let full_rect = egui::Rect::from_min_size(
|
||||
egui::pos2(thumb_x, thumb_top_y),
|
||||
egui::vec2(step, thumb_height),
|
||||
);
|
||||
let thumb_rect = full_rect.intersect(visible_rect);
|
||||
if thumb_rect.width() > 2.0 && thumb_rect.height() > 2.0 {
|
||||
let uv_min = egui::pos2(
|
||||
(thumb_rect.min.x - full_rect.min.x) / full_rect.width(),
|
||||
(thumb_rect.min.y - full_rect.min.y) / full_rect.height(),
|
||||
);
|
||||
let uv_max = egui::pos2(
|
||||
(thumb_rect.max.x - full_rect.min.x) / full_rect.width(),
|
||||
(thumb_rect.max.y - full_rect.min.y) / full_rect.height(),
|
||||
);
|
||||
painter.image(
|
||||
texture.id(),
|
||||
thumb_rect,
|
||||
egui::Rect::from_min_max(uv_min, uv_max),
|
||||
egui::Color32::WHITE,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the effective clip duration for a clip instance on a given layer.
|
||||
/// For groups on vector layers, the duration spans all consecutive keyframes
|
||||
/// where the group is present. For regular clips, returns the clip's internal duration.
|
||||
|
|
@ -2639,12 +2727,12 @@ impl TimelinePane {
|
|||
video_manager: &std::sync::Arc<std::sync::Mutex<lightningbeam_core::video::VideoManager>>,
|
||||
audio_cache: &HashMap<uuid::Uuid, Vec<ClipInstance>>,
|
||||
playback_time: f64,
|
||||
) -> (Vec<(egui::Rect, uuid::Uuid, f64, f64)>, Vec<AutomationLaneRender>) {
|
||||
) -> (Vec<(egui::Rect, uuid::Uuid, f64, f32)>, Vec<AutomationLaneRender>) {
|
||||
let painter = ui.painter().clone();
|
||||
let mut pending_lane_renders: Vec<AutomationLaneRender> = Vec::new();
|
||||
|
||||
// Collect video clip rects for hover detection (to avoid borrow conflicts)
|
||||
let mut video_clip_hovers: Vec<(egui::Rect, uuid::Uuid, f64, f64)> = Vec::new();
|
||||
let mut video_clip_hovers: Vec<(egui::Rect, uuid::Uuid, f64, f32)> = Vec::new();
|
||||
|
||||
// Track visible video clip IDs for texture cache cleanup
|
||||
let mut visible_video_clip_ids: std::collections::HashSet<uuid::Uuid> = std::collections::HashSet::new();
|
||||
|
|
@ -2959,69 +3047,29 @@ impl TimelinePane {
|
|||
egui::pos2(ci_rect.min.x, span_y_min),
|
||||
egui::pos2(ci_rect.max.x, span_y_max),
|
||||
);
|
||||
video_clip_hovers.push((hover_rect, ci.clip_id, ci.trim_start, ci_start));
|
||||
// 4th elem = clip's TRUE (unclamped) origin x, for correct
|
||||
// hover content time when scrolled partly off the left.
|
||||
video_clip_hovers.push((hover_rect, ci.clip_id, ci.trim_start, rect.min.x + sx));
|
||||
|
||||
let thumb_display_height = (thumb_y_max - span_y_min) - 4.0;
|
||||
if thumb_display_height > 8.0 {
|
||||
let video_mgr = video_manager.lock().unwrap();
|
||||
if let Some((tw, th, _)) = video_mgr.get_thumbnail_at(&ci.clip_id, 0.0) {
|
||||
let aspect = tw as f32 / th as f32;
|
||||
let thumb_display_width = thumb_display_height * aspect;
|
||||
let ci_width = ci_rect.width();
|
||||
let num_thumbs = ((ci_width / thumb_display_width).ceil() as usize).max(1);
|
||||
|
||||
for ti in 0..num_thumbs {
|
||||
let x_offset = ti as f32 * thumb_display_width;
|
||||
if x_offset >= ci_width { break; }
|
||||
|
||||
let time_offset = (x_offset as f64 + thumb_display_width as f64 * 0.5)
|
||||
/ self.pixels_per_second as f64;
|
||||
let content_time = ci.trim_start + time_offset;
|
||||
|
||||
if let Some((tw, th, rgba_data)) = video_mgr.get_thumbnail_at(&ci.clip_id, content_time) {
|
||||
let ts_key = (content_time * 1000.0) as i64;
|
||||
let cache_key = (ci.clip_id, ts_key);
|
||||
|
||||
let texture = self.video_thumbnail_textures
|
||||
.entry(cache_key)
|
||||
.or_insert_with(|| {
|
||||
let image = egui::ColorImage::from_rgba_unmultiplied(
|
||||
[tw as usize, th as usize],
|
||||
&rgba_data,
|
||||
);
|
||||
ui.ctx().load_texture(
|
||||
format!("vthumb_{}_{}", ci.clip_id, ts_key),
|
||||
image,
|
||||
egui::TextureOptions::LINEAR,
|
||||
)
|
||||
});
|
||||
|
||||
let full_rect = egui::Rect::from_min_size(
|
||||
egui::pos2(ci_rect.min.x + x_offset, ci_rect.min.y + 2.0),
|
||||
egui::vec2(thumb_display_width, thumb_display_height),
|
||||
);
|
||||
let thumb_rect = full_rect.intersect(ci_rect);
|
||||
|
||||
if thumb_rect.width() > 2.0 && thumb_rect.height() > 2.0 {
|
||||
let uv_min = egui::pos2(
|
||||
(thumb_rect.min.x - full_rect.min.x) / full_rect.width(),
|
||||
(thumb_rect.min.y - full_rect.min.y) / full_rect.height(),
|
||||
);
|
||||
let uv_max = egui::pos2(
|
||||
(thumb_rect.max.x - full_rect.min.x) / full_rect.width(),
|
||||
(thumb_rect.max.y - full_rect.min.y) / full_rect.height(),
|
||||
);
|
||||
|
||||
painter.image(
|
||||
texture.id(),
|
||||
thumb_rect,
|
||||
egui::Rect::from_min_max(uv_min, uv_max),
|
||||
egui::Color32::WHITE,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Tile from the clip's true origin (sx unclamped; ci_rect is
|
||||
// the clamped visible rect) — scrolls correctly off-screen.
|
||||
draw_video_thumbnail_strip(
|
||||
&painter,
|
||||
ui,
|
||||
&video_mgr,
|
||||
&mut self.video_thumbnail_textures,
|
||||
ci.clip_id,
|
||||
ci.trim_start,
|
||||
rect.min.x + sx,
|
||||
ex - sx,
|
||||
ci_rect,
|
||||
ci_rect.min.y + 2.0,
|
||||
thumb_display_height,
|
||||
self.pixels_per_second as f64,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3818,75 +3866,31 @@ impl TimelinePane {
|
|||
let thumb_display_height = clip_rect.height() - 4.0;
|
||||
if thumb_display_height > 8.0 {
|
||||
let video_mgr = video_manager.lock().unwrap();
|
||||
if let Some((tw, th, _)) = video_mgr.get_thumbnail_at(&clip_instance.clip_id, 0.0) {
|
||||
let aspect = tw as f32 / th as f32;
|
||||
let thumb_display_width = thumb_display_height * aspect;
|
||||
let thumb_step_px = thumb_display_width;
|
||||
|
||||
let clip_width = clip_rect.width();
|
||||
let num_thumbs = ((clip_width / thumb_step_px).ceil() as usize).max(1);
|
||||
|
||||
for i in 0..num_thumbs {
|
||||
let x_offset = i as f32 * thumb_step_px;
|
||||
if x_offset >= clip_width { break; }
|
||||
|
||||
// Map pixel position to content time
|
||||
let time_offset = (x_offset as f64 + thumb_display_width as f64 * 0.5)
|
||||
/ self.pixels_per_second as f64;
|
||||
let content_time = clip_instance.trim_start + time_offset;
|
||||
|
||||
if let Some((tw, th, rgba_data)) = video_mgr.get_thumbnail_at(
|
||||
&clip_instance.clip_id, content_time
|
||||
) {
|
||||
let ts_key = (content_time * 1000.0) as i64;
|
||||
let cache_key = (clip_instance.clip_id, ts_key);
|
||||
|
||||
let texture = self.video_thumbnail_textures
|
||||
.entry(cache_key)
|
||||
.or_insert_with(|| {
|
||||
let image = egui::ColorImage::from_rgba_unmultiplied(
|
||||
[tw as usize, th as usize],
|
||||
&rgba_data,
|
||||
);
|
||||
ui.ctx().load_texture(
|
||||
format!("vthumb_{}_{}", clip_instance.clip_id, ts_key),
|
||||
image,
|
||||
egui::TextureOptions::LINEAR,
|
||||
)
|
||||
});
|
||||
|
||||
let full_rect = egui::Rect::from_min_size(
|
||||
egui::pos2(clip_rect.min.x + x_offset, clip_rect.min.y + 2.0),
|
||||
egui::vec2(thumb_display_width, thumb_display_height),
|
||||
);
|
||||
let thumb_rect = full_rect.intersect(clip_rect);
|
||||
|
||||
if thumb_rect.width() > 2.0 && thumb_rect.height() > 2.0 {
|
||||
let uv_min = egui::pos2(
|
||||
(thumb_rect.min.x - full_rect.min.x) / full_rect.width(),
|
||||
(thumb_rect.min.y - full_rect.min.y) / full_rect.height(),
|
||||
);
|
||||
let uv_max = egui::pos2(
|
||||
(thumb_rect.max.x - full_rect.min.x) / full_rect.width(),
|
||||
(thumb_rect.max.y - full_rect.min.y) / full_rect.height(),
|
||||
);
|
||||
|
||||
painter.image(
|
||||
texture.id(),
|
||||
thumb_rect,
|
||||
egui::Rect::from_min_max(uv_min, uv_max),
|
||||
egui::Color32::WHITE,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Tile from the clip's true origin (start_x is unclamped;
|
||||
// clip_rect is the clamped visible rect) so the strip scrolls
|
||||
// correctly and shows the right content when partly off-screen.
|
||||
draw_video_thumbnail_strip(
|
||||
&painter,
|
||||
ui,
|
||||
&video_mgr,
|
||||
&mut self.video_thumbnail_textures,
|
||||
clip_instance.clip_id,
|
||||
clip_instance.trim_start,
|
||||
rect.min.x + start_x,
|
||||
end_x - start_x,
|
||||
clip_rect,
|
||||
clip_rect.min.y + 2.0,
|
||||
thumb_display_height,
|
||||
self.pixels_per_second as f64,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// VIDEO PREVIEW: Collect clip rect for hover detection
|
||||
// VIDEO PREVIEW: Collect clip rect for hover detection. Store the
|
||||
// clip's TRUE (unclamped) origin x so the hover content time is
|
||||
// correct even when the clip is scrolled partly off the left.
|
||||
if let lightningbeam_core::layer::AnyLayer::Video(_) = layer {
|
||||
video_clip_hovers.push((clip_rect, clip_instance.clip_id, clip_instance.trim_start, instance_start));
|
||||
video_clip_hovers.push((clip_rect, clip_instance.clip_id, clip_instance.trim_start, rect.min.x + start_x));
|
||||
}
|
||||
|
||||
// Draw border per segment (per loop iteration for looping clips)
|
||||
|
|
@ -5918,26 +5922,24 @@ impl PaneRenderer for TimelinePane {
|
|||
|
||||
// VIDEO HOVER DETECTION: Handle video clip hover tooltips AFTER input handling
|
||||
// This ensures hover events aren't consumed by the main input handler
|
||||
for (clip_rect, clip_id, trim_start, instance_start) in video_clip_hovers {
|
||||
for (clip_rect, clip_id, trim_start, clip_origin_x) in video_clip_hovers {
|
||||
let hover_response = ui.allocate_rect(clip_rect, egui::Sense::hover());
|
||||
|
||||
if hover_response.hovered() {
|
||||
if let Some(hover_pos) = hover_response.hover_pos() {
|
||||
// Calculate timestamp at hover position
|
||||
let hover_offset_pixels = hover_pos.x - clip_rect.min.x;
|
||||
let hover_offset_time = (hover_offset_pixels as f64) / (self.pixels_per_second as f64);
|
||||
let hover_timestamp = instance_start + hover_offset_time;
|
||||
|
||||
// Remap to clip content time accounting for trim
|
||||
let clip_content_time = trim_start + (hover_timestamp - instance_start);
|
||||
// Content time from the clip's TRUE origin. `clip_rect` is clamped
|
||||
// to the viewport, so using its left edge mislabels the frame when
|
||||
// the clip is scrolled partly off the left (same bug the strip had).
|
||||
let hover_offset_pixels = hover_pos.x - clip_origin_x;
|
||||
let clip_content_time = trim_start + (hover_offset_pixels as f64) / (self.pixels_per_second as f64);
|
||||
|
||||
// Try to get thumbnail from video manager
|
||||
let thumbnail_data: Option<(u32, u32, std::sync::Arc<Vec<u8>>)> = {
|
||||
let thumbnail_data: Option<(f64, u32, u32, std::sync::Arc<Vec<u8>>)> = {
|
||||
let video_mgr = shared.video_manager.lock().unwrap();
|
||||
video_mgr.get_thumbnail_at(&clip_id, clip_content_time)
|
||||
};
|
||||
|
||||
if let Some((thumb_width, thumb_height, ref thumb_data)) = thumbnail_data {
|
||||
if let Some((_, thumb_width, thumb_height, ref thumb_data)) = thumbnail_data {
|
||||
// Create texture from thumbnail
|
||||
let color_image = egui::ColorImage::from_rgba_unmultiplied(
|
||||
[thumb_width as usize, thumb_height as usize],
|
||||
|
|
|
|||
Loading…
Reference in New Issue