From 2dea1eab9eee121a92bd65e0938ebaa8cc0d3e56 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Wed, 17 Dec 2025 14:12:16 -0500 Subject: [PATCH] improve video import performance a bit --- daw-backend/Cargo.toml | 1 + daw-backend/src/audio/engine.rs | 9 +- .../lightningbeam-core/src/video.rs | 111 +++++++++++++++--- .../lightningbeam-editor/Cargo.toml | 1 + .../lightningbeam-editor/src/main.rs | 83 ++++++++++++- 5 files changed, 181 insertions(+), 24 deletions(-) diff --git a/daw-backend/Cargo.toml b/daw-backend/Cargo.toml index 951acc0..b5a2bd1 100644 --- a/daw-backend/Cargo.toml +++ b/daw-backend/Cargo.toml @@ -15,6 +15,7 @@ crossterm = "0.27" rand = "0.8" base64 = "0.22" pathdiff = "0.2" +rayon = "1.10" # Audio export hound = "3.5" diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index eb0e3fb..7e564b5 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -1523,7 +1523,7 @@ impl Engine { pool_index, detail_level, chunk_indices); // Get audio file data from pool if let Some(audio_file) = self.audio_pool.get_file(pool_index) { - println!("✅ [ENGINE] Found audio file in pool, spawning background thread"); + println!("✅ [ENGINE] Found audio file in pool, queuing work in thread pool"); // Clone necessary data for background thread let data = audio_file.data.clone(); let channels = audio_file.channels; @@ -1531,8 +1531,8 @@ impl Engine { let path = audio_file.path.clone(); let chunk_tx = self.chunk_generation_tx.clone(); - // Generate chunks in background thread to avoid blocking audio thread - std::thread::spawn(move || { + // Generate chunks using rayon's thread pool to avoid spawning thousands of threads + rayon::spawn(move || { // Create temporary AudioFile for chunk generation let temp_audio_file = crate::audio::pool::AudioFile::with_format( path, @@ -1563,6 +1563,9 @@ impl Engine { chunks: event_chunks, }); } + + // Yield to other threads to reduce CPU contention with video playback + std::thread::sleep(std::time::Duration::from_millis(1)); }); } else { eprintln!("❌ [ENGINE] Pool index {} not found for waveform generation", pool_index); diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 38b6e95..084e818 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -21,7 +21,7 @@ pub struct VideoMetadata { } /// Video decoder with LRU frame caching -struct VideoDecoder { +pub struct VideoDecoder { path: String, width: u32, // Original video width height: u32, // Original video height @@ -43,7 +43,9 @@ impl VideoDecoder { /// /// `max_width` and `max_height` specify the maximum output dimensions. /// Video will be scaled down if larger, preserving aspect ratio. - fn new(path: String, cache_size: usize, max_width: Option, max_height: Option) -> Result { + /// `build_keyframes` controls whether to build the keyframe index immediately (slow) + /// or defer it for async building later. + fn new(path: String, cache_size: usize, max_width: Option, max_height: Option, build_keyframes: bool) -> Result { ffmpeg::init().map_err(|e| e.to_string())?; let input = ffmpeg::format::input(&path) @@ -92,11 +94,16 @@ impl VideoDecoder { let fps = f64::from(video_stream.avg_frame_rate()); - // Build keyframe index for fast seeking - // This scans the video once to find all keyframe positions - eprintln!("[Video Decoder] Building keyframe index for {}", path); - let keyframe_positions = Self::build_keyframe_index(&path, stream_index)?; - eprintln!("[Video Decoder] Found {} keyframes", keyframe_positions.len()); + // Optionally build keyframe index for fast seeking + let keyframe_positions = if build_keyframes { + eprintln!("[Video Decoder] Building keyframe index for {}", path); + let positions = Self::build_keyframe_index(&path, stream_index)?; + eprintln!("[Video Decoder] Found {} keyframes", positions.len()); + positions + } else { + eprintln!("[Video Decoder] Deferring keyframe index building for {}", path); + Vec::new() + }; Ok(Self { path, @@ -118,6 +125,31 @@ impl VideoDecoder { }) } + /// Build keyframe index for this decoder + /// This can be called asynchronously after decoder creation + fn build_and_set_keyframe_index(&mut self) -> Result<(), String> { + eprintln!("[Video Decoder] Building keyframe index for {}", self.path); + let positions = Self::build_keyframe_index(&self.path, self.stream_index)?; + eprintln!("[Video Decoder] Found {} keyframes", positions.len()); + self.keyframe_positions = positions; + Ok(()) + } + + /// Get the output width (scaled dimensions) + pub fn get_output_width(&self) -> u32 { + self.output_width + } + + /// Get the output height (scaled dimensions) + pub fn get_output_height(&self) -> u32 { + self.output_height + } + + /// Decode a frame at the specified timestamp (public wrapper) + pub fn decode_frame(&mut self, timestamp: f64) -> Result, String> { + self.get_frame(timestamp) + } + /// Build an index of all keyframe positions in the video /// This enables fast seeking by knowing exactly where keyframes are fn build_keyframe_index(path: &str, stream_index: usize) -> Result, String> { @@ -407,6 +439,9 @@ impl VideoManager { /// /// `target_width` and `target_height` specify the maximum dimensions /// for decoded frames. Video will be scaled down if larger. + /// + /// The keyframe index is NOT built during this call - use `build_keyframe_index_async` + /// in a background thread to build it asynchronously. pub fn load_video( &mut self, clip_id: Uuid, @@ -417,12 +452,13 @@ impl VideoManager { // First probe the video for metadata let metadata = probe_video(&path)?; - // Create decoder with target dimensions + // Create decoder with target dimensions, without building keyframe index let decoder = VideoDecoder::new( path, self.cache_size, Some(target_width), Some(target_height), + false, // Don't build keyframe index synchronously )?; // Store decoder in pool @@ -431,6 +467,20 @@ impl VideoManager { Ok(metadata) } + /// Build keyframe index for a loaded video asynchronously + /// + /// This should be called from a background thread after load_video() + /// to avoid blocking the UI during import. + pub fn build_keyframe_index(&self, clip_id: &Uuid) -> Result<(), String> { + let decoder_arc = self.decoders.get(clip_id) + .ok_or_else(|| format!("Video clip {} not found", clip_id))?; + + let mut decoder = decoder_arc.lock() + .map_err(|e| format!("Failed to lock decoder: {}", e))?; + + decoder.build_and_set_keyframe_index() + } + /// Get a decoded frame for a specific clip at a specific timestamp /// /// Returns None if the clip is not loaded or decoding fails. @@ -467,10 +517,14 @@ impl VideoManager { Some(frame) } - /// Generate thumbnails for a video clip + /// Generate thumbnails for a video clip (single batch version - use generate_thumbnails_progressive instead) /// - /// Thumbnails are generated every 5 seconds at 64px width. + /// Thumbnails are generated every 5 seconds at 128px width. /// This should be called in a background thread to avoid blocking. + /// Thumbnails are inserted into the cache progressively as they're generated, + /// allowing the UI to display them immediately. + /// + /// DEPRECATED: Use generate_thumbnails_progressive which releases the lock between thumbnails. pub fn generate_thumbnails(&mut self, clip_id: &Uuid, duration: f64) -> Result<(), String> { let decoder_arc = self.decoders.get(clip_id) .ok_or("Clip not loaded")? @@ -479,7 +533,9 @@ impl VideoManager { let mut decoder = decoder_arc.lock() .map_err(|e| format!("Failed to lock decoder: {}", e))?; - let mut thumbnails = Vec::new(); + // Initialize thumbnail cache entry with empty vec + self.thumbnail_cache.insert(*clip_id, Vec::new()); + let interval = 5.0; // Generate thumbnail every 5 seconds let mut t = 0.0; @@ -505,18 +561,32 @@ impl VideoManager { thumb_height, ); - thumbnails.push((t, Arc::new(thumb_data))); + // Insert thumbnail into cache immediately so UI can display it + if let Some(thumbnails) = self.thumbnail_cache.get_mut(clip_id) { + thumbnails.push((t, Arc::new(thumb_data))); + } } t += interval; } - // Store thumbnails in cache - self.thumbnail_cache.insert(*clip_id, thumbnails); - Ok(()) } + /// Get the decoder Arc for a clip (for external thumbnail generation) + /// This allows external code to decode frames without holding the VideoManager lock + pub fn get_decoder(&self, clip_id: &Uuid) -> Option>> { + self.decoders.get(clip_id).cloned() + } + + /// Insert a thumbnail into the cache (for external thumbnail generation) + pub fn insert_thumbnail(&mut self, clip_id: &Uuid, timestamp: f64, data: Arc>) { + self.thumbnail_cache + .entry(*clip_id) + .or_insert_with(Vec::new) + .push((timestamp, data)); + } + /// Get the thumbnail closest to the specified timestamp /// /// Returns None if no thumbnails have been generated for this clip. @@ -582,6 +652,17 @@ impl Default for VideoManager { } /// Simple nearest-neighbor downsampling for RGBA images +pub fn downsample_rgba_public( + src: &[u8], + src_width: u32, + src_height: u32, + dst_width: u32, + dst_height: u32, +) -> Vec { + downsample_rgba(src, src_width, src_height, dst_width, dst_height) +} + +/// Simple nearest-neighbor downsampling for RGBA images (internal) fn downsample_rgba( src: &[u8], src_width: u32, diff --git a/lightningbeam-ui/lightningbeam-editor/Cargo.toml b/lightningbeam-ui/lightningbeam-editor/Cargo.toml index 3896e4d..fb5a100 100644 --- a/lightningbeam-ui/lightningbeam-editor/Cargo.toml +++ b/lightningbeam-ui/lightningbeam-editor/Cargo.toml @@ -44,6 +44,7 @@ lightningcss = "1.0.0-alpha.68" clap = { version = "4.5", features = ["derive"] } uuid = { version = "1.0", features = ["v4", "serde"] } petgraph = "0.6" +rayon = "1.10" # Native file dialogs rfd = "0.15" diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 3bd39ae..e5b5c49 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -54,6 +54,19 @@ struct Args { fn main() -> eframe::Result { println!("🚀 Starting Lightningbeam Editor..."); + // Configure rayon thread pool to use fewer threads, leaving cores free for video playback + let num_cpus = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + let waveform_threads = (num_cpus.saturating_sub(2)).max(2); // Leave 2 cores free, minimum 2 threads + rayon::ThreadPoolBuilder::new() + .num_threads(waveform_threads) + .thread_name(|i| format!("waveform-{}", i)) + .build_global() + .expect("Failed to build rayon thread pool"); + println!("✅ Configured waveform generation to use {} threads (leaving {} cores for video)", + waveform_threads, num_cpus - waveform_threads); + // Parse command line arguments let args = Args::parse(); @@ -2095,7 +2108,7 @@ impl EditorApp { let clip_id = clip.id; - // Load video into VideoManager + // Load video into VideoManager (without building keyframe index) let doc_width = self.action_executor.document().width as u32; let doc_height = self.action_executor.document().height as u32; @@ -2106,6 +2119,18 @@ impl EditorApp { } drop(video_mgr); + // Spawn background thread to build keyframe index asynchronously + let video_manager_clone = Arc::clone(&self.video_manager); + let keyframe_clip_id = clip_id; + std::thread::spawn(move || { + let video_mgr = video_manager_clone.lock().unwrap(); + if let Err(e) = video_mgr.build_keyframe_index(&keyframe_clip_id) { + eprintln!("Failed to build keyframe index: {}", e); + } else { + println!(" Built keyframe index for video clip {}", keyframe_clip_id); + } + }); + // Spawn background thread for audio extraction if video has audio if metadata.has_audio { if let Some(ref audio_controller) = self.audio_controller { @@ -2180,15 +2205,61 @@ impl EditorApp { } // Spawn background thread for thumbnail generation + // Get decoder once, then generate thumbnails without holding VideoManager lock let video_manager_clone = Arc::clone(&self.video_manager); let duration = metadata.duration; + let thumb_clip_id = clip_id; std::thread::spawn(move || { - let mut video_mgr = video_manager_clone.lock().unwrap(); - if let Err(e) = video_mgr.generate_thumbnails(&clip_id, duration) { - eprintln!("Failed to generate video thumbnails: {}", e); - } else { - println!(" Generated thumbnails for video clip {}", clip_id); + // Get decoder Arc with brief lock + let decoder_arc = { + let video_mgr = video_manager_clone.lock().unwrap(); + match video_mgr.get_decoder(&thumb_clip_id) { + Some(arc) => arc, + None => { + eprintln!("Failed to get decoder for thumbnail generation"); + return; + } + } + }; + // VideoManager lock released - video can now be displayed! + + let interval = 5.0; + let mut t = 0.0; + let mut thumbnail_count = 0; + + while t < duration { + // Decode frame WITHOUT holding VideoManager lock + let thumb_opt = { + let mut decoder = decoder_arc.lock().unwrap(); + match decoder.decode_frame(t) { + Ok(rgba_data) => { + let w = decoder.get_output_width(); + let h = decoder.get_output_height(); + Some((rgba_data, w, h)) + } + Err(_) => None, + } + }; + + // Downsample without any locks + if let Some((rgba_data, w, h)) = thumb_opt { + use lightningbeam_core::video::downsample_rgba_public; + let thumb_w = 128u32; + let thumb_h = (h as f32 / w as f32 * thumb_w as f32) as u32; + let thumb_data = downsample_rgba_public(&rgba_data, w, h, thumb_w, thumb_h); + + // Brief lock just to insert + { + let mut video_mgr = video_manager_clone.lock().unwrap(); + video_mgr.insert_thumbnail(&thumb_clip_id, t, Arc::new(thumb_data)); + } + thumbnail_count += 1; + } + + t += interval; } + + println!(" Generated {} thumbnails for video clip {}", thumbnail_count, thumb_clip_id); }); // Add clip to document