From 8ac5f52f283ead7d3ae113100560eb5cc9ad7314 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Wed, 11 Feb 2026 14:38:58 -0500 Subject: [PATCH] Render audio waveforms on gpu --- daw-backend/src/audio/engine.rs | 35 + daw-backend/src/command/types.rs | 4 + lightningbeam-ui/Cargo.lock | 3 + .../lightningbeam-editor/Cargo.toml | 2 + .../lightningbeam-editor/src/main.rs | 255 +++---- .../src/panes/asset_library.rs | 53 +- .../lightningbeam-editor/src/panes/mod.rs | 13 +- .../src/panes/shaders/waveform.wgsl | 157 +++++ .../src/panes/shaders/waveform_mipgen.wgsl | 64 ++ .../src/panes/timeline.rs | 419 ++---------- .../lightningbeam-editor/src/waveform_gpu.rs | 624 ++++++++++++++++++ .../src/waveform_image_cache.rs | 326 --------- 12 files changed, 1100 insertions(+), 855 deletions(-) create mode 100644 lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl create mode 100644 lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform_mipgen.wgsl create mode 100644 lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs delete mode 100644 lightningbeam-ui/lightningbeam-editor/src/waveform_image_cache.rs diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index 7e564b5..cbe9e32 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -1812,6 +1812,16 @@ impl Engine { None => QueryResponse::PoolFileInfo(Err(format!("Pool index {} not found", pool_index))), } } + Query::GetPoolAudioSamples(pool_index) => { + match self.audio_pool.get_file(pool_index) { + Some(file) => QueryResponse::PoolAudioSamples(Ok(( + file.data.clone(), + file.sample_rate, + file.channels, + ))), + None => QueryResponse::PoolAudioSamples(Err(format!("Pool index {} not found", pool_index))), + } + } Query::ExportAudio(settings, output_path) => { // Perform export directly - this will block the audio thread but that's okay // since we're exporting and not playing back anyway @@ -2898,6 +2908,31 @@ impl EngineController { Err("Query timeout".to_string()) } + /// Get raw audio samples from pool (samples, sample_rate, channels) + pub fn get_pool_audio_samples(&mut self, pool_index: usize) -> Result<(Vec, u32, u32), String> { + if let Err(_) = self.query_tx.push(Query::GetPoolAudioSamples(pool_index)) { + return Err("Failed to send query - queue full".to_string()); + } + + let start = std::time::Instant::now(); + let timeout = std::time::Duration::from_secs(5); // Longer timeout for large audio data + + while start.elapsed() < timeout { + if let Ok(response) = self.query_response_rx.pop() { + match response { + QueryResponse::PoolAudioSamples(result) => return result, + QueryResponse::AudioExported(result) => { + self.cached_export_response = Some(result); + } + _ => {} + } + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + + Err("Query timeout".to_string()) + } + /// Request waveform chunks to be generated /// This is an asynchronous command - chunks will be returned via WaveformChunksReady events pub fn generate_waveform_chunks( diff --git a/daw-backend/src/command/types.rs b/daw-backend/src/command/types.rs index 1322a5c..7ab3563 100644 --- a/daw-backend/src/command/types.rs +++ b/daw-backend/src/command/types.rs @@ -299,6 +299,8 @@ pub enum Query { AddAudioClipSync(TrackId, usize, f64, f64, f64), /// Add an audio file to the pool synchronously (path, data, channels, sample_rate) - returns pool index AddAudioFileSync(String, Vec, u32, u32), + /// Get raw audio samples from pool (pool_index) - returns (samples, sample_rate, channels) + GetPoolAudioSamples(usize), /// Get a clone of the current project for serialization GetProject, /// Set the project (replaces current project state) @@ -368,6 +370,8 @@ pub enum QueryResponse { AudioClipInstanceAdded(Result), /// Audio file added to pool (returns pool index) AudioFileAddedSync(Result), + /// Raw audio samples from pool (samples, sample_rate, channels) + PoolAudioSamples(Result<(Vec, u32, u32), String>), /// Project retrieved ProjectRetrieved(Result, String>), /// Project set diff --git a/lightningbeam-ui/Cargo.lock b/lightningbeam-ui/Cargo.lock index 07f4b45..ba9b184 100644 --- a/lightningbeam-ui/Cargo.lock +++ b/lightningbeam-ui/Cargo.lock @@ -2867,6 +2867,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if", "crunchy", "num-traits", @@ -3433,6 +3434,7 @@ dependencies = [ name = "lightningbeam-editor" version = "0.1.0" dependencies = [ + "bytemuck", "clap", "cpal", "daw-backend", @@ -3443,6 +3445,7 @@ dependencies = [ "egui_extras", "egui_node_graph2", "ffmpeg-next", + "half", "image", "kurbo 0.12.0", "lightningbeam-core", diff --git a/lightningbeam-ui/lightningbeam-editor/Cargo.toml b/lightningbeam-ui/lightningbeam-editor/Cargo.toml index fb5a100..785be0c 100644 --- a/lightningbeam-ui/lightningbeam-editor/Cargo.toml +++ b/lightningbeam-ui/lightningbeam-editor/Cargo.toml @@ -20,6 +20,8 @@ egui_node_graph2 = { git = "https://github.com/PVDoriginal/egui_node_graph2" } # GPU wgpu = { workspace = true } vello = { workspace = true } +half = { version = "2.4", features = ["bytemuck"] } +bytemuck = { version = "1.14", features = ["derive"] } kurbo = { workspace = true } peniko = { workspace = true } diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 2b3d62f..e521d5a 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -19,7 +19,7 @@ use menu::{MenuAction, MenuSystem}; mod theme; use theme::{Theme, ThemeMode}; -mod waveform_image_cache; +mod waveform_gpu; mod config; use config::AppConfig; @@ -689,21 +689,16 @@ struct EditorApp { /// Prevents repeated backend queries for the same MIDI clip /// Format: (timestamp, note_number, is_note_on) midi_event_cache: HashMap>, - /// Cache for audio waveform data (keyed by audio_pool_index) - /// Prevents repeated backend queries for the same audio file - /// Format: Vec of WaveformPeak (min/max pairs) - waveform_cache: HashMap>, - /// Chunk-based waveform cache for multi-resolution waveforms - /// Format: (pool_index, detail_level, chunk_index) -> Vec - waveform_chunk_cache: HashMap<(usize, u8, u32), Vec>, /// Cache for audio file durations to avoid repeated queries /// Format: pool_index -> duration in seconds audio_duration_cache: HashMap, - /// Track which audio pool indices got new waveform data this frame (for thumbnail invalidation) + /// Track which audio pool indices got new raw audio data this frame (for thumbnail invalidation) audio_pools_with_new_waveforms: HashSet, - /// Cache for rendered waveform images (GPU textures) - /// Stores pre-rendered waveform tiles at various zoom levels for fast blitting - waveform_image_cache: waveform_image_cache::WaveformImageCache, + /// Raw audio sample cache for GPU waveform rendering + /// Format: pool_index -> (samples, sample_rate, channels) + raw_audio_cache: HashMap, u32, u32)>, + /// Pool indices that need GPU texture upload (set when raw audio arrives, cleared after upload) + waveform_gpu_dirty: HashSet, /// Current file path (None if not yet saved) current_file_path: Option, /// Application configuration (recent files, etc.) @@ -896,11 +891,10 @@ impl EditorApp { paint_bucket_gap_tolerance: 5.0, // Default gap tolerance polygon_sides: 5, // Default to pentagon midi_event_cache: HashMap::new(), // Initialize empty MIDI event cache - waveform_cache: HashMap::new(), // Initialize empty waveform cache - waveform_chunk_cache: HashMap::new(), // Initialize empty chunk-based waveform cache audio_duration_cache: HashMap::new(), // Initialize empty audio duration cache - audio_pools_with_new_waveforms: HashSet::new(), // Track pool indices with new waveforms - waveform_image_cache: waveform_image_cache::WaveformImageCache::new(), // Initialize waveform image cache + audio_pools_with_new_waveforms: HashSet::new(), // Track pool indices with new raw audio + raw_audio_cache: HashMap::new(), + waveform_gpu_dirty: HashSet::new(), current_file_path: None, // No file loaded initially config, file_command_tx, @@ -1465,78 +1459,6 @@ impl EditorApp { } } - /// Fetch waveform data from backend for a specific audio pool index - /// Returns cached data if available, otherwise tries to assemble from chunks - /// For thumbnails, uses Level 0 (overview) chunks which are fast to generate - fn fetch_waveform(&mut self, pool_index: usize) -> Option> { - // Check if already cached in old waveform cache - if let Some(waveform) = self.waveform_cache.get(&pool_index) { - return Some(waveform.clone()); - } - - // Try to assemble from Level 0 (overview) chunks - perfect for thumbnails - // Level 0 = 1 peak/sec, so a 200s file only needs 200 peaks (very fast) - - // Get audio file duration (use cached value to avoid repeated queries) - let audio_file_duration = if let Some(&duration) = self.audio_duration_cache.get(&pool_index) { - duration - } else { - // Duration not cached - query it once and cache - if let Some(ref controller_arc) = self.audio_controller { - let mut controller = controller_arc.lock().unwrap(); - match controller.get_pool_file_info(pool_index) { - Ok((duration, _, _)) => { - self.audio_duration_cache.insert(pool_index, duration); - duration - } - Err(_) => return None, - } - } else { - return None; - } - }; - - // Assemble Level 0 chunks for the entire file - let detail_level = 0; // Level 0 (overview) - let chunk_time_span = 60.0; // 60 seconds per chunk - let total_chunks = (audio_file_duration / chunk_time_span).ceil() as u32; - - let mut assembled_peaks = Vec::new(); - let mut missing_chunks = Vec::new(); - - // Check if all required chunks are available - for chunk_idx in 0..total_chunks { - let key = (pool_index, detail_level, chunk_idx); - if let Some(chunk_peaks) = self.waveform_chunk_cache.get(&key) { - assembled_peaks.extend_from_slice(chunk_peaks); - } else { - missing_chunks.push(chunk_idx); - } - } - - // If any chunks are missing, request them (but only if we have a controller) - if !missing_chunks.is_empty() { - if let Some(ref controller_arc) = self.audio_controller { - let mut controller = controller_arc.lock().unwrap(); - let _ = controller.generate_waveform_chunks( - pool_index, - detail_level, - missing_chunks, - 2, // High priority for thumbnails - ); - } - return None; // Will retry next frame when chunks arrive - } - - // All chunks available - cache and return - if !assembled_peaks.is_empty() { - self.waveform_cache.insert(pool_index, assembled_peaks.clone()); - return Some(assembled_peaks); - } - - None - } - fn switch_layout(&mut self, index: usize) { self.current_layout_index = index; self.current_layout = self.layouts[index].layout.clone(); @@ -2299,9 +2221,8 @@ impl EditorApp { self.sync_audio_layers_to_backend(); eprintln!("πŸ“Š [APPLY] Step 6: Sync audio layers took {:.2}ms", step6_start.elapsed().as_secs_f64() * 1000.0); - // Fetch waveforms for all audio clips in the loaded project + // Fetch raw audio for all audio clips in the loaded project let step7_start = std::time::Instant::now(); - // Collect pool indices first to avoid borrowing issues let pool_indices: Vec = self.action_executor.document() .audio_clips.values() .filter_map(|clip| { @@ -2313,13 +2234,23 @@ impl EditorApp { }) .collect(); - let mut waveforms_fetched = 0; + let mut raw_fetched = 0; for pool_index in pool_indices { - if self.fetch_waveform(pool_index).is_some() { - waveforms_fetched += 1; + if !self.raw_audio_cache.contains_key(&pool_index) { + if let Some(ref controller_arc) = self.audio_controller { + let mut controller = controller_arc.lock().unwrap(); + match controller.get_pool_audio_samples(pool_index) { + Ok((samples, sr, ch)) => { + self.raw_audio_cache.insert(pool_index, (samples, sr, ch)); + self.waveform_gpu_dirty.insert(pool_index); + raw_fetched += 1; + } + Err(e) => eprintln!("Failed to fetch raw audio for pool {}: {}", pool_index, e), + } + } } } - eprintln!("πŸ“Š [APPLY] Step 7: Fetched {} waveforms in {:.2}ms", waveforms_fetched, step7_start.elapsed().as_secs_f64() * 1000.0); + eprintln!("πŸ“Š [APPLY] Step 7: Fetched {} raw audio samples in {:.2}ms", raw_fetched, step7_start.elapsed().as_secs_f64() * 1000.0); // Reset playback state self.playback_time = 0.0; @@ -2427,9 +2358,17 @@ impl EditorApp { let clip = AudioClip::new_sampled(&name, pool_index, duration); let clip_id = self.action_executor.document_mut().add_audio_clip(clip); - // Fetch waveform from backend and cache it for rendering - if let Some(waveform) = self.fetch_waveform(pool_index) { - println!("βœ… Cached waveform with {} peaks", waveform.len()); + // Fetch raw audio samples for GPU waveform rendering + if let Some(ref controller_arc) = self.audio_controller { + let mut controller = controller_arc.lock().unwrap(); + match controller.get_pool_audio_samples(pool_index) { + Ok((samples, sr, ch)) => { + println!("βœ… Cached {} raw audio samples for GPU waveform", samples.len()); + self.raw_audio_cache.insert(pool_index, (samples, sr, ch)); + self.waveform_gpu_dirty.insert(pool_index); + } + Err(e) => eprintln!("Failed to fetch raw audio: {}", e), + } } println!("Imported audio '{}' ({:.1}s, {}ch, {}Hz) - ID: {}", @@ -3053,9 +2992,16 @@ impl EditorApp { audio_clip_id ); - // Fetch waveform from backend and cache it for rendering - if let Some(waveform) = self.fetch_waveform(pool_index) { - println!(" Cached waveform with {} peaks", waveform.len()); + // Fetch raw audio samples for GPU waveform rendering + if let Some(ref controller_arc) = self.audio_controller { + let mut controller = controller_arc.lock().unwrap(); + match controller.get_pool_audio_samples(pool_index) { + Ok((samples, sr, ch)) => { + self.raw_audio_cache.insert(pool_index, (samples, sr, ch)); + self.waveform_gpu_dirty.insert(pool_index); + } + Err(e) => eprintln!("Failed to fetch raw audio for extracted audio: {}", e), + } } // Auto-place extracted audio if the video was auto-placed @@ -3107,14 +3053,13 @@ impl eframe::App for EditorApp { // Will switch to editor mode when file finishes loading } - // Fetch missing waveforms on-demand (for lazy loading after project load) - // Collect pool indices that need waveforms - let missing_waveforms: Vec = self.action_executor.document() + // Fetch missing raw audio on-demand (for lazy loading after project load) + // Collect pool indices that need raw audio data + let missing_raw_audio: Vec = self.action_executor.document() .audio_clips.values() .filter_map(|clip| { if let lightningbeam_core::clip::AudioClipType::Sampled { audio_pool_index } = &clip.clip_type { - // Check if not already cached - if !self.waveform_cache.contains_key(audio_pool_index) { + if !self.raw_audio_cache.contains_key(audio_pool_index) { Some(*audio_pool_index) } else { None @@ -3125,9 +3070,19 @@ impl eframe::App for EditorApp { }) .collect(); - // Fetch missing waveforms - for pool_index in missing_waveforms { - self.fetch_waveform(pool_index); + // Fetch missing raw audio samples + for pool_index in missing_raw_audio { + if let Some(ref controller_arc) = self.audio_controller { + let mut controller = controller_arc.lock().unwrap(); + match controller.get_pool_audio_samples(pool_index) { + Ok((samples, sr, ch)) => { + self.raw_audio_cache.insert(pool_index, (samples, sr, ch)); + self.waveform_gpu_dirty.insert(pool_index); + self.audio_pools_with_new_waveforms.insert(pool_index); + } + Err(e) => eprintln!("Failed to fetch raw audio for pool {}: {}", pool_index, e), + } + } } // Initialize and update effect thumbnail generator (GPU-based effect previews) @@ -3310,29 +3265,22 @@ impl eframe::App for EditorApp { ); ctx.request_repaint(); } - AudioEvent::WaveformChunksReady { pool_index, detail_level, chunks } => { - // Store waveform chunks in the cache - let mut all_peaks = Vec::new(); - for (chunk_index, _time_range, peaks) in chunks { - let key = (pool_index, detail_level, chunk_index); - self.waveform_chunk_cache.insert(key, peaks.clone()); - all_peaks.extend(peaks); + AudioEvent::WaveformChunksReady { pool_index, .. } => { + // Fetch raw audio for GPU waveform if not already cached + if !self.raw_audio_cache.contains_key(&pool_index) { + if let Some(ref controller_arc) = self.audio_controller { + let mut controller = controller_arc.lock().unwrap(); + match controller.get_pool_audio_samples(pool_index) { + Ok((samples, sr, ch)) => { + self.raw_audio_cache.insert(pool_index, (samples, sr, ch)); + self.waveform_gpu_dirty.insert(pool_index); + self.audio_pools_with_new_waveforms.insert(pool_index); + } + Err(e) => eprintln!("Failed to fetch raw audio for pool {}: {}", pool_index, e), + } + } } - // If this is Level 0 (overview), also populate the old waveform_cache - // so asset library thumbnails can use it immediately - if detail_level == 0 && !all_peaks.is_empty() { - println!("πŸ’Ύ [EVENT] Storing {} Level 0 peaks for pool {} in waveform_cache", all_peaks.len(), pool_index); - self.waveform_cache.insert(pool_index, all_peaks); - // Mark this pool index as having new waveform data (for thumbnail invalidation) - self.audio_pools_with_new_waveforms.insert(pool_index); - println!("πŸ”” [EVENT] Marked pool {} for thumbnail invalidation", pool_index); - } - - // Invalidate image cache for this pool index - // (The waveform tiles will be regenerated with new chunk data) - self.waveform_image_cache.invalidate_audio(pool_index); - ctx.request_repaint(); } // Recording events @@ -3394,18 +3342,22 @@ impl eframe::App for EditorApp { } ctx.request_repaint(); } - AudioEvent::RecordingStopped(_backend_clip_id, pool_index, waveform) => { - println!("🎀 Recording stopped: pool_index={}, {} peaks", pool_index, waveform.len()); + AudioEvent::RecordingStopped(_backend_clip_id, pool_index, _waveform) => { + println!("🎀 Recording stopped: pool_index={}", pool_index); - // Store waveform for the recorded clip - if !waveform.is_empty() { - self.waveform_cache.insert(pool_index, waveform.clone()); - self.audio_pools_with_new_waveforms.insert(pool_index); + // Fetch raw audio samples for GPU waveform rendering + if let Some(ref controller_arc) = self.audio_controller { + let mut controller = controller_arc.lock().unwrap(); + match controller.get_pool_audio_samples(pool_index) { + Ok((samples, sr, ch)) => { + self.raw_audio_cache.insert(pool_index, (samples, sr, ch)); + self.waveform_gpu_dirty.insert(pool_index); + self.audio_pools_with_new_waveforms.insert(pool_index); + } + Err(e) => eprintln!("Failed to fetch raw audio after recording: {}", e), + } } - // Invalidate waveform image cache so tiles are regenerated - self.waveform_image_cache.invalidate_audio(pool_index); - // Get accurate duration from backend (not calculated from waveform peaks) let duration = if let Some(ref controller_arc) = self.audio_controller { let mut controller = controller_arc.lock().unwrap(); @@ -3821,10 +3773,9 @@ impl eframe::App for EditorApp { polygon_sides: &mut self.polygon_sides, layer_to_track_map: &self.layer_to_track_map, midi_event_cache: &self.midi_event_cache, - waveform_cache: &self.waveform_cache, - waveform_chunk_cache: &self.waveform_chunk_cache, - waveform_image_cache: &mut self.waveform_image_cache, audio_pools_with_new_waveforms: &self.audio_pools_with_new_waveforms, + raw_audio_cache: &self.raw_audio_cache, + waveform_gpu_dirty: &mut self.waveform_gpu_dirty, effect_to_load: &mut self.effect_to_load, effect_thumbnail_requests: &mut effect_thumbnail_requests, effect_thumbnail_cache: self.effect_thumbnail_generator.as_ref() @@ -4052,14 +4003,12 @@ struct RenderContext<'a> { layer_to_track_map: &'a std::collections::HashMap, /// Cache of MIDI events for rendering (keyed by backend midi_clip_id) midi_event_cache: &'a HashMap>, - /// Cache of waveform data for rendering (keyed by audio_pool_index) - waveform_cache: &'a HashMap>, - /// Chunk-based waveform cache for multi-resolution waveforms - waveform_chunk_cache: &'a HashMap<(usize, u8, u32), Vec>, - /// Cache of rendered waveform images (GPU textures) - waveform_image_cache: &'a mut waveform_image_cache::WaveformImageCache, - /// Audio pool indices with new waveform data this frame (for thumbnail invalidation) + /// Audio pool indices with new raw audio data this frame (for thumbnail invalidation) audio_pools_with_new_waveforms: &'a HashSet, + /// Raw audio samples for GPU waveform rendering (pool_index -> (samples, sample_rate, channels)) + raw_audio_cache: &'a HashMap, u32, u32)>, + /// Pool indices needing GPU texture upload + waveform_gpu_dirty: &'a mut HashSet, /// Effect ID to load into shader editor (set by asset library, consumed by shader editor) effect_to_load: &'a mut Option, /// Queue for effect thumbnail requests @@ -4540,10 +4489,9 @@ fn render_pane( paint_bucket_gap_tolerance: ctx.paint_bucket_gap_tolerance, polygon_sides: ctx.polygon_sides, midi_event_cache: ctx.midi_event_cache, - waveform_cache: ctx.waveform_cache, - waveform_chunk_cache: ctx.waveform_chunk_cache, - waveform_image_cache: ctx.waveform_image_cache, audio_pools_with_new_waveforms: ctx.audio_pools_with_new_waveforms, + raw_audio_cache: ctx.raw_audio_cache, + waveform_gpu_dirty: ctx.waveform_gpu_dirty, effect_to_load: ctx.effect_to_load, effect_thumbnail_requests: ctx.effect_thumbnail_requests, effect_thumbnail_cache: ctx.effect_thumbnail_cache, @@ -4608,10 +4556,9 @@ fn render_pane( paint_bucket_gap_tolerance: ctx.paint_bucket_gap_tolerance, polygon_sides: ctx.polygon_sides, midi_event_cache: ctx.midi_event_cache, - waveform_cache: ctx.waveform_cache, - waveform_chunk_cache: ctx.waveform_chunk_cache, - waveform_image_cache: ctx.waveform_image_cache, audio_pools_with_new_waveforms: ctx.audio_pools_with_new_waveforms, + raw_audio_cache: ctx.raw_audio_cache, + waveform_gpu_dirty: ctx.waveform_gpu_dirty, effect_to_load: ctx.effect_to_load, effect_thumbnail_requests: ctx.effect_thumbnail_requests, effect_thumbnail_cache: ctx.effect_thumbnail_cache, diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs index d92a572..ab26ff1 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs @@ -17,6 +17,42 @@ use uuid::Uuid; use super::{DragClipType, DraggingAsset, NodePath, PaneRenderer, SharedPaneState}; use crate::widgets::ImeTextField; +/// Derive min/max peak pairs from raw audio samples for thumbnail rendering. +/// Downsamples to `num_peaks` (min, max) pairs by scanning chunks of samples. +fn peaks_from_raw_audio( + raw: &(Vec, u32, u32), // (samples, sample_rate, channels) + num_peaks: usize, +) -> Vec<(f32, f32)> { + let (samples, _sr, channels) = raw; + let ch = (*channels as usize).max(1); + let total_frames = samples.len() / ch; + if total_frames == 0 || num_peaks == 0 { + return vec![]; + } + let frames_per_peak = (total_frames as f64 / num_peaks as f64).max(1.0); + let mut peaks = Vec::with_capacity(num_peaks); + for i in 0..num_peaks { + let start = (i as f64 * frames_per_peak) as usize; + let end = (((i + 1) as f64 * frames_per_peak) as usize).min(total_frames); + let mut min_val = f32::MAX; + let mut max_val = f32::MIN; + for frame in start..end { + // Mix all channels together for the thumbnail + let mut sample = 0.0f32; + for c in 0..ch { + sample += samples[frame * ch + c]; + } + sample /= ch as f32; + min_val = min_val.min(sample); + max_val = max_val.max(sample); + } + if min_val <= max_val { + peaks.push((min_val, max_val)); + } + } + peaks +} + // Thumbnail constants const THUMBNAIL_SIZE: u32 = 64; const THUMBNAIL_PREVIEW_SECONDS: f64 = 10.0; @@ -1790,8 +1826,8 @@ impl AssetLibraryPane { if asset_category == AssetCategory::Audio && !self.thumbnail_cache.has(&asset_id) { if let Some(clip) = document.audio_clips.get(&asset_id) { if let AudioClipType::Sampled { audio_pool_index } = &clip.clip_type { - shared.waveform_cache.get(audio_pool_index) - .map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect()) + shared.raw_audio_cache.get(audio_pool_index) + .map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize)) } else { None } @@ -2380,8 +2416,8 @@ impl AssetLibraryPane { match &clip.clip_type { AudioClipType::Sampled { audio_pool_index } => { let wave_color = egui::Color32::from_rgb(100, 200, 100); - let waveform: Option> = shared.waveform_cache.get(audio_pool_index) - .map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect()); + let waveform: Option> = shared.raw_audio_cache.get(audio_pool_index) + .map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize)); if let Some(ref peaks) = waveform { Some(generate_waveform_thumbnail(peaks, bg_color, wave_color)) } else { @@ -2525,8 +2561,8 @@ impl AssetLibraryPane { match &clip.clip_type { AudioClipType::Sampled { audio_pool_index } => { let wave_color = egui::Color32::from_rgb(100, 200, 100); - let waveform: Option> = shared.waveform_cache.get(audio_pool_index) - .map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect()); + let waveform: Option> = shared.raw_audio_cache.get(audio_pool_index) + .map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize)); if let Some(ref peaks) = waveform { Some(generate_waveform_thumbnail(peaks, bg_color, wave_color)) } else { @@ -2850,9 +2886,8 @@ impl AssetLibraryPane { if asset_category == AssetCategory::Audio && !self.thumbnail_cache.has(&asset_id) { if let Some(clip) = document.audio_clips.get(&asset_id) { if let AudioClipType::Sampled { audio_pool_index } = &clip.clip_type { - // Use cached waveform data (populated by fetch_waveform in main.rs) - let waveform = shared.waveform_cache.get(audio_pool_index) - .map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect()); + let waveform: Option> = shared.raw_audio_cache.get(audio_pool_index) + .map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize)); if waveform.is_some() { println!("🎡 Found waveform for pool {} (asset {})", audio_pool_index, asset_id); } else { diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs index e7127a6..973e6d6 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs @@ -189,15 +189,12 @@ pub struct SharedPaneState<'a> { pub polygon_sides: &'a mut u32, /// Cache of MIDI events for rendering (keyed by backend midi_clip_id) pub midi_event_cache: &'a std::collections::HashMap>, - /// Cache of waveform data for rendering (keyed by audio_pool_index) - pub waveform_cache: &'a std::collections::HashMap>, - /// Chunk-based waveform cache for multi-resolution waveforms - /// Format: (pool_index, detail_level, chunk_index) -> Vec - pub waveform_chunk_cache: &'a std::collections::HashMap<(usize, u8, u32), Vec>, - /// Cache of rendered waveform images (GPU textures) for fast blitting - pub waveform_image_cache: &'a mut crate::waveform_image_cache::WaveformImageCache, - /// Audio pool indices that got new waveform data this frame (for thumbnail invalidation) + /// Audio pool indices that got new raw audio data this frame (for thumbnail invalidation) pub audio_pools_with_new_waveforms: &'a std::collections::HashSet, + /// Raw audio samples for GPU waveform rendering (pool_index -> (samples, sample_rate, channels)) + pub raw_audio_cache: &'a std::collections::HashMap, u32, u32)>, + /// Pool indices needing GPU waveform texture upload + pub waveform_gpu_dirty: &'a mut std::collections::HashSet, /// Effect ID to load into shader editor (set by asset library, consumed by shader editor) pub effect_to_load: &'a mut Option, /// Queue for effect thumbnail requests (effect IDs to generate thumbnails for) diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl new file mode 100644 index 0000000..f5d4616 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl @@ -0,0 +1,157 @@ +// Waveform rendering shader for audio data stored in 2D Rgba16Float textures. +// Audio samples are packed row-major into 2D: frame_index = y * tex_width + x +// Mip levels use min/max reduction (4 consecutive samples per level). +// +// At full zoom, minβ‰ˆmax β†’ renders as a continuous wave. +// At zoom-out, min/max spread β†’ renders as filled peak region. +// +// display_mode: 0 = combined (mono mix), 1 = split (left top, right bottom) + +struct Params { + // Clip rectangle in screen pixels (min.x, min.y, max.x, max.y) + clip_rect: vec4, + // Timeline viewport parameters + viewport_start_time: f32, + pixels_per_second: f32, + // Audio file properties + audio_duration: f32, + sample_rate: f32, + // Clip placement + clip_start_time: f32, + trim_start: f32, + // Texture layout + tex_width: f32, + total_frames: f32, // total frame count in this texture segment + segment_start_frame: f32, // first frame this texture covers (for multi-texture) + display_mode: f32, // 0 = combined, 1 = split stereo + // Appearance + tint_color: vec4, + // Screen dimensions for coordinate conversion + screen_size: vec2, + _pad: vec2, +} + +@group(0) @binding(0) var peak_tex: texture_2d; +@group(0) @binding(1) var peak_sampler: sampler; +@group(0) @binding(2) var params: Params; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +// Fullscreen triangle (3 vertices, no vertex buffer) +@vertex +fn vs_main(@builtin(vertex_index) vi: u32) -> VertexOutput { + var out: VertexOutput; + let x = f32(i32(vi) / 2) * 4.0 - 1.0; + let y = f32(i32(vi) % 2) * 4.0 - 1.0; + out.position = vec4(x, y, 0.0, 1.0); + out.uv = vec2((x + 1.0) * 0.5, (1.0 - y) * 0.5); + return out; +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let frag_x = in.position.x; + let frag_y = in.position.y; + + // Clip to the clip rectangle + if frag_x < params.clip_rect.x || frag_x > params.clip_rect.z || + frag_y < params.clip_rect.y || frag_y > params.clip_rect.w { + discard; + } + + // Fragment X position β†’ audio time + let timeline_time = params.viewport_start_time + (frag_x - params.clip_rect.x) / params.pixels_per_second; + let audio_time = timeline_time - params.clip_start_time + params.trim_start; + + // Audio time β†’ frame index + let frame_f = audio_time * params.sample_rate - params.segment_start_frame; + if frame_f < 0.0 || frame_f >= params.total_frames { + discard; + } + + // Determine mip level based on how many audio frames map to one pixel + let frames_per_pixel = params.sample_rate / params.pixels_per_second; + // Each mip level reduces by 4x in sample count (2x in each texture dimension) + let mip_f = max(0.0, log2(frames_per_pixel) / 2.0); + let max_mip = f32(textureNumLevels(peak_tex) - 1u); + let mip = min(mip_f, max_mip); + + // Frame index at the chosen mip level + let mip_floor = u32(mip); + let reduction = pow(4.0, f32(mip_floor)); + let mip_frame = frame_f / reduction; + + // Convert 1D mip-space index to 2D UV coordinates + let mip_tex_width = params.tex_width / pow(2.0, f32(mip_floor)); + let mip_tex_height = ceil(params.total_frames / reduction / mip_tex_width); + let texel_x = mip_frame % mip_tex_width; + let texel_y = floor(mip_frame / mip_tex_width); + let uv = vec2((texel_x + 0.5) / mip_tex_width, (texel_y + 0.5) / mip_tex_height); + + // Sample the peak texture at computed mip level + // R = left_min, G = left_max, B = right_min, A = right_max + let peak = textureSampleLevel(peak_tex, peak_sampler, uv, mip); + + let clip_height = params.clip_rect.w - params.clip_rect.y; + let clip_top = params.clip_rect.y; + + if params.display_mode < 0.5 { + // Combined mode: merge both channels + let wave_min = min(peak.r, peak.b); + let wave_max = max(peak.g, peak.a); + + let center_y = clip_top + clip_height * 0.5; + let scale = clip_height * 0.45; + + let y_top = center_y - wave_max * scale; + let y_bot = center_y - wave_min * scale; + + // At least 1px tall for visibility + let y_top_adj = min(y_top, center_y - 0.5); + let y_bot_adj = max(y_bot, center_y + 0.5); + + if frag_y >= y_top_adj && frag_y <= y_bot_adj { + return params.tint_color; + } + } else { + // Split stereo mode: left channel in top half, right channel in bottom half + let half_height = clip_height * 0.5; + let mid_y = clip_top + half_height; + + // Determine which channel this fragment belongs to + if frag_y < mid_y { + // Top half: left channel + let center_y = clip_top + half_height * 0.5; + let scale = half_height * 0.45; + + let y_top = center_y - peak.g * scale; // left_max + let y_bot = center_y - peak.r * scale; // left_min + + let y_top_adj = min(y_top, center_y - 0.5); + let y_bot_adj = max(y_bot, center_y + 0.5); + + if frag_y >= y_top_adj && frag_y <= y_bot_adj { + return params.tint_color; + } + } else { + // Bottom half: right channel + let center_y = mid_y + half_height * 0.5; + let scale = half_height * 0.45; + + let y_top = center_y - peak.a * scale; // right_max + let y_bot = center_y - peak.b * scale; // right_min + + let y_top_adj = min(y_top, center_y - 0.5); + let y_bot_adj = max(y_bot, center_y + 0.5); + + if frag_y >= y_top_adj && frag_y <= y_bot_adj { + return params.tint_color; + } + } + } + + discard; +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform_mipgen.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform_mipgen.wgsl new file mode 100644 index 0000000..975223a --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform_mipgen.wgsl @@ -0,0 +1,64 @@ +// Min/max mipmap generation for 1D audio data packed into 2D textures. +// Each dest texel reduces 4 CONSECUTIVE source texels in audio order, +// using 1Dβ†’2D coordinate conversion since the 2D layout is linearized 1D. +// Row wrapping is handled by the modulo/division mapping: +// x = index % width, y = index / width +// So sample index 2048 with width=2048 maps to (0, 1), not (0, 0). +// +// Texture format: Rgba16Float +// R = left_min, G = left_max, B = right_min, A = right_max +// At mip 0 (raw samples): R=G=left_sample, B=A=right_sample + +struct MipParams { + src_width: u32, + dst_width: u32, + src_sample_count: u32, // valid texels in source level + _pad: u32, +} + +@group(0) @binding(0) var src_mip: texture_2d; +@group(0) @binding(1) var dst_mip: texture_storage_2d; +@group(0) @binding(2) var params: MipParams; + +@compute @workgroup_size(64) +fn main(@builtin(global_invocation_id) id: vec3) { + // Each thread handles one dest texel + let dst_1d = id.x; + let dst_size = textureDimensions(dst_mip); + if dst_1d >= dst_size.x * dst_size.y { + return; + } + + let dst_x = dst_1d % params.dst_width; + let dst_y = dst_1d / params.dst_width; + + // Map to 4 consecutive source texels in 1D audio order + let src_base = dst_1d * 4u; + var result = vec4(0.0); + var initialized = false; + + for (var i = 0u; i < 4u; i++) { + let src_1d = src_base + i; + if src_1d >= params.src_sample_count { + break; + } + // 1D β†’ 2D: wraps across rows naturally + let src_x = src_1d % params.src_width; + let src_y = src_1d / params.src_width; + let s = textureLoad(src_mip, vec2(src_x, src_y), 0); + + if !initialized { + result = s; + initialized = true; + } else { + result = vec4( + min(result.r, s.r), // left_min + max(result.g, s.g), // left_max + min(result.b, s.b), // right_min + max(result.a, s.a), // right_max + ); + } + } + + textureStore(dst_mip, vec2(dst_x, dst_y), result); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index 1fc4765..5fe0852 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -607,311 +607,6 @@ impl TimelinePane { } } - /// Calculate which waveform tiles are visible in the viewport - fn calculate_visible_tiles( - audio_pool_index: usize, - clip_start_time: f64, - clip_duration: f64, - clip_rect: egui::Rect, - timeline_left_edge: f32, - viewport_start_time: f64, - pixels_per_second: f64, - zoom_bucket: u32, - height: u32, - trim_start: f64, - audio_file_duration: f64, - ) -> Vec { - use crate::waveform_image_cache::{WaveformCacheKey, TILE_WIDTH_PIXELS}; - - // Calculate clip position in screen space (including timeline offset) - let clip_start_x = timeline_left_edge + ((clip_start_time - viewport_start_time) * pixels_per_second) as f32; - let clip_width = (clip_duration * pixels_per_second) as f32; - - // Check if clip is visible - if clip_start_x + clip_width < clip_rect.min.x || clip_start_x > clip_rect.max.x { - return vec![]; // Clip not visible - } - - // Calculate tile duration in seconds (based on zoom bucket, not current pixels_per_second) - let seconds_per_pixel_in_tile = 1.0 / zoom_bucket as f64; - let tile_duration_seconds = TILE_WIDTH_PIXELS as f64 * seconds_per_pixel_in_tile; - - // Calculate visible time range within the clip - let visible_start_pixel = (clip_rect.min.x - clip_start_x).max(0.0); - let visible_end_pixel = (clip_rect.max.x - clip_start_x).min(clip_width); - - // Convert screen pixels to time within clip - let visible_start_time_in_clip = (visible_start_pixel as f64) / pixels_per_second; - let visible_end_time_in_clip = (visible_end_pixel as f64) / pixels_per_second; - - // Convert to audio file coordinates (tiles are indexed by audio file position) - let visible_audio_start = trim_start + visible_start_time_in_clip; - let visible_audio_end = (trim_start + visible_end_time_in_clip).min(audio_file_duration); - - // Calculate which tiles from the audio file cover this range - let start_tile = (visible_audio_start / tile_duration_seconds).floor() as u32; - let end_tile = ((visible_audio_end / tile_duration_seconds).ceil() as u32).max(start_tile + 1); - - // Generate cache keys for visible tiles - let mut keys = Vec::new(); - for tile_idx in start_tile..end_tile { - keys.push(WaveformCacheKey { - audio_pool_index, - zoom_bucket, - tile_index: tile_idx, - height, - }); - } - - keys - } - - /// Calculate tiles for pre-caching (1-2 screens ahead/behind) - fn calculate_precache_tiles( - visible_tiles: &[crate::waveform_image_cache::WaveformCacheKey], - viewport_width_pixels: f32, - ) -> Vec { - use crate::waveform_image_cache::{WaveformCacheKey, TILE_WIDTH_PIXELS}; - - if visible_tiles.is_empty() { - return vec![]; - } - - // Calculate how many tiles = 1-2 screens - let tiles_per_screen = ((viewport_width_pixels as usize + TILE_WIDTH_PIXELS - 1) - / TILE_WIDTH_PIXELS) as u32; - let precache_count = tiles_per_screen * 2; // 2 screens worth - - let first_visible = visible_tiles.first().unwrap(); - let last_visible = visible_tiles.last().unwrap(); - - let mut precache = Vec::new(); - - // Tiles before viewport - for i in 1..=precache_count { - if let Some(tile_idx) = first_visible.tile_index.checked_sub(i) { - precache.push(WaveformCacheKey { - audio_pool_index: first_visible.audio_pool_index, - zoom_bucket: first_visible.zoom_bucket, - tile_index: tile_idx, - height: first_visible.height, - }); - } - } - - // Tiles after viewport (with bounds check based on clip duration) - for i in 1..=precache_count { - let tile_idx = last_visible.tile_index + i; - precache.push(WaveformCacheKey { - audio_pool_index: first_visible.audio_pool_index, - zoom_bucket: first_visible.zoom_bucket, - tile_index: tile_idx, - height: first_visible.height, - }); - } - - precache - } - - /// Select appropriate detail level based on zoom (pixels per second) - /// - /// Detail levels: - /// - Level 0 (Overview): 1 peak/sec - for extreme zoom out (0-2 pps) - /// - Level 1 (Low): 10 peaks/sec - for zoomed out view (2-20 pps) - /// - Level 2 (Medium): 100 peaks/sec - for normal view (20-200 pps) - /// - Level 3 (High): 1000 peaks/sec - for zoomed in (200-2000 pps) - /// - Level 4 (Max): Full resolution - for maximum zoom (>2000 pps) - fn select_detail_level(pixels_per_second: f64) -> u8 { - if pixels_per_second < 2.0 { - 0 // Overview - } else if pixels_per_second < 20.0 { - 1 // Low - } else if pixels_per_second < 200.0 { - 2 // Medium - } else if pixels_per_second < 2000.0 { - 3 // High - } else { - 4 // Max (full resolution) - } - } - - /// Assemble waveform peaks from chunks for the ENTIRE audio file - /// - /// Returns peaks for the entire audio file, or None if chunks are not available - /// This assembles a complete waveform from chunks at the appropriate detail level - fn assemble_peaks_from_chunks( - waveform_chunk_cache: &std::collections::HashMap<(usize, u8, u32), Vec>, - audio_pool_index: usize, - detail_level: u8, - audio_file_duration: f64, - audio_controller: Option<&std::sync::Arc>>, - ) -> Option> { - // Calculate chunk time span based on detail level - let chunk_time_span = match detail_level { - 0 => 60.0, // Level 0: 60 seconds per chunk - 1 => 30.0, // Level 1: 30 seconds per chunk - 2 => 10.0, // Level 2: 10 seconds per chunk - 3 => 5.0, // Level 3: 5 seconds per chunk - 4 => 1.0, // Level 4: 1 second per chunk - _ => 10.0, // Default - }; - - // Calculate total number of chunks needed for entire audio file - let total_chunks = (audio_file_duration / chunk_time_span).ceil() as u32; - - let mut assembled_peaks = Vec::new(); - let mut missing_chunks = Vec::new(); - - // Check if all required chunks are available - for chunk_idx in 0..total_chunks { - let key = (audio_pool_index, detail_level, chunk_idx); - if let Some(chunk_peaks) = waveform_chunk_cache.get(&key) { - assembled_peaks.extend_from_slice(chunk_peaks); - } else { - // Track missing chunk - missing_chunks.push(chunk_idx); - } - } - - // If any chunks are missing, request them and return None - if !missing_chunks.is_empty() { - if let Some(controller_arc) = audio_controller { - let mut controller = controller_arc.lock().unwrap(); - let _ = controller.generate_waveform_chunks( - audio_pool_index, - detail_level, - missing_chunks, - 1, // Medium priority - ); - } - return None; - } - - Some(assembled_peaks) - } - - /// Render waveform visualization using cached texture tiles - /// This is much faster than line-based rendering for many clips - #[allow(clippy::too_many_arguments)] - fn render_audio_waveform( - painter: &egui::Painter, - clip_rect: egui::Rect, - timeline_left_edge: f32, - audio_pool_index: usize, - clip_start_time: f64, - clip_duration: f64, - trim_start: f64, - audio_file_duration: f64, - viewport_start_time: f64, - pixels_per_second: f64, - waveform_image_cache: &mut crate::waveform_image_cache::WaveformImageCache, - waveform_peaks: &[daw_backend::WaveformPeak], - ctx: &egui::Context, - tint_color: egui::Color32, - ) { - use crate::waveform_image_cache::{calculate_zoom_bucket, TILE_WIDTH_PIXELS}; - - if waveform_peaks.is_empty() { - return; - } - - // Calculate zoom bucket - let zoom_bucket = calculate_zoom_bucket(pixels_per_second); - - // Calculate visible tiles - let visible_tiles = Self::calculate_visible_tiles( - audio_pool_index, - clip_start_time, - clip_duration, - clip_rect, - timeline_left_edge, - viewport_start_time, - pixels_per_second, - zoom_bucket, - clip_rect.height() as u32, - trim_start, - audio_file_duration, - ); - - // Render each tile - for key in &visible_tiles { - let texture = waveform_image_cache.get_or_create( - *key, - ctx, - waveform_peaks, - audio_file_duration, - ); - - // Calculate tile position in audio file (tiles now represent fixed portions of the audio file) - // Each pixel in the tile texture represents (1.0 / zoom_bucket) seconds - let seconds_per_pixel_in_tile = 1.0 / key.zoom_bucket as f64; - let tile_audio_start = key.tile_index as f64 * TILE_WIDTH_PIXELS as f64 * seconds_per_pixel_in_tile; - let tile_audio_end = tile_audio_start + TILE_WIDTH_PIXELS as f64 * seconds_per_pixel_in_tile; - - // Calculate which portion of this tile is visible in the trimmed clip - let visible_audio_start = tile_audio_start.max(trim_start); - let visible_audio_end = tile_audio_end.min(trim_start + clip_duration).min(audio_file_duration); - - if visible_audio_start >= visible_audio_end { - continue; // No visible portion - } - - // Calculate UV coordinates (only show the portion within trim bounds) - let uv_min_x = ((visible_audio_start - tile_audio_start) / (tile_audio_end - tile_audio_start)).max(0.0) as f32; - let uv_max_x = ((visible_audio_end - tile_audio_start) / (tile_audio_end - tile_audio_start)).min(1.0) as f32; - - // Map audio file position to timeline position - // Audio time trim_start corresponds to timeline position clip_start_time - let tile_timeline_start = clip_start_time + (visible_audio_start - trim_start); - let tile_timeline_end = clip_start_time + (visible_audio_end - trim_start); - - // Convert to screen space - let tile_screen_x = timeline_left_edge + ((tile_timeline_start - viewport_start_time) * pixels_per_second) as f32; - let tile_screen_width = ((tile_timeline_end - tile_timeline_start) * pixels_per_second) as f32; - - // Create unclipped tile rect - let unclipped_tile_rect = egui::Rect::from_min_size( - egui::pos2(tile_screen_x, clip_rect.min.y), - egui::vec2(tile_screen_width, clip_rect.height()), - ); - - // Clip to the visible clip rectangle - let tile_rect = unclipped_tile_rect.intersect(clip_rect); - - if tile_rect.width() <= 0.0 || tile_rect.height() <= 0.0 { - continue; // Nothing visible - } - - // Adjust UV coordinates based on how much the tile was clipped - let uv_span = uv_max_x - uv_min_x; - let adjusted_uv_min_x = if unclipped_tile_rect.width() > 0.0 { - uv_min_x + ((tile_rect.min.x - unclipped_tile_rect.min.x) / unclipped_tile_rect.width()) * uv_span - } else { - uv_min_x - }; - let adjusted_uv_max_x = if unclipped_tile_rect.width() > 0.0 { - uv_min_x + ((tile_rect.max.x - unclipped_tile_rect.min.x) / unclipped_tile_rect.width()) * uv_span - } else { - uv_max_x - }; - - // Blit texture with adjusted UV coordinates - painter.image( - texture.id(), - tile_rect, - egui::Rect::from_min_max(egui::pos2(adjusted_uv_min_x, 0.0), egui::pos2(adjusted_uv_max_x, 1.0)), - tint_color, - ); - } - - // Pre-cache adjacent tiles (non-blocking) - let precache_tiles = Self::calculate_precache_tiles(&visible_tiles, clip_rect.width()); - // Create temporary HashMap with just this clip's waveform for pre-caching - let mut temp_waveform_cache = std::collections::HashMap::new(); - temp_waveform_cache.insert(audio_pool_index, waveform_peaks.to_vec()); - waveform_image_cache.precache_tiles(&precache_tiles, ctx, &temp_waveform_cache, audio_file_duration); - } - /// Render layer header column (left side with track names and controls) fn render_layer_headers( &mut self, @@ -1198,10 +893,9 @@ impl TimelinePane { active_layer_id: &Option, selection: &lightningbeam_core::selection::Selection, midi_event_cache: &std::collections::HashMap>, - waveform_cache: &std::collections::HashMap>, - waveform_chunk_cache: &std::collections::HashMap<(usize, u8, u32), Vec>, - waveform_image_cache: &mut crate::waveform_image_cache::WaveformImageCache, - audio_controller: Option<&std::sync::Arc>>, + raw_audio_cache: &std::collections::HashMap, u32, u32)>, + waveform_gpu_dirty: &mut std::collections::HashSet, + target_format: wgpu::TextureFormat, ) -> Vec<(egui::Rect, uuid::Uuid, f64, f64)> { let painter = ui.painter(); @@ -1492,59 +1186,68 @@ impl TimelinePane { ); } } - // Sampled Audio: Draw waveform + // Sampled Audio: Draw waveform via GPU lightningbeam_core::clip::AudioClipType::Sampled { audio_pool_index } => { - // Get audio file duration from backend - let audio_file_duration = if let Some(ref controller_arc) = audio_controller { - let mut controller = controller_arc.lock().unwrap(); - controller.get_pool_file_info(*audio_pool_index) - .ok() - .map(|(duration, _, _)| duration) - .unwrap_or(clip.duration) // Fallback to clip duration - } else { - clip.duration // Fallback if no controller - }; + if let Some((samples, sr, ch)) = raw_audio_cache.get(audio_pool_index) { + let total_frames = samples.len() / (*ch).max(1) as usize; + let audio_file_duration = total_frames as f64 / *sr as f64; + let screen_size = ui.ctx().screen_rect().size(); - // Select detail level based on zoom - let requested_level = Self::select_detail_level(self.pixels_per_second as f64); + let pending_upload = if waveform_gpu_dirty.contains(audio_pool_index) { + waveform_gpu_dirty.remove(audio_pool_index); + Some(crate::waveform_gpu::PendingUpload { + samples: samples.clone(), + sample_rate: *sr, + channels: *ch, + }) + } else { + None + }; - // Try to assemble peaks from chunks with progressive fallback to lower detail levels - let mut peaks_to_render = None; - for level in (0..=requested_level).rev() { - if let Some(peaks) = Self::assemble_peaks_from_chunks( - waveform_chunk_cache, - *audio_pool_index, - level, - audio_file_duration, - audio_controller, - ) { - peaks_to_render = Some(peaks); - break; - } - } + let tint = [ + bright_color.r() as f32 / 255.0, + bright_color.g() as f32 / 255.0, + bright_color.b() as f32 / 255.0, + bright_color.a() as f32 / 255.0, + ]; - // Final fallback to old waveform_cache if no chunks available at any level - let peaks_to_render = peaks_to_render - .or_else(|| waveform_cache.get(audio_pool_index).cloned()) - .unwrap_or_default(); - - if !peaks_to_render.is_empty() { - Self::render_audio_waveform( - painter, - clip_rect, - rect.min.x, - *audio_pool_index, - instance_start, - preview_clip_duration, - preview_trim_start, - audio_file_duration, - self.viewport_start_time, - self.pixels_per_second as f64, - waveform_image_cache, - &peaks_to_render, - ui.ctx(), - bright_color, // Use bright color for waveform (lighter than background) + let clip_screen_start = rect.min.x + ((instance_start - self.viewport_start_time) * self.pixels_per_second as f64) as f32; + let clip_screen_end = clip_screen_start + (preview_clip_duration * self.pixels_per_second as f64) as f32; + let waveform_rect = egui::Rect::from_min_max( + egui::pos2(clip_screen_start.max(clip_rect.min.x), clip_rect.min.y), + egui::pos2(clip_screen_end.min(clip_rect.max.x), clip_rect.max.y), ); + + if waveform_rect.width() > 0.0 && waveform_rect.height() > 0.0 { + let callback = crate::waveform_gpu::WaveformCallback { + pool_index: *audio_pool_index, + segment_index: 0, + params: crate::waveform_gpu::WaveformParams { + clip_rect: [waveform_rect.min.x, waveform_rect.min.y, waveform_rect.max.x, waveform_rect.max.y], + viewport_start_time: self.viewport_start_time as f32, + pixels_per_second: self.pixels_per_second as f32, + audio_duration: audio_file_duration as f32, + sample_rate: *sr as f32, + clip_start_time: instance_start as f32, + trim_start: preview_trim_start as f32, + tex_width: crate::waveform_gpu::tex_width() as f32, + total_frames: total_frames as f32, + segment_start_frame: 0.0, + display_mode: 0.0, + _pad1: [0.0, 0.0], + tint_color: tint, + screen_size: [screen_size.x, screen_size.y], + _pad: [0.0, 0.0], + }, + target_format, + pending_upload, + }; + + ui.painter().add(egui_wgpu::Callback::new_paint_callback( + waveform_rect, + callback, + )); + } } } // Recording in progress: no visualization yet @@ -2375,7 +2078,7 @@ impl PaneRenderer for TimelinePane { // Render layer rows with clipping ui.set_clip_rect(content_rect.intersect(original_clip_rect)); - let video_clip_hovers = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.selection, shared.midi_event_cache, shared.waveform_cache, shared.waveform_chunk_cache, shared.waveform_image_cache, shared.audio_controller); + let video_clip_hovers = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.selection, shared.midi_event_cache, shared.raw_audio_cache, shared.waveform_gpu_dirty, shared.target_format); // Render playhead on top (clip to timeline area) ui.set_clip_rect(timeline_rect.intersect(original_clip_rect)); diff --git a/lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs b/lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs new file mode 100644 index 0000000..afe6bd1 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs @@ -0,0 +1,624 @@ +/// GPU-based waveform rendering using 2D textures with custom min/max mipmaps. +/// +/// Raw audio samples are packed into Rgba16Float textures (R=left_min, G=left_max, +/// B=right_min, A=right_max). At mip 0, min=max=raw sample. Higher mip levels +/// are generated by a compute shader that reduces 4 consecutive samples per level. +/// +/// Audio frames are packed row-major into 2D textures with a fixed power-of-2 width. +/// Long audio is split across multiple textures. + +use std::collections::HashMap; +use wgpu; +use wgpu::util::DeviceExt; + +/// Fixed texture width (power of 2) for all waveform textures +const TEX_WIDTH: u32 = 2048; + +/// Maximum number of texture segments per audio clip +const MAX_SEGMENTS: u32 = 16; + +/// GPU resources for all waveform textures, stored in CallbackResources +pub struct WaveformGpuResources { + /// Per-audio-pool-index GPU data + pub entries: HashMap, + /// Shared render pipeline + render_pipeline: wgpu::RenderPipeline, + /// Shared mipgen compute pipeline + mipgen_pipeline: wgpu::ComputePipeline, + /// Bind group layout for render shader (texture + sampler + uniforms) + render_bind_group_layout: wgpu::BindGroupLayout, + /// Bind group layout for mipgen shader (src texture + dst storage + params) + mipgen_bind_group_layout: wgpu::BindGroupLayout, + /// Sampler for waveform texture (nearest, since we do manual LOD selection) + sampler: wgpu::Sampler, +} + +/// GPU data for a single audio file +pub struct WaveformGpuEntry { + /// One texture per segment (for long audio split across multiple textures) + pub textures: Vec, + /// Texture views for each segment (full mip chain) + pub texture_views: Vec, + /// Bind groups for the render shader (one per segment) + pub render_bind_groups: Vec, + /// Uniform buffers for each segment (updated per-frame via queue.write_buffer) + pub uniform_buffers: Vec, + /// Frames covered by each texture segment + pub frames_per_segment: u32, + /// Total frame count + pub total_frames: u64, + /// Sample rate + pub sample_rate: u32, + /// Number of channels in source audio + pub channels: u32, +} + +/// Parameters passed to the waveform render shader +#[repr(C)] +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +pub struct WaveformParams { + pub clip_rect: [f32; 4], + pub viewport_start_time: f32, + pub pixels_per_second: f32, + pub audio_duration: f32, + pub sample_rate: f32, + pub clip_start_time: f32, + pub trim_start: f32, + pub tex_width: f32, + pub total_frames: f32, + pub segment_start_frame: f32, + pub display_mode: f32, + pub _pad1: [f32; 2], // align tint_color to 16 bytes (WGSL vec4 alignment) + pub tint_color: [f32; 4], + pub screen_size: [f32; 2], + pub _pad: [f32; 2], +} + +/// Parameters for the mipgen compute shader +#[repr(C)] +#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] +struct MipgenParams { + src_width: u32, + dst_width: u32, + src_sample_count: u32, + _pad: u32, +} + +/// Callback for rendering a single waveform segment +pub struct WaveformCallback { + pub pool_index: usize, + pub segment_index: usize, + pub params: WaveformParams, + pub target_format: wgpu::TextureFormat, + /// Raw audio data for upload if this is the first time we see this pool_index + pub pending_upload: Option, +} + +/// Raw audio data waiting to be uploaded to GPU +pub struct PendingUpload { + pub samples: Vec, + pub sample_rate: u32, + pub channels: u32, +} + +impl WaveformGpuResources { + pub fn new(device: &wgpu::Device, target_format: wgpu::TextureFormat) -> Self { + // Render shader + let render_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("waveform_render_shader"), + source: wgpu::ShaderSource::Wgsl(include_str!("panes/shaders/waveform.wgsl").into()), + }); + + // Mipgen compute shader + let mipgen_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("waveform_mipgen_shader"), + source: wgpu::ShaderSource::Wgsl( + include_str!("panes/shaders/waveform_mipgen.wgsl").into(), + ), + }); + + // Render bind group layout: texture + sampler + uniform buffer + let render_bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("waveform_render_bgl"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + + // Mipgen bind group layout: src texture + dst storage texture + params + let mipgen_bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("waveform_mipgen_bgl"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::StorageTexture { + access: wgpu::StorageTextureAccess::WriteOnly, + format: wgpu::TextureFormat::Rgba16Float, + view_dimension: wgpu::TextureViewDimension::D2, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + + // Render pipeline + let render_pipeline_layout = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("waveform_render_pipeline_layout"), + bind_group_layouts: &[&render_bind_group_layout], + push_constant_ranges: &[], + }); + + let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("waveform_render_pipeline"), + layout: Some(&render_pipeline_layout), + vertex: wgpu::VertexState { + module: &render_shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &render_shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: target_format, + blend: Some(wgpu::BlendState::ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + // Mipgen compute pipeline + let mipgen_pipeline_layout = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("waveform_mipgen_pipeline_layout"), + bind_group_layouts: &[&mipgen_bind_group_layout], + push_constant_ranges: &[], + }); + + let mipgen_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some("waveform_mipgen_pipeline"), + layout: Some(&mipgen_pipeline_layout), + module: &mipgen_shader, + entry_point: Some("main"), + compilation_options: Default::default(), + cache: None, + }); + + // Sampler: nearest filtering for explicit mip level selection + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("waveform_sampler"), + mag_filter: wgpu::FilterMode::Nearest, + min_filter: wgpu::FilterMode::Nearest, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + + Self { + entries: HashMap::new(), + render_pipeline, + mipgen_pipeline, + render_bind_group_layout, + mipgen_bind_group_layout, + sampler, + } + } + + /// Upload raw audio samples and generate mipmaps for a given pool index. + /// Returns command buffers that need to be submitted (for mipmap compute dispatches). + pub fn upload_audio( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + pool_index: usize, + samples: &[f32], + sample_rate: u32, + channels: u32, + ) -> Vec { + // Remove old entry if exists + self.entries.remove(&pool_index); + + let total_frames = samples.len() / channels.max(1) as usize; + if total_frames == 0 { + return Vec::new(); + } + + let max_frames_per_segment = (TEX_WIDTH as u64) + * (device.limits().max_texture_dimension_2d as u64); + let segment_count = + ((total_frames as u64 + max_frames_per_segment - 1) / max_frames_per_segment) as usize; + let frames_per_segment = if segment_count == 1 { + total_frames as u32 + } else { + (max_frames_per_segment as u32).min(total_frames as u32) + }; + + let mut textures = Vec::new(); + let mut texture_views = Vec::new(); + let mut render_bind_groups = Vec::new(); + let mut uniform_buffers = Vec::new(); + let mut all_command_buffers = Vec::new(); + + for seg in 0..segment_count { + let seg_start_frame = seg as u64 * frames_per_segment as u64; + let seg_end_frame = ((seg + 1) as u64 * frames_per_segment as u64) + .min(total_frames as u64); + let seg_frame_count = (seg_end_frame - seg_start_frame) as u32; + + let tex_height = (seg_frame_count + TEX_WIDTH - 1) / TEX_WIDTH; + let mip_count = compute_mip_count(TEX_WIDTH, tex_height); + + // Create texture with mip levels + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some(&format!("waveform_{}_seg{}", pool_index, seg)), + size: wgpu::Extent3d { + width: TEX_WIDTH, + height: tex_height, + depth_or_array_layers: 1, + }, + mip_level_count: mip_count, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::STORAGE_BINDING + | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + + // Pack raw samples into Rgba16Float data for mip 0 + // R=left_min=left_sample, G=left_max=left_sample, B=right_min, A=right_max + let texel_count = (TEX_WIDTH * tex_height) as usize; + let mut mip0_data: Vec = vec![half::f16::ZERO; texel_count * 4]; + + for frame in 0..seg_frame_count as usize { + let global_frame = seg_start_frame as usize + frame; + let sample_offset = global_frame * channels as usize; + + let left = if sample_offset < samples.len() { + samples[sample_offset] + } else { + 0.0 + }; + let right = if channels >= 2 && sample_offset + 1 < samples.len() { + samples[sample_offset + 1] + } else { + left // Mono: duplicate left to right + }; + + let texel_offset = frame * 4; + mip0_data[texel_offset] = half::f16::from_f32(left); // R = left_min + mip0_data[texel_offset + 1] = half::f16::from_f32(left); // G = left_max + mip0_data[texel_offset + 2] = half::f16::from_f32(right); // B = right_min + mip0_data[texel_offset + 3] = half::f16::from_f32(right); // A = right_max + } + + // Upload mip 0 + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + bytemuck::cast_slice(&mip0_data), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(TEX_WIDTH * 8), // 4 channels Γ— 2 bytes (f16) + rows_per_image: Some(tex_height), + }, + wgpu::Extent3d { + width: TEX_WIDTH, + height: tex_height, + depth_or_array_layers: 1, + }, + ); + + // Generate mipmaps via compute shader + let cmds = self.generate_mipmaps( + device, + &texture, + TEX_WIDTH, + tex_height, + mip_count, + seg_frame_count, + ); + all_command_buffers.extend(cmds); + + // Create view for full mip chain + let view = texture.create_view(&wgpu::TextureViewDescriptor { + label: Some(&format!("waveform_{}_seg{}_view", pool_index, seg)), + ..Default::default() + }); + + // Create uniform buffer placeholder (will be filled per-draw in paint) + let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some(&format!("waveform_{}_seg{}_uniforms", pool_index, seg)), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + // Create render bind group + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(&format!("waveform_{}_seg{}_bg", pool_index, seg)), + layout: &self.render_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&self.sampler), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: uniform_buffer.as_entire_binding(), + }, + ], + }); + + textures.push(texture); + texture_views.push(view); + render_bind_groups.push(bind_group); + uniform_buffers.push(uniform_buffer); + } + + self.entries.insert( + pool_index, + WaveformGpuEntry { + textures, + texture_views, + render_bind_groups, + uniform_buffers, + frames_per_segment, + total_frames: total_frames as u64, + sample_rate, + channels, + }, + ); + + all_command_buffers + } + + /// Generate mipmaps for a texture using the compute shader. + fn generate_mipmaps( + &self, + device: &wgpu::Device, + texture: &wgpu::Texture, + base_width: u32, + base_height: u32, + mip_count: u32, + base_sample_count: u32, + ) -> Vec { + if mip_count <= 1 { + return Vec::new(); + } + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("waveform_mipgen_encoder"), + }); + + let mut src_width = base_width; + let mut src_height = base_height; + let mut src_sample_count = base_sample_count; + + for level in 1..mip_count { + let dst_width = (src_width / 2).max(1); + let dst_height = (src_height / 2).max(1); + let dst_sample_count = (src_sample_count + 3) / 4; // ceil(src/4) + + // Create views for specific mip levels + let src_view = texture.create_view(&wgpu::TextureViewDescriptor { + label: Some(&format!("mipgen_src_level_{}", level - 1)), + base_mip_level: level - 1, + mip_level_count: Some(1), + ..Default::default() + }); + let dst_view = texture.create_view(&wgpu::TextureViewDescriptor { + label: Some(&format!("mipgen_dst_level_{}", level)), + base_mip_level: level, + mip_level_count: Some(1), + ..Default::default() + }); + + // Create params buffer + let params = MipgenParams { + src_width, + dst_width, + src_sample_count, + _pad: 0, + }; + let params_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some(&format!("mipgen_params_level_{}", level)), + contents: bytemuck::cast_slice(&[params]), + usage: wgpu::BufferUsages::UNIFORM, + }); + + // Create bind group for this dispatch + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some(&format!("mipgen_bg_level_{}", level)), + layout: &self.mipgen_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&src_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&dst_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: params_buffer.as_entire_binding(), + }, + ], + }); + + // Dispatch compute + let total_dst_texels = dst_width * dst_height; + let workgroup_count = (total_dst_texels + 63) / 64; + + let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { + label: Some(&format!("mipgen_pass_level_{}", level)), + timestamp_writes: None, + }); + pass.set_pipeline(&self.mipgen_pipeline); + pass.set_bind_group(0, &bind_group, &[]); + pass.dispatch_workgroups(workgroup_count, 1, 1); + drop(pass); + + src_width = dst_width; + src_height = dst_height; + src_sample_count = dst_sample_count; + } + + vec![encoder.finish()] + } +} + +impl egui_wgpu::CallbackTrait for WaveformCallback { + fn prepare( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + _screen_descriptor: &egui_wgpu::ScreenDescriptor, + _egui_encoder: &mut wgpu::CommandEncoder, + resources: &mut egui_wgpu::CallbackResources, + ) -> Vec { + // Initialize resources if needed + if !resources.contains::() { + resources.insert(WaveformGpuResources::new(device, self.target_format)); + } + + let gpu_resources: &mut WaveformGpuResources = resources.get_mut().unwrap(); + + // Upload audio data if pending + let mut cmds = Vec::new(); + if let Some(ref upload) = self.pending_upload { + let new_cmds = gpu_resources.upload_audio( + device, + queue, + self.pool_index, + &upload.samples, + upload.sample_rate, + upload.channels, + ); + cmds.extend(new_cmds); + } + + // Update uniform buffer for this draw + if let Some(entry) = gpu_resources.entries.get(&self.pool_index) { + if self.segment_index < entry.uniform_buffers.len() { + queue.write_buffer( + &entry.uniform_buffers[self.segment_index], + 0, + bytemuck::cast_slice(&[self.params]), + ); + } + } + + cmds + } + + fn paint( + &self, + _info: eframe::egui::PaintCallbackInfo, + render_pass: &mut wgpu::RenderPass<'static>, + resources: &egui_wgpu::CallbackResources, + ) { + let gpu_resources: &WaveformGpuResources = match resources.get() { + Some(r) => r, + None => return, + }; + + let entry = match gpu_resources.entries.get(&self.pool_index) { + Some(e) => e, + None => return, + }; + + if self.segment_index >= entry.render_bind_groups.len() { + return; + } + + render_pass.set_pipeline(&gpu_resources.render_pipeline); + render_pass.set_bind_group(0, &entry.render_bind_groups[self.segment_index], &[]); + render_pass.draw(0..3, 0..1); // Fullscreen triangle + } +} + +/// Compute number of mip levels for given dimensions +fn compute_mip_count(width: u32, height: u32) -> u32 { + let max_dim = width.max(height); + (max_dim as f32).log2().floor() as u32 + 1 +} + +/// Calculate how many texture segments are needed for a given frame count +pub fn segment_count_for_frames(total_frames: u64, max_texture_height: u32) -> u32 { + let max_frames_per_segment = TEX_WIDTH as u64 * max_texture_height as u64; + ((total_frames + max_frames_per_segment - 1) / max_frames_per_segment) as u32 +} + +/// Get the fixed texture width used for all waveform textures +pub fn tex_width() -> u32 { + TEX_WIDTH +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/waveform_image_cache.rs b/lightningbeam-ui/lightningbeam-editor/src/waveform_image_cache.rs deleted file mode 100644 index acee617..0000000 --- a/lightningbeam-ui/lightningbeam-editor/src/waveform_image_cache.rs +++ /dev/null @@ -1,326 +0,0 @@ -use eframe::egui; -use std::collections::{HashMap, VecDeque}; -use std::time::Instant; - -/// Tile width is constant at 1024 pixels per tile -pub const TILE_WIDTH_PIXELS: usize = 1024; - -/// Unique identifier for a cached waveform image tile -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct WaveformCacheKey { - /// Audio pool index from backend - pub audio_pool_index: usize, - /// Zoom bucket (power of 2: 1, 2, 4, 8, 16, etc.) - pub zoom_bucket: u32, - /// Tile index (which tile in the sequence for this audio clip) - pub tile_index: u32, - /// Clip height in pixels (for cache invalidation on resize) - pub height: u32, -} - -/// Cached waveform image with metadata -pub struct CachedWaveform { - /// The rendered texture handle - pub texture: egui::TextureHandle, - /// Size in bytes (for memory tracking) - pub size_bytes: usize, - /// Last access time (for LRU eviction) - pub last_accessed: Instant, - /// Width of the image in pixels - pub width_pixels: u32, - /// Height of the image in pixels - pub height_pixels: u32, -} - -/// Main cache structure -pub struct WaveformImageCache { - /// Map from cache key to rendered texture - cache: HashMap, - /// LRU queue (most recent at back) - lru_queue: VecDeque, - /// Current total memory usage in bytes - total_bytes: usize, - /// Maximum memory usage (100 MB default) - max_bytes: usize, - /// Statistics - hits: u64, - misses: u64, -} - -impl WaveformImageCache { - /// Create a new waveform image cache with 100 MB limit - pub fn new() -> Self { - Self { - cache: HashMap::new(), - lru_queue: VecDeque::new(), - total_bytes: 0, - max_bytes: 100 * 1024 * 1024, // 100 MB - hits: 0, - misses: 0, - } - } - - /// Clear all cached textures - pub fn clear(&mut self) { - self.cache.clear(); - self.lru_queue.clear(); - self.total_bytes = 0; - // Note: hits/misses preserved for debugging - } - - /// Get cache statistics: (hits, misses, total_bytes, num_entries) - pub fn stats(&self) -> (u64, u64, usize, usize) { - (self.hits, self.misses, self.total_bytes, self.cache.len()) - } - - /// Evict least recently used entries until under memory limit - fn evict_lru(&mut self) { - while self.total_bytes > self.max_bytes && !self.lru_queue.is_empty() { - if let Some(key) = self.lru_queue.pop_front() { - if let Some(cached) = self.cache.remove(&key) { - self.total_bytes -= cached.size_bytes; - // Texture automatically freed when CachedWaveform dropped - } - } - } - } - - /// Update LRU queue when a key is accessed - fn touch(&mut self, key: WaveformCacheKey) { - // Remove key from its current position in LRU queue - self.lru_queue.retain(|&k| k != key); - // Add to back (most recent) - self.lru_queue.push_back(key); - } - - /// Get cached texture or generate new one - pub fn get_or_create( - &mut self, - key: WaveformCacheKey, - ctx: &egui::Context, - waveform: &[daw_backend::WaveformPeak], - audio_file_duration: f64, - ) -> egui::TextureHandle { - // Check if already cached - let texture = if let Some(cached) = self.cache.get_mut(&key) { - // Cache hit - self.hits += 1; - cached.last_accessed = Instant::now(); - Some(cached.texture.clone()) - } else { - None - }; - - if let Some(texture) = texture { - self.touch(key); - return texture; - } - - // Cache miss - generate new tile - self.misses += 1; - - // Render waveform to image - let color_image = render_waveform_to_image( - waveform, - key.tile_index, - audio_file_duration, - key.zoom_bucket, - key.height, - ); - - // Upload to GPU as texture - let texture_name = format!( - "waveform_{}_{}_{}", - key.audio_pool_index, key.zoom_bucket, key.tile_index - ); - let texture = ctx.load_texture( - texture_name, - color_image, - egui::TextureOptions::LINEAR, - ); - - // Calculate memory usage - let size_bytes = TILE_WIDTH_PIXELS * key.height as usize * 4; - - // Store in cache - let cached = CachedWaveform { - texture: texture.clone(), - size_bytes, - last_accessed: Instant::now(), - width_pixels: TILE_WIDTH_PIXELS as u32, - height_pixels: key.height, - }; - - self.total_bytes += size_bytes; - self.cache.insert(key, cached); - self.touch(key); - - // Evict if over limit - self.evict_lru(); - - texture - } - - /// Pre-cache tiles for smooth scrolling - pub fn precache_tiles( - &mut self, - keys: &[WaveformCacheKey], - ctx: &egui::Context, - waveform_peak_cache: &HashMap>, - audio_file_duration: f64, - ) { - // Limit pre-caching to avoid frame time spike - const MAX_PRECACHE_PER_FRAME: usize = 2; - - let mut precached = 0; - - for key in keys { - if precached >= MAX_PRECACHE_PER_FRAME { - break; - } - - // Skip if already cached - if self.cache.contains_key(key) { - continue; - } - - // Get waveform peaks - if let Some(waveform) = waveform_peak_cache.get(&key.audio_pool_index) { - // Generate and cache - let _ = self.get_or_create(*key, ctx, waveform, audio_file_duration); - precached += 1; - } - } - } - - /// Remove all entries for a specific audio file - pub fn invalidate_audio(&mut self, audio_pool_index: usize) { - let keys_to_remove: Vec = self - .cache - .keys() - .filter(|k| k.audio_pool_index == audio_pool_index) - .copied() - .collect(); - - for key in keys_to_remove { - if let Some(cached) = self.cache.remove(&key) { - self.total_bytes -= cached.size_bytes; - } - } - - // Also clean up LRU queue - self.lru_queue.retain(|key| key.audio_pool_index != audio_pool_index); - } - - /// Remove all entries with a specific height (for window resize) - pub fn invalidate_height(&mut self, old_height: u32) { - let keys_to_remove: Vec = self - .cache - .keys() - .filter(|k| k.height == old_height) - .copied() - .collect(); - - for key in keys_to_remove { - if let Some(cached) = self.cache.remove(&key) { - self.total_bytes -= cached.size_bytes; - } - } - - // Also clean up LRU queue - self.lru_queue.retain(|key| key.height != old_height); - } -} - -impl Default for WaveformImageCache { - fn default() -> Self { - Self::new() - } -} - -/// Calculate zoom bucket from pixels_per_second -/// Rounds to nearest power of 2: 1, 2, 4, 8, 16, 32, 64, 128, 256 -pub fn calculate_zoom_bucket(pixels_per_second: f64) -> u32 { - if pixels_per_second <= 1.0 { - return 1; - } - - // Round to nearest power of 2 - let log2 = pixels_per_second.log2(); - let rounded = log2.round(); - 2u32.pow(rounded as u32) -} - -/// Render a waveform tile to a ColorImage -fn render_waveform_to_image( - waveform: &[daw_backend::WaveformPeak], - tile_index: u32, - audio_file_duration: f64, - zoom_bucket: u32, - height: u32, -) -> egui::ColorImage { - let width = TILE_WIDTH_PIXELS; - let height = height as usize; - - // Create RGBA buffer (transparent background) - let mut pixels = vec![0u8; width * height * 4]; - - // Render as white - will be tinted at render time with clip background color - let waveform_color = egui::Color32::WHITE; - - // Calculate time range for this tile (tiles represent fixed portions of the audio file) - // Each pixel represents (1.0 / zoom_bucket) seconds - let seconds_per_pixel = 1.0 / zoom_bucket as f64; - let tile_start_time = tile_index as f64 * TILE_WIDTH_PIXELS as f64 * seconds_per_pixel; - let tile_end_time = (tile_start_time + width as f64 * seconds_per_pixel).min(audio_file_duration); - - // Calculate which waveform peaks correspond to this tile - let peak_start_idx = ((tile_start_time / audio_file_duration) * waveform.len() as f64) as usize; - let peak_end_idx = ((tile_end_time / audio_file_duration) * waveform.len() as f64) as usize; - let peak_end_idx = peak_end_idx.min(waveform.len()); - - if peak_start_idx >= waveform.len() { - // Tile is beyond the end of the audio clip - return transparent image - return egui::ColorImage::from_rgba_unmultiplied([width, height], &pixels); - } - - let tile_peaks = &waveform[peak_start_idx..peak_end_idx]; - if tile_peaks.is_empty() { - return egui::ColorImage::from_rgba_unmultiplied([width, height], &pixels); - } - - // Calculate the actual time range this tile covers in the audio file - // This may be less than the full tile width if the audio file is shorter than the tile's time span - let actual_time_covered = tile_end_time - tile_start_time; - let actual_pixel_width = (actual_time_covered / seconds_per_pixel).min(width as f64); - - // Render waveform to pixel buffer - // Distribute peaks only across the valid pixel range, not the entire tile width - let pixels_per_peak = actual_pixel_width / tile_peaks.len() as f64; - - for (peak_idx, peak) in tile_peaks.iter().enumerate() { - let x_start = (peak_idx as f64 * pixels_per_peak).floor() as usize; - let x_end = ((peak_idx + 1) as f64 * pixels_per_peak).ceil() as usize; - let x_end = x_end.min(width); - - // Calculate Y range for this peak - let center_y = height as f64 / 2.0; - let max_y = (center_y + (peak.max as f64 * height as f64 * 0.45)).round() as usize; - let min_y = (center_y + (peak.min as f64 * height as f64 * 0.45)).round() as usize; - let min_y = min_y.min(height - 1); - let max_y = max_y.min(height - 1); - - // Fill vertical span for this peak - for x in x_start..x_end { - for y in min_y..=max_y { - let pixel_idx = (y * width + x) * 4; - pixels[pixel_idx] = waveform_color.r(); - pixels[pixel_idx + 1] = waveform_color.g(); - pixels[pixel_idx + 2] = waveform_color.b(); - pixels[pixel_idx + 3] = waveform_color.a(); - } - } - } - - egui::ColorImage::from_rgba_unmultiplied([width, height], &pixels) -}