skyler
/
Lightningbeam
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Render audio waveforms on gpu

This commit is contained in:
Skyler Lehmkuhl 2026-02-11 14:38:58 -05:00
parent fc58f29ccd
commit 8ac5f52f28
12 changed files with 1100 additions and 855 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
daw-backend/src/audio/engine.rs
View File

@ -1812,6 +1812,16 @@ impl Engine {
None => QueryResponse::PoolFileInfo(Err(format!("Pool index {} not found", pool_index))),
}
}
Query::GetPoolAudioSamples(pool_index) => {
match self.audio_pool.get_file(pool_index) {
Some(file) => QueryResponse::PoolAudioSamples(Ok((
file.data.clone(),
file.sample_rate,
file.channels,
))),
None => QueryResponse::PoolAudioSamples(Err(format!("Pool index {} not found", pool_index))),
}
}
Query::ExportAudio(settings, output_path) => {
// Perform export directly - this will block the audio thread but that's okay
// since we're exporting and not playing back anyway
@ -2898,6 +2908,31 @@ impl EngineController {
Err("Query timeout".to_string())
}
/// Get raw audio samples from pool (samples, sample_rate, channels)
pub fn get_pool_audio_samples(&mut self, pool_index: usize) -> Result<(Vec<f32>, u32, u32), String> {
if let Err(_) = self.query_tx.push(Query::GetPoolAudioSamples(pool_index)) {
return Err("Failed to send query - queue full".to_string());
}
let start = std::time::Instant::now();
let timeout = std::time::Duration::from_secs(5); // Longer timeout for large audio data
while start.elapsed() < timeout {
if let Ok(response) = self.query_response_rx.pop() {
match response {
QueryResponse::PoolAudioSamples(result) => return result,
QueryResponse::AudioExported(result) => {
self.cached_export_response = Some(result);
}
_ => {}
}
}
std::thread::sleep(std::time::Duration::from_millis(1));
}
Err("Query timeout".to_string())
}
/// Request waveform chunks to be generated
/// This is an asynchronous command - chunks will be returned via WaveformChunksReady events
pub fn generate_waveform_chunks(

4
daw-backend/src/command/types.rs
View File

@ -299,6 +299,8 @@ pub enum Query {
AddAudioClipSync(TrackId, usize, f64, f64, f64),
/// Add an audio file to the pool synchronously (path, data, channels, sample_rate) - returns pool index
AddAudioFileSync(String, Vec<f32>, u32, u32),
/// Get raw audio samples from pool (pool_index) - returns (samples, sample_rate, channels)
GetPoolAudioSamples(usize),
/// Get a clone of the current project for serialization
GetProject,
/// Set the project (replaces current project state)
@ -368,6 +370,8 @@ pub enum QueryResponse {
AudioClipInstanceAdded(Result<AudioClipInstanceId, String>),
/// Audio file added to pool (returns pool index)
AudioFileAddedSync(Result<usize, String>),
/// Raw audio samples from pool (samples, sample_rate, channels)
PoolAudioSamples(Result<(Vec<f32>, u32, u32), String>),
/// Project retrieved
ProjectRetrieved(Result<Box<crate::audio::project::Project>, String>),
/// Project set

3
lightningbeam-ui/Cargo.lock generated
View File

@ -2867,6 +2867,7 @@ version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [
"bytemuck",
"cfg-if",
"crunchy",
"num-traits",
@ -3433,6 +3434,7 @@ dependencies = [
name = "lightningbeam-editor"
version = "0.1.0"
dependencies = [
"bytemuck",
"clap",
"cpal",
"daw-backend",
@ -3443,6 +3445,7 @@ dependencies = [
"egui_extras",
"egui_node_graph2",
"ffmpeg-next",
"half",
"image",
"kurbo 0.12.0",
"lightningbeam-core",

2
lightningbeam-ui/lightningbeam-editor/Cargo.toml
View File

@ -20,6 +20,8 @@ egui_node_graph2 = { git = "https://github.com/PVDoriginal/egui_node_graph2" }
# GPU
wgpu = { workspace = true }
vello = { workspace = true }
half = { version = "2.4", features = ["bytemuck"] }
bytemuck = { version = "1.14", features = ["derive"] }
kurbo = { workspace = true }
peniko = { workspace = true }

255
lightningbeam-ui/lightningbeam-editor/src/main.rs
View File

@ -19,7 +19,7 @@ use menu::{MenuAction, MenuSystem};
mod theme;
use theme::{Theme, ThemeMode};
mod waveform_image_cache;
mod waveform_gpu;
mod config;
use config::AppConfig;
@ -689,21 +689,16 @@ struct EditorApp {
/// Prevents repeated backend queries for the same MIDI clip
/// Format: (timestamp, note_number, is_note_on)
midi_event_cache: HashMap<u32, Vec<(f64, u8, bool)>>,
/// Cache for audio waveform data (keyed by audio_pool_index)
/// Prevents repeated backend queries for the same audio file
/// Format: Vec of WaveformPeak (min/max pairs)
waveform_cache: HashMap<usize, Vec<daw_backend::WaveformPeak>>,
/// Chunk-based waveform cache for multi-resolution waveforms
/// Format: (pool_index, detail_level, chunk_index) -> Vec<WaveformPeak>
waveform_chunk_cache: HashMap<(usize, u8, u32), Vec<daw_backend::WaveformPeak>>,
/// Cache for audio file durations to avoid repeated queries
/// Format: pool_index -> duration in seconds
audio_duration_cache: HashMap<usize, f64>,
/// Track which audio pool indices got new waveform data this frame (for thumbnail invalidation)
/// Track which audio pool indices got new raw audio data this frame (for thumbnail invalidation)
audio_pools_with_new_waveforms: HashSet<usize>,
/// Cache for rendered waveform images (GPU textures)
/// Stores pre-rendered waveform tiles at various zoom levels for fast blitting
waveform_image_cache: waveform_image_cache::WaveformImageCache,
/// Raw audio sample cache for GPU waveform rendering
/// Format: pool_index -> (samples, sample_rate, channels)
raw_audio_cache: HashMap<usize, (Vec<f32>, u32, u32)>,
/// Pool indices that need GPU texture upload (set when raw audio arrives, cleared after upload)
waveform_gpu_dirty: HashSet<usize>,
/// Current file path (None if not yet saved)
current_file_path: Option<std::path::PathBuf>,
/// Application configuration (recent files, etc.)
@ -896,11 +891,10 @@ impl EditorApp {
paint_bucket_gap_tolerance: 5.0, // Default gap tolerance
polygon_sides: 5, // Default to pentagon
midi_event_cache: HashMap::new(), // Initialize empty MIDI event cache
waveform_cache: HashMap::new(), // Initialize empty waveform cache
waveform_chunk_cache: HashMap::new(), // Initialize empty chunk-based waveform cache
audio_duration_cache: HashMap::new(), // Initialize empty audio duration cache
audio_pools_with_new_waveforms: HashSet::new(), // Track pool indices with new waveforms
waveform_image_cache: waveform_image_cache::WaveformImageCache::new(), // Initialize waveform image cache
audio_pools_with_new_waveforms: HashSet::new(), // Track pool indices with new raw audio
raw_audio_cache: HashMap::new(),
waveform_gpu_dirty: HashSet::new(),
current_file_path: None, // No file loaded initially
config,
file_command_tx,
@ -1465,78 +1459,6 @@ impl EditorApp {
}
}
/// Fetch waveform data from backend for a specific audio pool index
/// Returns cached data if available, otherwise tries to assemble from chunks
/// For thumbnails, uses Level 0 (overview) chunks which are fast to generate
fn fetch_waveform(&mut self, pool_index: usize) -> Option<Vec<daw_backend::WaveformPeak>> {
// Check if already cached in old waveform cache
if let Some(waveform) = self.waveform_cache.get(&pool_index) {
return Some(waveform.clone());
}
// Try to assemble from Level 0 (overview) chunks - perfect for thumbnails
// Level 0 = 1 peak/sec, so a 200s file only needs 200 peaks (very fast)
// Get audio file duration (use cached value to avoid repeated queries)
let audio_file_duration = if let Some(&duration) = self.audio_duration_cache.get(&pool_index) {
duration
} else {
// Duration not cached - query it once and cache
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
match controller.get_pool_file_info(pool_index) {
Ok((duration, _, _)) => {
self.audio_duration_cache.insert(pool_index, duration);
duration
}
Err(_) => return None,
}
} else {
return None;
}
};
// Assemble Level 0 chunks for the entire file
let detail_level = 0; // Level 0 (overview)
let chunk_time_span = 60.0; // 60 seconds per chunk
let total_chunks = (audio_file_duration / chunk_time_span).ceil() as u32;
let mut assembled_peaks = Vec::new();
let mut missing_chunks = Vec::new();
// Check if all required chunks are available
for chunk_idx in 0..total_chunks {
let key = (pool_index, detail_level, chunk_idx);
if let Some(chunk_peaks) = self.waveform_chunk_cache.get(&key) {
assembled_peaks.extend_from_slice(chunk_peaks);
} else {
missing_chunks.push(chunk_idx);
}
}
// If any chunks are missing, request them (but only if we have a controller)
if !missing_chunks.is_empty() {
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
let _ = controller.generate_waveform_chunks(
pool_index,
detail_level,
missing_chunks,
2, // High priority for thumbnails
);
}
return None; // Will retry next frame when chunks arrive
}
// All chunks available - cache and return
if !assembled_peaks.is_empty() {
self.waveform_cache.insert(pool_index, assembled_peaks.clone());
return Some(assembled_peaks);
}
None
}
fn switch_layout(&mut self, index: usize) {
self.current_layout_index = index;
self.current_layout = self.layouts[index].layout.clone();
@ -2299,9 +2221,8 @@ impl EditorApp {
self.sync_audio_layers_to_backend();
eprintln!("📊 [APPLY] Step 6: Sync audio layers took {:.2}ms", step6_start.elapsed().as_secs_f64() * 1000.0);
// Fetch waveforms for all audio clips in the loaded project
// Fetch raw audio for all audio clips in the loaded project
let step7_start = std::time::Instant::now();
// Collect pool indices first to avoid borrowing issues
let pool_indices: Vec<usize> = self.action_executor.document()
.audio_clips.values()
.filter_map(|clip| {
@ -2313,13 +2234,23 @@ impl EditorApp {
})
.collect();
let mut waveforms_fetched = 0;
let mut raw_fetched = 0;
for pool_index in pool_indices {
if self.fetch_waveform(pool_index).is_some() {
waveforms_fetched += 1;
if !self.raw_audio_cache.contains_key(&pool_index) {
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
match controller.get_pool_audio_samples(pool_index) {
Ok((samples, sr, ch)) => {
self.raw_audio_cache.insert(pool_index, (samples, sr, ch));
self.waveform_gpu_dirty.insert(pool_index);
raw_fetched += 1;
}
Err(e) => eprintln!("Failed to fetch raw audio for pool {}: {}", pool_index, e),
}
}
}
}
eprintln!("📊 [APPLY] Step 7: Fetched {} waveforms in {:.2}ms", waveforms_fetched, step7_start.elapsed().as_secs_f64() * 1000.0);
eprintln!("📊 [APPLY] Step 7: Fetched {} raw audio samples in {:.2}ms", raw_fetched, step7_start.elapsed().as_secs_f64() * 1000.0);
// Reset playback state
self.playback_time = 0.0;
@ -2427,9 +2358,17 @@ impl EditorApp {
let clip = AudioClip::new_sampled(&name, pool_index, duration);
let clip_id = self.action_executor.document_mut().add_audio_clip(clip);
// Fetch waveform from backend and cache it for rendering
if let Some(waveform) = self.fetch_waveform(pool_index) {
println!("✅ Cached waveform with {} peaks", waveform.len());
// Fetch raw audio samples for GPU waveform rendering
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
match controller.get_pool_audio_samples(pool_index) {
Ok((samples, sr, ch)) => {
println!("✅ Cached {} raw audio samples for GPU waveform", samples.len());
self.raw_audio_cache.insert(pool_index, (samples, sr, ch));
self.waveform_gpu_dirty.insert(pool_index);
}
Err(e) => eprintln!("Failed to fetch raw audio: {}", e),
}
}
println!("Imported audio '{}' ({:.1}s, {}ch, {}Hz) - ID: {}",
@ -3053,9 +2992,16 @@ impl EditorApp {
audio_clip_id
);
// Fetch waveform from backend and cache it for rendering
if let Some(waveform) = self.fetch_waveform(pool_index) {
println!(" Cached waveform with {} peaks", waveform.len());
// Fetch raw audio samples for GPU waveform rendering
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
match controller.get_pool_audio_samples(pool_index) {
Ok((samples, sr, ch)) => {
self.raw_audio_cache.insert(pool_index, (samples, sr, ch));
self.waveform_gpu_dirty.insert(pool_index);
}
Err(e) => eprintln!("Failed to fetch raw audio for extracted audio: {}", e),
}
}
// Auto-place extracted audio if the video was auto-placed
@ -3107,14 +3053,13 @@ impl eframe::App for EditorApp {
// Will switch to editor mode when file finishes loading
}
// Fetch missing waveforms on-demand (for lazy loading after project load)
// Collect pool indices that need waveforms
let missing_waveforms: Vec<usize> = self.action_executor.document()
// Fetch missing raw audio on-demand (for lazy loading after project load)
// Collect pool indices that need raw audio data
let missing_raw_audio: Vec<usize> = self.action_executor.document()
.audio_clips.values()
.filter_map(|clip| {
if let lightningbeam_core::clip::AudioClipType::Sampled { audio_pool_index } = &clip.clip_type {
// Check if not already cached
if !self.waveform_cache.contains_key(audio_pool_index) {
if !self.raw_audio_cache.contains_key(audio_pool_index) {
Some(*audio_pool_index)
} else {
None
@ -3125,9 +3070,19 @@ impl eframe::App for EditorApp {
})
.collect();
// Fetch missing waveforms
for pool_index in missing_waveforms {
self.fetch_waveform(pool_index);
// Fetch missing raw audio samples
for pool_index in missing_raw_audio {
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
match controller.get_pool_audio_samples(pool_index) {
Ok((samples, sr, ch)) => {
self.raw_audio_cache.insert(pool_index, (samples, sr, ch));
self.waveform_gpu_dirty.insert(pool_index);
self.audio_pools_with_new_waveforms.insert(pool_index);
}
Err(e) => eprintln!("Failed to fetch raw audio for pool {}: {}", pool_index, e),
}
}
}
// Initialize and update effect thumbnail generator (GPU-based effect previews)
@ -3310,29 +3265,22 @@ impl eframe::App for EditorApp {
);
ctx.request_repaint();
}
AudioEvent::WaveformChunksReady { pool_index, detail_level, chunks } => {
// Store waveform chunks in the cache
let mut all_peaks = Vec::new();
for (chunk_index, _time_range, peaks) in chunks {
let key = (pool_index, detail_level, chunk_index);
self.waveform_chunk_cache.insert(key, peaks.clone());
all_peaks.extend(peaks);
AudioEvent::WaveformChunksReady { pool_index, .. } => {
// Fetch raw audio for GPU waveform if not already cached
if !self.raw_audio_cache.contains_key(&pool_index) {
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
match controller.get_pool_audio_samples(pool_index) {
Ok((samples, sr, ch)) => {
self.raw_audio_cache.insert(pool_index, (samples, sr, ch));
self.waveform_gpu_dirty.insert(pool_index);
self.audio_pools_with_new_waveforms.insert(pool_index);
}
Err(e) => eprintln!("Failed to fetch raw audio for pool {}: {}", pool_index, e),
}
}
}
// If this is Level 0 (overview), also populate the old waveform_cache
// so asset library thumbnails can use it immediately
if detail_level == 0 && !all_peaks.is_empty() {
println!("💾 [EVENT] Storing {} Level 0 peaks for pool {} in waveform_cache", all_peaks.len(), pool_index);
self.waveform_cache.insert(pool_index, all_peaks);
// Mark this pool index as having new waveform data (for thumbnail invalidation)
self.audio_pools_with_new_waveforms.insert(pool_index);
println!("🔔 [EVENT] Marked pool {} for thumbnail invalidation", pool_index);
}
// Invalidate image cache for this pool index
// (The waveform tiles will be regenerated with new chunk data)
self.waveform_image_cache.invalidate_audio(pool_index);
ctx.request_repaint();
}
// Recording events
@ -3394,18 +3342,22 @@ impl eframe::App for EditorApp {
}
ctx.request_repaint();
}
AudioEvent::RecordingStopped(_backend_clip_id, pool_index, waveform) => {
println!("🎤 Recording stopped: pool_index={}, {} peaks", pool_index, waveform.len());
AudioEvent::RecordingStopped(_backend_clip_id, pool_index, _waveform) => {
println!("🎤 Recording stopped: pool_index={}", pool_index);
// Store waveform for the recorded clip
if !waveform.is_empty() {
self.waveform_cache.insert(pool_index, waveform.clone());
self.audio_pools_with_new_waveforms.insert(pool_index);
// Fetch raw audio samples for GPU waveform rendering
if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
match controller.get_pool_audio_samples(pool_index) {
Ok((samples, sr, ch)) => {
self.raw_audio_cache.insert(pool_index, (samples, sr, ch));
self.waveform_gpu_dirty.insert(pool_index);
self.audio_pools_with_new_waveforms.insert(pool_index);
}
Err(e) => eprintln!("Failed to fetch raw audio after recording: {}", e),
}
}
// Invalidate waveform image cache so tiles are regenerated
self.waveform_image_cache.invalidate_audio(pool_index);
// Get accurate duration from backend (not calculated from waveform peaks)
let duration = if let Some(ref controller_arc) = self.audio_controller {
let mut controller = controller_arc.lock().unwrap();
@ -3821,10 +3773,9 @@ impl eframe::App for EditorApp {
polygon_sides: &mut self.polygon_sides,
layer_to_track_map: &self.layer_to_track_map,
midi_event_cache: &self.midi_event_cache,
waveform_cache: &self.waveform_cache,
waveform_chunk_cache: &self.waveform_chunk_cache,
waveform_image_cache: &mut self.waveform_image_cache,
audio_pools_with_new_waveforms: &self.audio_pools_with_new_waveforms,
raw_audio_cache: &self.raw_audio_cache,
waveform_gpu_dirty: &mut self.waveform_gpu_dirty,
effect_to_load: &mut self.effect_to_load,
effect_thumbnail_requests: &mut effect_thumbnail_requests,
effect_thumbnail_cache: self.effect_thumbnail_generator.as_ref()
@ -4052,14 +4003,12 @@ struct RenderContext<'a> {
layer_to_track_map: &'a std::collections::HashMap<Uuid, daw_backend::TrackId>,
/// Cache of MIDI events for rendering (keyed by backend midi_clip_id)
midi_event_cache: &'a HashMap<u32, Vec<(f64, u8, bool)>>,
/// Cache of waveform data for rendering (keyed by audio_pool_index)
waveform_cache: &'a HashMap<usize, Vec<daw_backend::WaveformPeak>>,
/// Chunk-based waveform cache for multi-resolution waveforms
waveform_chunk_cache: &'a HashMap<(usize, u8, u32), Vec<daw_backend::WaveformPeak>>,
/// Cache of rendered waveform images (GPU textures)
waveform_image_cache: &'a mut waveform_image_cache::WaveformImageCache,
/// Audio pool indices with new waveform data this frame (for thumbnail invalidation)
/// Audio pool indices with new raw audio data this frame (for thumbnail invalidation)
audio_pools_with_new_waveforms: &'a HashSet<usize>,
/// Raw audio samples for GPU waveform rendering (pool_index -> (samples, sample_rate, channels))
raw_audio_cache: &'a HashMap<usize, (Vec<f32>, u32, u32)>,
/// Pool indices needing GPU texture upload
waveform_gpu_dirty: &'a mut HashSet<usize>,
/// Effect ID to load into shader editor (set by asset library, consumed by shader editor)
effect_to_load: &'a mut Option<Uuid>,
/// Queue for effect thumbnail requests
@ -4540,10 +4489,9 @@ fn render_pane(
paint_bucket_gap_tolerance: ctx.paint_bucket_gap_tolerance,
polygon_sides: ctx.polygon_sides,
midi_event_cache: ctx.midi_event_cache,
waveform_cache: ctx.waveform_cache,
waveform_chunk_cache: ctx.waveform_chunk_cache,
waveform_image_cache: ctx.waveform_image_cache,
audio_pools_with_new_waveforms: ctx.audio_pools_with_new_waveforms,
raw_audio_cache: ctx.raw_audio_cache,
waveform_gpu_dirty: ctx.waveform_gpu_dirty,
effect_to_load: ctx.effect_to_load,
effect_thumbnail_requests: ctx.effect_thumbnail_requests,
effect_thumbnail_cache: ctx.effect_thumbnail_cache,
@ -4608,10 +4556,9 @@ fn render_pane(
paint_bucket_gap_tolerance: ctx.paint_bucket_gap_tolerance,
polygon_sides: ctx.polygon_sides,
midi_event_cache: ctx.midi_event_cache,
waveform_cache: ctx.waveform_cache,
waveform_chunk_cache: ctx.waveform_chunk_cache,
waveform_image_cache: ctx.waveform_image_cache,
audio_pools_with_new_waveforms: ctx.audio_pools_with_new_waveforms,
raw_audio_cache: ctx.raw_audio_cache,
waveform_gpu_dirty: ctx.waveform_gpu_dirty,
effect_to_load: ctx.effect_to_load,
effect_thumbnail_requests: ctx.effect_thumbnail_requests,
effect_thumbnail_cache: ctx.effect_thumbnail_cache,

53
lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs
View File

@ -17,6 +17,42 @@ use uuid::Uuid;
use super::{DragClipType, DraggingAsset, NodePath, PaneRenderer, SharedPaneState};
use crate::widgets::ImeTextField;
/// Derive min/max peak pairs from raw audio samples for thumbnail rendering.
/// Downsamples to `num_peaks` (min, max) pairs by scanning chunks of samples.
fn peaks_from_raw_audio(
raw: &(Vec<f32>, u32, u32), // (samples, sample_rate, channels)
num_peaks: usize,
) -> Vec<(f32, f32)> {
let (samples, _sr, channels) = raw;
let ch = (*channels as usize).max(1);
let total_frames = samples.len() / ch;
if total_frames == 0 || num_peaks == 0 {
return vec![];
}
let frames_per_peak = (total_frames as f64 / num_peaks as f64).max(1.0);
let mut peaks = Vec::with_capacity(num_peaks);
for i in 0..num_peaks {
let start = (i as f64 * frames_per_peak) as usize;
let end = (((i + 1) as f64 * frames_per_peak) as usize).min(total_frames);
let mut min_val = f32::MAX;
let mut max_val = f32::MIN;
for frame in start..end {
// Mix all channels together for the thumbnail
let mut sample = 0.0f32;
for c in 0..ch {
sample += samples[frame * ch + c];
}
sample /= ch as f32;
min_val = min_val.min(sample);
max_val = max_val.max(sample);
}
if min_val <= max_val {
peaks.push((min_val, max_val));
}
}
peaks
}
// Thumbnail constants
const THUMBNAIL_SIZE: u32 = 64;
const THUMBNAIL_PREVIEW_SECONDS: f64 = 10.0;
@ -1790,8 +1826,8 @@ impl AssetLibraryPane {
if asset_category == AssetCategory::Audio && !self.thumbnail_cache.has(&asset_id) {
if let Some(clip) = document.audio_clips.get(&asset_id) {
if let AudioClipType::Sampled { audio_pool_index } = &clip.clip_type {
shared.waveform_cache.get(audio_pool_index)
.map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect())
shared.raw_audio_cache.get(audio_pool_index)
.map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize))
} else {
None
}
@ -2380,8 +2416,8 @@ impl AssetLibraryPane {
match &clip.clip_type {
AudioClipType::Sampled { audio_pool_index } => {
let wave_color = egui::Color32::from_rgb(100, 200, 100);
let waveform: Option<Vec<(f32, f32)>> = shared.waveform_cache.get(audio_pool_index)
.map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect());
let waveform: Option<Vec<(f32, f32)>> = shared.raw_audio_cache.get(audio_pool_index)
.map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize));
if let Some(ref peaks) = waveform {
Some(generate_waveform_thumbnail(peaks, bg_color, wave_color))
} else {
@ -2525,8 +2561,8 @@ impl AssetLibraryPane {
match &clip.clip_type {
AudioClipType::Sampled { audio_pool_index } => {
let wave_color = egui::Color32::from_rgb(100, 200, 100);
let waveform: Option<Vec<(f32, f32)>> = shared.waveform_cache.get(audio_pool_index)
.map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect());
let waveform: Option<Vec<(f32, f32)>> = shared.raw_audio_cache.get(audio_pool_index)
.map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize));
if let Some(ref peaks) = waveform {
Some(generate_waveform_thumbnail(peaks, bg_color, wave_color))
} else {
@ -2850,9 +2886,8 @@ impl AssetLibraryPane {
if asset_category == AssetCategory::Audio && !self.thumbnail_cache.has(&asset_id) {
if let Some(clip) = document.audio_clips.get(&asset_id) {
if let AudioClipType::Sampled { audio_pool_index } = &clip.clip_type {
// Use cached waveform data (populated by fetch_waveform in main.rs)
let waveform = shared.waveform_cache.get(audio_pool_index)
.map(|peaks| peaks.iter().map(|p| (p.min, p.max)).collect());
let waveform: Option<Vec<(f32, f32)>> = shared.raw_audio_cache.get(audio_pool_index)
.map(|raw| peaks_from_raw_audio(raw, THUMBNAIL_SIZE as usize));
if waveform.is_some() {
println!("🎵 Found waveform for pool {} (asset {})", audio_pool_index, asset_id);
} else {

13
lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs
View File

@ -189,15 +189,12 @@ pub struct SharedPaneState<'a> {
pub polygon_sides: &'a mut u32,
/// Cache of MIDI events for rendering (keyed by backend midi_clip_id)
pub midi_event_cache: &'a std::collections::HashMap<u32, Vec<(f64, u8, bool)>>,
/// Cache of waveform data for rendering (keyed by audio_pool_index)
pub waveform_cache: &'a std::collections::HashMap<usize, Vec<daw_backend::WaveformPeak>>,
/// Chunk-based waveform cache for multi-resolution waveforms
/// Format: (pool_index, detail_level, chunk_index) -> Vec<WaveformPeak>
pub waveform_chunk_cache: &'a std::collections::HashMap<(usize, u8, u32), Vec<daw_backend::WaveformPeak>>,
/// Cache of rendered waveform images (GPU textures) for fast blitting
pub waveform_image_cache: &'a mut crate::waveform_image_cache::WaveformImageCache,
/// Audio pool indices that got new waveform data this frame (for thumbnail invalidation)
/// Audio pool indices that got new raw audio data this frame (for thumbnail invalidation)
pub audio_pools_with_new_waveforms: &'a std::collections::HashSet<usize>,
/// Raw audio samples for GPU waveform rendering (pool_index -> (samples, sample_rate, channels))
pub raw_audio_cache: &'a std::collections::HashMap<usize, (Vec<f32>, u32, u32)>,
/// Pool indices needing GPU waveform texture upload
pub waveform_gpu_dirty: &'a mut std::collections::HashSet<usize>,
/// Effect ID to load into shader editor (set by asset library, consumed by shader editor)
pub effect_to_load: &'a mut Option<Uuid>,
/// Queue for effect thumbnail requests (effect IDs to generate thumbnails for)

157
lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl Normal file
View File

@ -0,0 +1,157 @@
// Waveform rendering shader for audio data stored in 2D Rgba16Float textures.
// Audio samples are packed row-major into 2D: frame_index = y * tex_width + x
// Mip levels use min/max reduction (4 consecutive samples per level).
//
// At full zoom, min≈max → renders as a continuous wave.
// At zoom-out, min/max spread → renders as filled peak region.
//
// display_mode: 0 = combined (mono mix), 1 = split (left top, right bottom)
struct Params {
// Clip rectangle in screen pixels (min.x, min.y, max.x, max.y)
clip_rect: vec4<f32>,
// Timeline viewport parameters
viewport_start_time: f32,
pixels_per_second: f32,
// Audio file properties
audio_duration: f32,
sample_rate: f32,
// Clip placement
clip_start_time: f32,
trim_start: f32,
// Texture layout
tex_width: f32,
total_frames: f32, // total frame count in this texture segment
segment_start_frame: f32, // first frame this texture covers (for multi-texture)
display_mode: f32, // 0 = combined, 1 = split stereo
// Appearance
tint_color: vec4<f32>,
// Screen dimensions for coordinate conversion
screen_size: vec2<f32>,
_pad: vec2<f32>,
}
@group(0) @binding(0) var peak_tex: texture_2d<f32>;
@group(0) @binding(1) var peak_sampler: sampler;
@group(0) @binding(2) var<uniform> params: Params;
struct VertexOutput {
@builtin(position) position: vec4<f32>,
@location(0) uv: vec2<f32>,
}
// Fullscreen triangle (3 vertices, no vertex buffer)
@vertex
fn vs_main(@builtin(vertex_index) vi: u32) -> VertexOutput {
var out: VertexOutput;
let x = f32(i32(vi) / 2) * 4.0 - 1.0;
let y = f32(i32(vi) % 2) * 4.0 - 1.0;
out.position = vec4(x, y, 0.0, 1.0);
out.uv = vec2((x + 1.0) * 0.5, (1.0 - y) * 0.5);
return out;
}
@fragment
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
let frag_x = in.position.x;
let frag_y = in.position.y;
// Clip to the clip rectangle
if frag_x < params.clip_rect.x || frag_x > params.clip_rect.z ||
frag_y < params.clip_rect.y || frag_y > params.clip_rect.w {
discard;
}
// Fragment X position → audio time
let timeline_time = params.viewport_start_time + (frag_x - params.clip_rect.x) / params.pixels_per_second;
let audio_time = timeline_time - params.clip_start_time + params.trim_start;
// Audio time → frame index
let frame_f = audio_time * params.sample_rate - params.segment_start_frame;
if frame_f < 0.0 || frame_f >= params.total_frames {
discard;
}
// Determine mip level based on how many audio frames map to one pixel
let frames_per_pixel = params.sample_rate / params.pixels_per_second;
// Each mip level reduces by 4x in sample count (2x in each texture dimension)
let mip_f = max(0.0, log2(frames_per_pixel) / 2.0);
let max_mip = f32(textureNumLevels(peak_tex) - 1u);
let mip = min(mip_f, max_mip);
// Frame index at the chosen mip level
let mip_floor = u32(mip);
let reduction = pow(4.0, f32(mip_floor));
let mip_frame = frame_f / reduction;
// Convert 1D mip-space index to 2D UV coordinates
let mip_tex_width = params.tex_width / pow(2.0, f32(mip_floor));
let mip_tex_height = ceil(params.total_frames / reduction / mip_tex_width);
let texel_x = mip_frame % mip_tex_width;
let texel_y = floor(mip_frame / mip_tex_width);
let uv = vec2((texel_x + 0.5) / mip_tex_width, (texel_y + 0.5) / mip_tex_height);
// Sample the peak texture at computed mip level
// R = left_min, G = left_max, B = right_min, A = right_max
let peak = textureSampleLevel(peak_tex, peak_sampler, uv, mip);
let clip_height = params.clip_rect.w - params.clip_rect.y;
let clip_top = params.clip_rect.y;
if params.display_mode < 0.5 {
// Combined mode: merge both channels
let wave_min = min(peak.r, peak.b);
let wave_max = max(peak.g, peak.a);
let center_y = clip_top + clip_height * 0.5;
let scale = clip_height * 0.45;
let y_top = center_y - wave_max * scale;
let y_bot = center_y - wave_min * scale;
// At least 1px tall for visibility
let y_top_adj = min(y_top, center_y - 0.5);
let y_bot_adj = max(y_bot, center_y + 0.5);
if frag_y >= y_top_adj && frag_y <= y_bot_adj {
return params.tint_color;
}
} else {
// Split stereo mode: left channel in top half, right channel in bottom half
let half_height = clip_height * 0.5;
let mid_y = clip_top + half_height;
// Determine which channel this fragment belongs to
if frag_y < mid_y {
// Top half: left channel
let center_y = clip_top + half_height * 0.5;
let scale = half_height * 0.45;
let y_top = center_y - peak.g * scale; // left_max
let y_bot = center_y - peak.r * scale; // left_min
let y_top_adj = min(y_top, center_y - 0.5);
let y_bot_adj = max(y_bot, center_y + 0.5);
if frag_y >= y_top_adj && frag_y <= y_bot_adj {
return params.tint_color;
}
} else {
// Bottom half: right channel
let center_y = mid_y + half_height * 0.5;
let scale = half_height * 0.45;
let y_top = center_y - peak.a * scale; // right_max
let y_bot = center_y - peak.b * scale; // right_min
let y_top_adj = min(y_top, center_y - 0.5);
let y_bot_adj = max(y_bot, center_y + 0.5);
if frag_y >= y_top_adj && frag_y <= y_bot_adj {
return params.tint_color;
}
}
}
discard;
}

64
lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform_mipgen.wgsl Normal file
View File

@ -0,0 +1,64 @@
// Min/max mipmap generation for 1D audio data packed into 2D textures.
// Each dest texel reduces 4 CONSECUTIVE source texels in audio order,
// using 1D→2D coordinate conversion since the 2D layout is linearized 1D.
// Row wrapping is handled by the modulo/division mapping:
// x = index % width, y = index / width
// So sample index 2048 with width=2048 maps to (0, 1), not (0, 0).
//
// Texture format: Rgba16Float
// R = left_min, G = left_max, B = right_min, A = right_max
// At mip 0 (raw samples): R=G=left_sample, B=A=right_sample
struct MipParams {
src_width: u32,
dst_width: u32,
src_sample_count: u32, // valid texels in source level
_pad: u32,
}
@group(0) @binding(0) var src_mip: texture_2d<f32>;
@group(0) @binding(1) var dst_mip: texture_storage_2d<rgba16float, write>;
@group(0) @binding(2) var<uniform> params: MipParams;
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) id: vec3<u32>) {
// Each thread handles one dest texel
let dst_1d = id.x;
let dst_size = textureDimensions(dst_mip);
if dst_1d >= dst_size.x * dst_size.y {
return;
}
let dst_x = dst_1d % params.dst_width;
let dst_y = dst_1d / params.dst_width;
// Map to 4 consecutive source texels in 1D audio order
let src_base = dst_1d * 4u;
var result = vec4(0.0);
var initialized = false;
for (var i = 0u; i < 4u; i++) {
let src_1d = src_base + i;
if src_1d >= params.src_sample_count {
break;
}
// 1D → 2D: wraps across rows naturally
let src_x = src_1d % params.src_width;
let src_y = src_1d / params.src_width;
let s = textureLoad(src_mip, vec2(src_x, src_y), 0);
if !initialized {
result = s;
initialized = true;
} else {
result = vec4(
min(result.r, s.r), // left_min
max(result.g, s.g), // left_max
min(result.b, s.b), // right_min
max(result.a, s.a), // right_max
);
}
}
textureStore(dst_mip, vec2(dst_x, dst_y), result);
}

419
lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs
View File

@ -607,311 +607,6 @@ impl TimelinePane {
}
}
/// Calculate which waveform tiles are visible in the viewport
fn calculate_visible_tiles(
audio_pool_index: usize,
clip_start_time: f64,
clip_duration: f64,
clip_rect: egui::Rect,
timeline_left_edge: f32,
viewport_start_time: f64,
pixels_per_second: f64,
zoom_bucket: u32,
height: u32,
trim_start: f64,
audio_file_duration: f64,
) -> Vec<crate::waveform_image_cache::WaveformCacheKey> {
use crate::waveform_image_cache::{WaveformCacheKey, TILE_WIDTH_PIXELS};
// Calculate clip position in screen space (including timeline offset)
let clip_start_x = timeline_left_edge + ((clip_start_time - viewport_start_time) * pixels_per_second) as f32;
let clip_width = (clip_duration * pixels_per_second) as f32;
// Check if clip is visible
if clip_start_x + clip_width < clip_rect.min.x || clip_start_x > clip_rect.max.x {
return vec![]; // Clip not visible
}
// Calculate tile duration in seconds (based on zoom bucket, not current pixels_per_second)
let seconds_per_pixel_in_tile = 1.0 / zoom_bucket as f64;
let tile_duration_seconds = TILE_WIDTH_PIXELS as f64 * seconds_per_pixel_in_tile;
// Calculate visible time range within the clip
let visible_start_pixel = (clip_rect.min.x - clip_start_x).max(0.0);
let visible_end_pixel = (clip_rect.max.x - clip_start_x).min(clip_width);
// Convert screen pixels to time within clip
let visible_start_time_in_clip = (visible_start_pixel as f64) / pixels_per_second;
let visible_end_time_in_clip = (visible_end_pixel as f64) / pixels_per_second;
// Convert to audio file coordinates (tiles are indexed by audio file position)
let visible_audio_start = trim_start + visible_start_time_in_clip;
let visible_audio_end = (trim_start + visible_end_time_in_clip).min(audio_file_duration);
// Calculate which tiles from the audio file cover this range
let start_tile = (visible_audio_start / tile_duration_seconds).floor() as u32;
let end_tile = ((visible_audio_end / tile_duration_seconds).ceil() as u32).max(start_tile + 1);
// Generate cache keys for visible tiles
let mut keys = Vec::new();
for tile_idx in start_tile..end_tile {
keys.push(WaveformCacheKey {
audio_pool_index,
zoom_bucket,
tile_index: tile_idx,
height,
});
}
keys
}
/// Calculate tiles for pre-caching (1-2 screens ahead/behind)
fn calculate_precache_tiles(
visible_tiles: &[crate::waveform_image_cache::WaveformCacheKey],
viewport_width_pixels: f32,
) -> Vec<crate::waveform_image_cache::WaveformCacheKey> {
use crate::waveform_image_cache::{WaveformCacheKey, TILE_WIDTH_PIXELS};
if visible_tiles.is_empty() {
return vec![];
}
// Calculate how many tiles = 1-2 screens
let tiles_per_screen = ((viewport_width_pixels as usize + TILE_WIDTH_PIXELS - 1)
/ TILE_WIDTH_PIXELS) as u32;
let precache_count = tiles_per_screen * 2; // 2 screens worth
let first_visible = visible_tiles.first().unwrap();
let last_visible = visible_tiles.last().unwrap();
let mut precache = Vec::new();
// Tiles before viewport
for i in 1..=precache_count {
if let Some(tile_idx) = first_visible.tile_index.checked_sub(i) {
precache.push(WaveformCacheKey {
audio_pool_index: first_visible.audio_pool_index,
zoom_bucket: first_visible.zoom_bucket,
tile_index: tile_idx,
height: first_visible.height,
});
}
}
// Tiles after viewport (with bounds check based on clip duration)
for i in 1..=precache_count {
let tile_idx = last_visible.tile_index + i;
precache.push(WaveformCacheKey {
audio_pool_index: first_visible.audio_pool_index,
zoom_bucket: first_visible.zoom_bucket,
tile_index: tile_idx,
height: first_visible.height,
});
}
precache
}
/// Select appropriate detail level based on zoom (pixels per second)
///
/// Detail levels:
/// - Level 0 (Overview): 1 peak/sec - for extreme zoom out (0-2 pps)
/// - Level 1 (Low): 10 peaks/sec - for zoomed out view (2-20 pps)
/// - Level 2 (Medium): 100 peaks/sec - for normal view (20-200 pps)
/// - Level 3 (High): 1000 peaks/sec - for zoomed in (200-2000 pps)
/// - Level 4 (Max): Full resolution - for maximum zoom (>2000 pps)
fn select_detail_level(pixels_per_second: f64) -> u8 {
if pixels_per_second < 2.0 {
0 // Overview
} else if pixels_per_second < 20.0 {
1 // Low
} else if pixels_per_second < 200.0 {
2 // Medium
} else if pixels_per_second < 2000.0 {
3 // High
} else {
4 // Max (full resolution)
}
}
/// Assemble waveform peaks from chunks for the ENTIRE audio file
///
/// Returns peaks for the entire audio file, or None if chunks are not available
/// This assembles a complete waveform from chunks at the appropriate detail level
fn assemble_peaks_from_chunks(
waveform_chunk_cache: &std::collections::HashMap<(usize, u8, u32), Vec<daw_backend::WaveformPeak>>,
audio_pool_index: usize,
detail_level: u8,
audio_file_duration: f64,
audio_controller: Option<&std::sync::Arc<std::sync::Mutex<daw_backend::EngineController>>>,
) -> Option<Vec<daw_backend::WaveformPeak>> {
// Calculate chunk time span based on detail level
let chunk_time_span = match detail_level {
0 => 60.0, // Level 0: 60 seconds per chunk
1 => 30.0, // Level 1: 30 seconds per chunk
2 => 10.0, // Level 2: 10 seconds per chunk
3 => 5.0, // Level 3: 5 seconds per chunk
4 => 1.0, // Level 4: 1 second per chunk
_ => 10.0, // Default
};
// Calculate total number of chunks needed for entire audio file
let total_chunks = (audio_file_duration / chunk_time_span).ceil() as u32;
let mut assembled_peaks = Vec::new();
let mut missing_chunks = Vec::new();
// Check if all required chunks are available
for chunk_idx in 0..total_chunks {
let key = (audio_pool_index, detail_level, chunk_idx);
if let Some(chunk_peaks) = waveform_chunk_cache.get(&key) {
assembled_peaks.extend_from_slice(chunk_peaks);
} else {
// Track missing chunk
missing_chunks.push(chunk_idx);
}
}
// If any chunks are missing, request them and return None
if !missing_chunks.is_empty() {
if let Some(controller_arc) = audio_controller {
let mut controller = controller_arc.lock().unwrap();
let _ = controller.generate_waveform_chunks(
audio_pool_index,
detail_level,
missing_chunks,
1, // Medium priority
);
}
return None;
}
Some(assembled_peaks)
}
/// Render waveform visualization using cached texture tiles
/// This is much faster than line-based rendering for many clips
#[allow(clippy::too_many_arguments)]
fn render_audio_waveform(
painter: &egui::Painter,
clip_rect: egui::Rect,
timeline_left_edge: f32,
audio_pool_index: usize,
clip_start_time: f64,
clip_duration: f64,
trim_start: f64,
audio_file_duration: f64,
viewport_start_time: f64,
pixels_per_second: f64,
waveform_image_cache: &mut crate::waveform_image_cache::WaveformImageCache,
waveform_peaks: &[daw_backend::WaveformPeak],
ctx: &egui::Context,
tint_color: egui::Color32,
) {
use crate::waveform_image_cache::{calculate_zoom_bucket, TILE_WIDTH_PIXELS};
if waveform_peaks.is_empty() {
return;
}
// Calculate zoom bucket
let zoom_bucket = calculate_zoom_bucket(pixels_per_second);
// Calculate visible tiles
let visible_tiles = Self::calculate_visible_tiles(
audio_pool_index,
clip_start_time,
clip_duration,
clip_rect,
timeline_left_edge,
viewport_start_time,
pixels_per_second,
zoom_bucket,
clip_rect.height() as u32,
trim_start,
audio_file_duration,
);
// Render each tile
for key in &visible_tiles {
let texture = waveform_image_cache.get_or_create(
*key,
ctx,
waveform_peaks,
audio_file_duration,
);
// Calculate tile position in audio file (tiles now represent fixed portions of the audio file)
// Each pixel in the tile texture represents (1.0 / zoom_bucket) seconds
let seconds_per_pixel_in_tile = 1.0 / key.zoom_bucket as f64;
let tile_audio_start = key.tile_index as f64 * TILE_WIDTH_PIXELS as f64 * seconds_per_pixel_in_tile;
let tile_audio_end = tile_audio_start + TILE_WIDTH_PIXELS as f64 * seconds_per_pixel_in_tile;
// Calculate which portion of this tile is visible in the trimmed clip
let visible_audio_start = tile_audio_start.max(trim_start);
let visible_audio_end = tile_audio_end.min(trim_start + clip_duration).min(audio_file_duration);
if visible_audio_start >= visible_audio_end {
continue; // No visible portion
}
// Calculate UV coordinates (only show the portion within trim bounds)
let uv_min_x = ((visible_audio_start - tile_audio_start) / (tile_audio_end - tile_audio_start)).max(0.0) as f32;
let uv_max_x = ((visible_audio_end - tile_audio_start) / (tile_audio_end - tile_audio_start)).min(1.0) as f32;
// Map audio file position to timeline position
// Audio time trim_start corresponds to timeline position clip_start_time
let tile_timeline_start = clip_start_time + (visible_audio_start - trim_start);
let tile_timeline_end = clip_start_time + (visible_audio_end - trim_start);
// Convert to screen space
let tile_screen_x = timeline_left_edge + ((tile_timeline_start - viewport_start_time) * pixels_per_second) as f32;
let tile_screen_width = ((tile_timeline_end - tile_timeline_start) * pixels_per_second) as f32;
// Create unclipped tile rect
let unclipped_tile_rect = egui::Rect::from_min_size(
egui::pos2(tile_screen_x, clip_rect.min.y),
egui::vec2(tile_screen_width, clip_rect.height()),
);
// Clip to the visible clip rectangle
let tile_rect = unclipped_tile_rect.intersect(clip_rect);
if tile_rect.width() <= 0.0 || tile_rect.height() <= 0.0 {
continue; // Nothing visible
}
// Adjust UV coordinates based on how much the tile was clipped
let uv_span = uv_max_x - uv_min_x;
let adjusted_uv_min_x = if unclipped_tile_rect.width() > 0.0 {
uv_min_x + ((tile_rect.min.x - unclipped_tile_rect.min.x) / unclipped_tile_rect.width()) * uv_span
} else {
uv_min_x
};
let adjusted_uv_max_x = if unclipped_tile_rect.width() > 0.0 {
uv_min_x + ((tile_rect.max.x - unclipped_tile_rect.min.x) / unclipped_tile_rect.width()) * uv_span
} else {
uv_max_x
};
// Blit texture with adjusted UV coordinates
painter.image(
texture.id(),
tile_rect,
egui::Rect::from_min_max(egui::pos2(adjusted_uv_min_x, 0.0), egui::pos2(adjusted_uv_max_x, 1.0)),
tint_color,
);
}
// Pre-cache adjacent tiles (non-blocking)
let precache_tiles = Self::calculate_precache_tiles(&visible_tiles, clip_rect.width());
// Create temporary HashMap with just this clip's waveform for pre-caching
let mut temp_waveform_cache = std::collections::HashMap::new();
temp_waveform_cache.insert(audio_pool_index, waveform_peaks.to_vec());
waveform_image_cache.precache_tiles(&precache_tiles, ctx, &temp_waveform_cache, audio_file_duration);
}
/// Render layer header column (left side with track names and controls)
fn render_layer_headers(
&mut self,
@ -1198,10 +893,9 @@ impl TimelinePane {
active_layer_id: &Option<uuid::Uuid>,
selection: &lightningbeam_core::selection::Selection,
midi_event_cache: &std::collections::HashMap<u32, Vec<(f64, u8, bool)>>,
waveform_cache: &std::collections::HashMap<usize, Vec<daw_backend::WaveformPeak>>,
waveform_chunk_cache: &std::collections::HashMap<(usize, u8, u32), Vec<daw_backend::WaveformPeak>>,
waveform_image_cache: &mut crate::waveform_image_cache::WaveformImageCache,
audio_controller: Option<&std::sync::Arc<std::sync::Mutex<daw_backend::EngineController>>>,
raw_audio_cache: &std::collections::HashMap<usize, (Vec<f32>, u32, u32)>,
waveform_gpu_dirty: &mut std::collections::HashSet<usize>,
target_format: wgpu::TextureFormat,
) -> Vec<(egui::Rect, uuid::Uuid, f64, f64)> {
let painter = ui.painter();
@ -1492,59 +1186,68 @@ impl TimelinePane {
);
}
}
// Sampled Audio: Draw waveform
// Sampled Audio: Draw waveform via GPU
lightningbeam_core::clip::AudioClipType::Sampled { audio_pool_index } => {
// Get audio file duration from backend
let audio_file_duration = if let Some(ref controller_arc) = audio_controller {
let mut controller = controller_arc.lock().unwrap();
controller.get_pool_file_info(*audio_pool_index)
.ok()
.map(|(duration, _, _)| duration)
.unwrap_or(clip.duration) // Fallback to clip duration
} else {
clip.duration // Fallback if no controller
};
if let Some((samples, sr, ch)) = raw_audio_cache.get(audio_pool_index) {
let total_frames = samples.len() / (*ch).max(1) as usize;
let audio_file_duration = total_frames as f64 / *sr as f64;
let screen_size = ui.ctx().screen_rect().size();
// Select detail level based on zoom
let requested_level = Self::select_detail_level(self.pixels_per_second as f64);
let pending_upload = if waveform_gpu_dirty.contains(audio_pool_index) {
waveform_gpu_dirty.remove(audio_pool_index);
Some(crate::waveform_gpu::PendingUpload {
samples: samples.clone(),
sample_rate: *sr,
channels: *ch,
})
} else {
None
};
// Try to assemble peaks from chunks with progressive fallback to lower detail levels
let mut peaks_to_render = None;
for level in (0..=requested_level).rev() {
if let Some(peaks) = Self::assemble_peaks_from_chunks(
waveform_chunk_cache,
*audio_pool_index,
level,
audio_file_duration,
audio_controller,
) {
peaks_to_render = Some(peaks);
break;
}
}
let tint = [
bright_color.r() as f32 / 255.0,
bright_color.g() as f32 / 255.0,
bright_color.b() as f32 / 255.0,
bright_color.a() as f32 / 255.0,
];
// Final fallback to old waveform_cache if no chunks available at any level
let peaks_to_render = peaks_to_render
.or_else(|| waveform_cache.get(audio_pool_index).cloned())
.unwrap_or_default();
if !peaks_to_render.is_empty() {
Self::render_audio_waveform(
painter,
clip_rect,
rect.min.x,
*audio_pool_index,
instance_start,
preview_clip_duration,
preview_trim_start,
audio_file_duration,
self.viewport_start_time,
self.pixels_per_second as f64,
waveform_image_cache,
&peaks_to_render,
ui.ctx(),
bright_color, // Use bright color for waveform (lighter than background)
let clip_screen_start = rect.min.x + ((instance_start - self.viewport_start_time) * self.pixels_per_second as f64) as f32;
let clip_screen_end = clip_screen_start + (preview_clip_duration * self.pixels_per_second as f64) as f32;
let waveform_rect = egui::Rect::from_min_max(
egui::pos2(clip_screen_start.max(clip_rect.min.x), clip_rect.min.y),
egui::pos2(clip_screen_end.min(clip_rect.max.x), clip_rect.max.y),
);
if waveform_rect.width() > 0.0 && waveform_rect.height() > 0.0 {
let callback = crate::waveform_gpu::WaveformCallback {
pool_index: *audio_pool_index,
segment_index: 0,
params: crate::waveform_gpu::WaveformParams {
clip_rect: [waveform_rect.min.x, waveform_rect.min.y, waveform_rect.max.x, waveform_rect.max.y],
viewport_start_time: self.viewport_start_time as f32,
pixels_per_second: self.pixels_per_second as f32,
audio_duration: audio_file_duration as f32,
sample_rate: *sr as f32,
clip_start_time: instance_start as f32,
trim_start: preview_trim_start as f32,
tex_width: crate::waveform_gpu::tex_width() as f32,
total_frames: total_frames as f32,
segment_start_frame: 0.0,
display_mode: 0.0,
_pad1: [0.0, 0.0],
tint_color: tint,
screen_size: [screen_size.x, screen_size.y],
_pad: [0.0, 0.0],
},
target_format,
pending_upload,
};
ui.painter().add(egui_wgpu::Callback::new_paint_callback(
waveform_rect,
callback,
));
}
}
}
// Recording in progress: no visualization yet
@ -2375,7 +2078,7 @@ impl PaneRenderer for TimelinePane {
// Render layer rows with clipping
ui.set_clip_rect(content_rect.intersect(original_clip_rect));
let video_clip_hovers = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.selection, shared.midi_event_cache, shared.waveform_cache, shared.waveform_chunk_cache, shared.waveform_image_cache, shared.audio_controller);
let video_clip_hovers = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.selection, shared.midi_event_cache, shared.raw_audio_cache, shared.waveform_gpu_dirty, shared.target_format);
// Render playhead on top (clip to timeline area)
ui.set_clip_rect(timeline_rect.intersect(original_clip_rect));

624
lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs Normal file
View File

@ -0,0 +1,624 @@
/// GPU-based waveform rendering using 2D textures with custom min/max mipmaps.
///
/// Raw audio samples are packed into Rgba16Float textures (R=left_min, G=left_max,
/// B=right_min, A=right_max). At mip 0, min=max=raw sample. Higher mip levels
/// are generated by a compute shader that reduces 4 consecutive samples per level.
///
/// Audio frames are packed row-major into 2D textures with a fixed power-of-2 width.
/// Long audio is split across multiple textures.
use std::collections::HashMap;
use wgpu;
use wgpu::util::DeviceExt;
/// Fixed texture width (power of 2) for all waveform textures
const TEX_WIDTH: u32 = 2048;
/// Maximum number of texture segments per audio clip
const MAX_SEGMENTS: u32 = 16;
/// GPU resources for all waveform textures, stored in CallbackResources
pub struct WaveformGpuResources {
/// Per-audio-pool-index GPU data
pub entries: HashMap<usize, WaveformGpuEntry>,
/// Shared render pipeline
render_pipeline: wgpu::RenderPipeline,
/// Shared mipgen compute pipeline
mipgen_pipeline: wgpu::ComputePipeline,
/// Bind group layout for render shader (texture + sampler + uniforms)
render_bind_group_layout: wgpu::BindGroupLayout,
/// Bind group layout for mipgen shader (src texture + dst storage + params)
mipgen_bind_group_layout: wgpu::BindGroupLayout,
/// Sampler for waveform texture (nearest, since we do manual LOD selection)
sampler: wgpu::Sampler,
}
/// GPU data for a single audio file
pub struct WaveformGpuEntry {
/// One texture per segment (for long audio split across multiple textures)
pub textures: Vec<wgpu::Texture>,
/// Texture views for each segment (full mip chain)
pub texture_views: Vec<wgpu::TextureView>,
/// Bind groups for the render shader (one per segment)
pub render_bind_groups: Vec<wgpu::BindGroup>,
/// Uniform buffers for each segment (updated per-frame via queue.write_buffer)
pub uniform_buffers: Vec<wgpu::Buffer>,
/// Frames covered by each texture segment
pub frames_per_segment: u32,
/// Total frame count
pub total_frames: u64,
/// Sample rate
pub sample_rate: u32,
/// Number of channels in source audio
pub channels: u32,
}
/// Parameters passed to the waveform render shader
#[repr(C)]
#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)]
pub struct WaveformParams {
pub clip_rect: [f32; 4],
pub viewport_start_time: f32,
pub pixels_per_second: f32,
pub audio_duration: f32,
pub sample_rate: f32,
pub clip_start_time: f32,
pub trim_start: f32,
pub tex_width: f32,
pub total_frames: f32,
pub segment_start_frame: f32,
pub display_mode: f32,
pub _pad1: [f32; 2], // align tint_color to 16 bytes (WGSL vec4 alignment)
pub tint_color: [f32; 4],
pub screen_size: [f32; 2],
pub _pad: [f32; 2],
}
/// Parameters for the mipgen compute shader
#[repr(C)]
#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)]
struct MipgenParams {
src_width: u32,
dst_width: u32,
src_sample_count: u32,
_pad: u32,
}
/// Callback for rendering a single waveform segment
pub struct WaveformCallback {
pub pool_index: usize,
pub segment_index: usize,
pub params: WaveformParams,
pub target_format: wgpu::TextureFormat,
/// Raw audio data for upload if this is the first time we see this pool_index
pub pending_upload: Option<PendingUpload>,
}
/// Raw audio data waiting to be uploaded to GPU
pub struct PendingUpload {
pub samples: Vec<f32>,
pub sample_rate: u32,
pub channels: u32,
}
impl WaveformGpuResources {
pub fn new(device: &wgpu::Device, target_format: wgpu::TextureFormat) -> Self {
// Render shader
let render_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("waveform_render_shader"),
source: wgpu::ShaderSource::Wgsl(include_str!("panes/shaders/waveform.wgsl").into()),
});
// Mipgen compute shader
let mipgen_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("waveform_mipgen_shader"),
source: wgpu::ShaderSource::Wgsl(
include_str!("panes/shaders/waveform_mipgen.wgsl").into(),
),
});
// Render bind group layout: texture + sampler + uniform buffer
let render_bind_group_layout =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("waveform_render_bgl"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
],
});
// Mipgen bind group layout: src texture + dst storage texture + params
let mipgen_bind_group_layout =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("waveform_mipgen_bgl"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: false },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::StorageTexture {
access: wgpu::StorageTextureAccess::WriteOnly,
format: wgpu::TextureFormat::Rgba16Float,
view_dimension: wgpu::TextureViewDimension::D2,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
],
});
// Render pipeline
let render_pipeline_layout =
device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("waveform_render_pipeline_layout"),
bind_group_layouts: &[&render_bind_group_layout],
push_constant_ranges: &[],
});
let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("waveform_render_pipeline"),
layout: Some(&render_pipeline_layout),
vertex: wgpu::VertexState {
module: &render_shader,
entry_point: Some("vs_main"),
buffers: &[],
compilation_options: Default::default(),
},
fragment: Some(wgpu::FragmentState {
module: &render_shader,
entry_point: Some("fs_main"),
targets: &[Some(wgpu::ColorTargetState {
format: target_format,
blend: Some(wgpu::BlendState::ALPHA_BLENDING),
write_mask: wgpu::ColorWrites::ALL,
})],
compilation_options: Default::default(),
}),
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleList,
..Default::default()
},
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview: None,
cache: None,
});
// Mipgen compute pipeline
let mipgen_pipeline_layout =
device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("waveform_mipgen_pipeline_layout"),
bind_group_layouts: &[&mipgen_bind_group_layout],
push_constant_ranges: &[],
});
let mipgen_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some("waveform_mipgen_pipeline"),
layout: Some(&mipgen_pipeline_layout),
module: &mipgen_shader,
entry_point: Some("main"),
compilation_options: Default::default(),
cache: None,
});
// Sampler: nearest filtering for explicit mip level selection
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
label: Some("waveform_sampler"),
mag_filter: wgpu::FilterMode::Nearest,
min_filter: wgpu::FilterMode::Nearest,
mipmap_filter: wgpu::FilterMode::Nearest,
..Default::default()
});
Self {
entries: HashMap::new(),
render_pipeline,
mipgen_pipeline,
render_bind_group_layout,
mipgen_bind_group_layout,
sampler,
}
}
/// Upload raw audio samples and generate mipmaps for a given pool index.
/// Returns command buffers that need to be submitted (for mipmap compute dispatches).
pub fn upload_audio(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
pool_index: usize,
samples: &[f32],
sample_rate: u32,
channels: u32,
) -> Vec<wgpu::CommandBuffer> {
// Remove old entry if exists
self.entries.remove(&pool_index);
let total_frames = samples.len() / channels.max(1) as usize;
if total_frames == 0 {
return Vec::new();
}
let max_frames_per_segment = (TEX_WIDTH as u64)
* (device.limits().max_texture_dimension_2d as u64);
let segment_count =
((total_frames as u64 + max_frames_per_segment - 1) / max_frames_per_segment) as usize;
let frames_per_segment = if segment_count == 1 {
total_frames as u32
} else {
(max_frames_per_segment as u32).min(total_frames as u32)
};
let mut textures = Vec::new();
let mut texture_views = Vec::new();
let mut render_bind_groups = Vec::new();
let mut uniform_buffers = Vec::new();
let mut all_command_buffers = Vec::new();
for seg in 0..segment_count {
let seg_start_frame = seg as u64 * frames_per_segment as u64;
let seg_end_frame = ((seg + 1) as u64 * frames_per_segment as u64)
.min(total_frames as u64);
let seg_frame_count = (seg_end_frame - seg_start_frame) as u32;
let tex_height = (seg_frame_count + TEX_WIDTH - 1) / TEX_WIDTH;
let mip_count = compute_mip_count(TEX_WIDTH, tex_height);
// Create texture with mip levels
let texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some(&format!("waveform_{}_seg{}", pool_index, seg)),
size: wgpu::Extent3d {
width: TEX_WIDTH,
height: tex_height,
depth_or_array_layers: 1,
},
mip_level_count: mip_count,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba16Float,
usage: wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::STORAGE_BINDING
| wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
// Pack raw samples into Rgba16Float data for mip 0
// R=left_min=left_sample, G=left_max=left_sample, B=right_min, A=right_max
let texel_count = (TEX_WIDTH * tex_height) as usize;
let mut mip0_data: Vec<half::f16> = vec![half::f16::ZERO; texel_count * 4];
for frame in 0..seg_frame_count as usize {
let global_frame = seg_start_frame as usize + frame;
let sample_offset = global_frame * channels as usize;
let left = if sample_offset < samples.len() {
samples[sample_offset]
} else {
0.0
};
let right = if channels >= 2 && sample_offset + 1 < samples.len() {
samples[sample_offset + 1]
} else {
left // Mono: duplicate left to right
};
let texel_offset = frame * 4;
mip0_data[texel_offset] = half::f16::from_f32(left); // R = left_min
mip0_data[texel_offset + 1] = half::f16::from_f32(left); // G = left_max
mip0_data[texel_offset + 2] = half::f16::from_f32(right); // B = right_min
mip0_data[texel_offset + 3] = half::f16::from_f32(right); // A = right_max
}
// Upload mip 0
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
bytemuck::cast_slice(&mip0_data),
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(TEX_WIDTH * 8), // 4 channels × 2 bytes (f16)
rows_per_image: Some(tex_height),
},
wgpu::Extent3d {
width: TEX_WIDTH,
height: tex_height,
depth_or_array_layers: 1,
},
);
// Generate mipmaps via compute shader
let cmds = self.generate_mipmaps(
device,
&texture,
TEX_WIDTH,
tex_height,
mip_count,
seg_frame_count,
);
all_command_buffers.extend(cmds);
// Create view for full mip chain
let view = texture.create_view(&wgpu::TextureViewDescriptor {
label: Some(&format!("waveform_{}_seg{}_view", pool_index, seg)),
..Default::default()
});
// Create uniform buffer placeholder (will be filled per-draw in paint)
let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor {
label: Some(&format!("waveform_{}_seg{}_uniforms", pool_index, seg)),
size: std::mem::size_of::<WaveformParams>() as u64,
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
// Create render bind group
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some(&format!("waveform_{}_seg{}_bg", pool_index, seg)),
layout: &self.render_bind_group_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::TextureView(&view),
},
wgpu::BindGroupEntry {
binding: 1,
resource: wgpu::BindingResource::Sampler(&self.sampler),
},
wgpu::BindGroupEntry {
binding: 2,
resource: uniform_buffer.as_entire_binding(),
},
],
});
textures.push(texture);
texture_views.push(view);
render_bind_groups.push(bind_group);
uniform_buffers.push(uniform_buffer);
}
self.entries.insert(
pool_index,
WaveformGpuEntry {
textures,
texture_views,
render_bind_groups,
uniform_buffers,
frames_per_segment,
total_frames: total_frames as u64,
sample_rate,
channels,
},
);
all_command_buffers
}
/// Generate mipmaps for a texture using the compute shader.
fn generate_mipmaps(
&self,
device: &wgpu::Device,
texture: &wgpu::Texture,
base_width: u32,
base_height: u32,
mip_count: u32,
base_sample_count: u32,
) -> Vec<wgpu::CommandBuffer> {
if mip_count <= 1 {
return Vec::new();
}
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("waveform_mipgen_encoder"),
});
let mut src_width = base_width;
let mut src_height = base_height;
let mut src_sample_count = base_sample_count;
for level in 1..mip_count {
let dst_width = (src_width / 2).max(1);
let dst_height = (src_height / 2).max(1);
let dst_sample_count = (src_sample_count + 3) / 4; // ceil(src/4)
// Create views for specific mip levels
let src_view = texture.create_view(&wgpu::TextureViewDescriptor {
label: Some(&format!("mipgen_src_level_{}", level - 1)),
base_mip_level: level - 1,
mip_level_count: Some(1),
..Default::default()
});
let dst_view = texture.create_view(&wgpu::TextureViewDescriptor {
label: Some(&format!("mipgen_dst_level_{}", level)),
base_mip_level: level,
mip_level_count: Some(1),
..Default::default()
});
// Create params buffer
let params = MipgenParams {
src_width,
dst_width,
src_sample_count,
_pad: 0,
};
let params_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some(&format!("mipgen_params_level_{}", level)),
contents: bytemuck::cast_slice(&[params]),
usage: wgpu::BufferUsages::UNIFORM,
});
// Create bind group for this dispatch
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some(&format!("mipgen_bg_level_{}", level)),
layout: &self.mipgen_bind_group_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::TextureView(&src_view),
},
wgpu::BindGroupEntry {
binding: 1,
resource: wgpu::BindingResource::TextureView(&dst_view),
},
wgpu::BindGroupEntry {
binding: 2,
resource: params_buffer.as_entire_binding(),
},
],
});
// Dispatch compute
let total_dst_texels = dst_width * dst_height;
let workgroup_count = (total_dst_texels + 63) / 64;
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some(&format!("mipgen_pass_level_{}", level)),
timestamp_writes: None,
});
pass.set_pipeline(&self.mipgen_pipeline);
pass.set_bind_group(0, &bind_group, &[]);
pass.dispatch_workgroups(workgroup_count, 1, 1);
drop(pass);
src_width = dst_width;
src_height = dst_height;
src_sample_count = dst_sample_count;
}
vec![encoder.finish()]
}
}
impl egui_wgpu::CallbackTrait for WaveformCallback {
fn prepare(
&self,
device: &wgpu::Device,
queue: &wgpu::Queue,
_screen_descriptor: &egui_wgpu::ScreenDescriptor,
_egui_encoder: &mut wgpu::CommandEncoder,
resources: &mut egui_wgpu::CallbackResources,
) -> Vec<wgpu::CommandBuffer> {
// Initialize resources if needed
if !resources.contains::<WaveformGpuResources>() {
resources.insert(WaveformGpuResources::new(device, self.target_format));
}
let gpu_resources: &mut WaveformGpuResources = resources.get_mut().unwrap();
// Upload audio data if pending
let mut cmds = Vec::new();
if let Some(ref upload) = self.pending_upload {
let new_cmds = gpu_resources.upload_audio(
device,
queue,
self.pool_index,
&upload.samples,
upload.sample_rate,
upload.channels,
);
cmds.extend(new_cmds);
}
// Update uniform buffer for this draw
if let Some(entry) = gpu_resources.entries.get(&self.pool_index) {
if self.segment_index < entry.uniform_buffers.len() {
queue.write_buffer(
&entry.uniform_buffers[self.segment_index],
0,
bytemuck::cast_slice(&[self.params]),
);
}
}
cmds
}
fn paint(
&self,
_info: eframe::egui::PaintCallbackInfo,
render_pass: &mut wgpu::RenderPass<'static>,
resources: &egui_wgpu::CallbackResources,
) {
let gpu_resources: &WaveformGpuResources = match resources.get() {
Some(r) => r,
None => return,
};
let entry = match gpu_resources.entries.get(&self.pool_index) {
Some(e) => e,
None => return,
};
if self.segment_index >= entry.render_bind_groups.len() {
return;
}
render_pass.set_pipeline(&gpu_resources.render_pipeline);
render_pass.set_bind_group(0, &entry.render_bind_groups[self.segment_index], &[]);
render_pass.draw(0..3, 0..1); // Fullscreen triangle
}
}
/// Compute number of mip levels for given dimensions
fn compute_mip_count(width: u32, height: u32) -> u32 {
let max_dim = width.max(height);
(max_dim as f32).log2().floor() as u32 + 1
}
/// Calculate how many texture segments are needed for a given frame count
pub fn segment_count_for_frames(total_frames: u64, max_texture_height: u32) -> u32 {
let max_frames_per_segment = TEX_WIDTH as u64 * max_texture_height as u64;
((total_frames + max_frames_per_segment - 1) / max_frames_per_segment) as u32
}
/// Get the fixed texture width used for all waveform textures
pub fn tex_width() -> u32 {
TEX_WIDTH
}

326
lightningbeam-ui/lightningbeam-editor/src/waveform_image_cache.rs
View File

@ -1,326 +0,0 @@
use eframe::egui;
use std::collections::{HashMap, VecDeque};
use std::time::Instant;
/// Tile width is constant at 1024 pixels per tile
pub const TILE_WIDTH_PIXELS: usize = 1024;
/// Unique identifier for a cached waveform image tile
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct WaveformCacheKey {
/// Audio pool index from backend
pub audio_pool_index: usize,
/// Zoom bucket (power of 2: 1, 2, 4, 8, 16, etc.)
pub zoom_bucket: u32,
/// Tile index (which tile in the sequence for this audio clip)
pub tile_index: u32,
/// Clip height in pixels (for cache invalidation on resize)
pub height: u32,
}
/// Cached waveform image with metadata
pub struct CachedWaveform {
/// The rendered texture handle
pub texture: egui::TextureHandle,
/// Size in bytes (for memory tracking)
pub size_bytes: usize,
/// Last access time (for LRU eviction)
pub last_accessed: Instant,
/// Width of the image in pixels
pub width_pixels: u32,
/// Height of the image in pixels
pub height_pixels: u32,
}
/// Main cache structure
pub struct WaveformImageCache {
/// Map from cache key to rendered texture
cache: HashMap<WaveformCacheKey, CachedWaveform>,
/// LRU queue (most recent at back)
lru_queue: VecDeque<WaveformCacheKey>,
/// Current total memory usage in bytes
total_bytes: usize,
/// Maximum memory usage (100 MB default)
max_bytes: usize,
/// Statistics
hits: u64,
misses: u64,
}
impl WaveformImageCache {
/// Create a new waveform image cache with 100 MB limit
pub fn new() -> Self {
Self {
cache: HashMap::new(),
lru_queue: VecDeque::new(),
total_bytes: 0,
max_bytes: 100 * 1024 * 1024, // 100 MB
hits: 0,
misses: 0,
}
}
/// Clear all cached textures
pub fn clear(&mut self) {
self.cache.clear();
self.lru_queue.clear();
self.total_bytes = 0;
// Note: hits/misses preserved for debugging
}
/// Get cache statistics: (hits, misses, total_bytes, num_entries)
pub fn stats(&self) -> (u64, u64, usize, usize) {
(self.hits, self.misses, self.total_bytes, self.cache.len())
}
/// Evict least recently used entries until under memory limit
fn evict_lru(&mut self) {
while self.total_bytes > self.max_bytes && !self.lru_queue.is_empty() {
if let Some(key) = self.lru_queue.pop_front() {
if let Some(cached) = self.cache.remove(&key) {
self.total_bytes -= cached.size_bytes;
// Texture automatically freed when CachedWaveform dropped
}
}
}
}
/// Update LRU queue when a key is accessed
fn touch(&mut self, key: WaveformCacheKey) {
// Remove key from its current position in LRU queue
self.lru_queue.retain(|&k| k != key);
// Add to back (most recent)
self.lru_queue.push_back(key);
}
/// Get cached texture or generate new one
pub fn get_or_create(
&mut self,
key: WaveformCacheKey,
ctx: &egui::Context,
waveform: &[daw_backend::WaveformPeak],
audio_file_duration: f64,
) -> egui::TextureHandle {
// Check if already cached
let texture = if let Some(cached) = self.cache.get_mut(&key) {
// Cache hit
self.hits += 1;
cached.last_accessed = Instant::now();
Some(cached.texture.clone())
} else {
None
};
if let Some(texture) = texture {
self.touch(key);
return texture;
}
// Cache miss - generate new tile
self.misses += 1;
// Render waveform to image
let color_image = render_waveform_to_image(
waveform,
key.tile_index,
audio_file_duration,
key.zoom_bucket,
key.height,
);
// Upload to GPU as texture
let texture_name = format!(
"waveform_{}_{}_{}",
key.audio_pool_index, key.zoom_bucket, key.tile_index
);
let texture = ctx.load_texture(
texture_name,
color_image,
egui::TextureOptions::LINEAR,
);
// Calculate memory usage
let size_bytes = TILE_WIDTH_PIXELS * key.height as usize * 4;
// Store in cache
let cached = CachedWaveform {
texture: texture.clone(),
size_bytes,
last_accessed: Instant::now(),
width_pixels: TILE_WIDTH_PIXELS as u32,
height_pixels: key.height,
};
self.total_bytes += size_bytes;
self.cache.insert(key, cached);
self.touch(key);
// Evict if over limit
self.evict_lru();
texture
}
/// Pre-cache tiles for smooth scrolling
pub fn precache_tiles(
&mut self,
keys: &[WaveformCacheKey],
ctx: &egui::Context,
waveform_peak_cache: &HashMap<usize, Vec<daw_backend::WaveformPeak>>,
audio_file_duration: f64,
) {
// Limit pre-caching to avoid frame time spike
const MAX_PRECACHE_PER_FRAME: usize = 2;
let mut precached = 0;
for key in keys {
if precached >= MAX_PRECACHE_PER_FRAME {
break;
}
// Skip if already cached
if self.cache.contains_key(key) {
continue;
}
// Get waveform peaks
if let Some(waveform) = waveform_peak_cache.get(&key.audio_pool_index) {
// Generate and cache
let _ = self.get_or_create(*key, ctx, waveform, audio_file_duration);
precached += 1;
}
}
}
/// Remove all entries for a specific audio file
pub fn invalidate_audio(&mut self, audio_pool_index: usize) {
let keys_to_remove: Vec<WaveformCacheKey> = self
.cache
.keys()
.filter(|k| k.audio_pool_index == audio_pool_index)
.copied()
.collect();
for key in keys_to_remove {
if let Some(cached) = self.cache.remove(&key) {
self.total_bytes -= cached.size_bytes;
}
}
// Also clean up LRU queue
self.lru_queue.retain(|key| key.audio_pool_index != audio_pool_index);
}
/// Remove all entries with a specific height (for window resize)
pub fn invalidate_height(&mut self, old_height: u32) {
let keys_to_remove: Vec<WaveformCacheKey> = self
.cache
.keys()
.filter(|k| k.height == old_height)
.copied()
.collect();
for key in keys_to_remove {
if let Some(cached) = self.cache.remove(&key) {
self.total_bytes -= cached.size_bytes;
}
}
// Also clean up LRU queue
self.lru_queue.retain(|key| key.height != old_height);
}
}
impl Default for WaveformImageCache {
fn default() -> Self {
Self::new()
}
}
/// Calculate zoom bucket from pixels_per_second
/// Rounds to nearest power of 2: 1, 2, 4, 8, 16, 32, 64, 128, 256
pub fn calculate_zoom_bucket(pixels_per_second: f64) -> u32 {
if pixels_per_second <= 1.0 {
return 1;
}
// Round to nearest power of 2
let log2 = pixels_per_second.log2();
let rounded = log2.round();
2u32.pow(rounded as u32)
}
/// Render a waveform tile to a ColorImage
fn render_waveform_to_image(
waveform: &[daw_backend::WaveformPeak],
tile_index: u32,
audio_file_duration: f64,
zoom_bucket: u32,
height: u32,
) -> egui::ColorImage {
let width = TILE_WIDTH_PIXELS;
let height = height as usize;
// Create RGBA buffer (transparent background)
let mut pixels = vec![0u8; width * height * 4];
// Render as white - will be tinted at render time with clip background color
let waveform_color = egui::Color32::WHITE;
// Calculate time range for this tile (tiles represent fixed portions of the audio file)
// Each pixel represents (1.0 / zoom_bucket) seconds
let seconds_per_pixel = 1.0 / zoom_bucket as f64;
let tile_start_time = tile_index as f64 * TILE_WIDTH_PIXELS as f64 * seconds_per_pixel;
let tile_end_time = (tile_start_time + width as f64 * seconds_per_pixel).min(audio_file_duration);
// Calculate which waveform peaks correspond to this tile
let peak_start_idx = ((tile_start_time / audio_file_duration) * waveform.len() as f64) as usize;
let peak_end_idx = ((tile_end_time / audio_file_duration) * waveform.len() as f64) as usize;
let peak_end_idx = peak_end_idx.min(waveform.len());
if peak_start_idx >= waveform.len() {
// Tile is beyond the end of the audio clip - return transparent image
return egui::ColorImage::from_rgba_unmultiplied([width, height], &pixels);
}
let tile_peaks = &waveform[peak_start_idx..peak_end_idx];
if tile_peaks.is_empty() {
return egui::ColorImage::from_rgba_unmultiplied([width, height], &pixels);
}
// Calculate the actual time range this tile covers in the audio file
// This may be less than the full tile width if the audio file is shorter than the tile's time span
let actual_time_covered = tile_end_time - tile_start_time;
let actual_pixel_width = (actual_time_covered / seconds_per_pixel).min(width as f64);
// Render waveform to pixel buffer
// Distribute peaks only across the valid pixel range, not the entire tile width
let pixels_per_peak = actual_pixel_width / tile_peaks.len() as f64;
for (peak_idx, peak) in tile_peaks.iter().enumerate() {
let x_start = (peak_idx as f64 * pixels_per_peak).floor() as usize;
let x_end = ((peak_idx + 1) as f64 * pixels_per_peak).ceil() as usize;
let x_end = x_end.min(width);
// Calculate Y range for this peak
let center_y = height as f64 / 2.0;
let max_y = (center_y + (peak.max as f64 * height as f64 * 0.45)).round() as usize;
let min_y = (center_y + (peak.min as f64 * height as f64 * 0.45)).round() as usize;
let min_y = min_y.min(height - 1);
let max_y = max_y.min(height - 1);
// Fill vertical span for this peak
for x in x_start..x_end {
for y in min_y..=max_y {
let pixel_idx = (y * width + x) * 4;
pixels[pixel_idx] = waveform_color.r();
pixels[pixel_idx + 1] = waveform_color.g();
pixels[pixel_idx + 2] = waveform_color.b();
pixels[pixel_idx + 3] = waveform_color.a();
}
}
}
egui::ColorImage::from_rgba_unmultiplied([width, height], &pixels)
}