skyler
/
Lightningbeam
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix lag spike when first displaying audio clip

This commit is contained in:
Skyler Lehmkuhl 2026-03-01 09:22:18 -05:00
parent b87e4325c2
commit 6bd400d353
2 changed files with 59 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs
View File

@ -181,6 +181,10 @@ pub struct TimelinePane {
/// Whether to display time as seconds or measures
time_display_format: TimeDisplayFormat,
/// Waveform upload progress: pool_index -> frames uploaded so far.
/// Tracks chunked GPU uploads across frames to avoid hitches.
waveform_upload_progress: std::collections::HashMap<usize, usize>,
}
/// Check if a clip type can be dropped on a layer type
@ -367,6 +371,7 @@ impl TimelinePane {
layer_control_clicked: false,
context_menu_clip: None,
time_display_format: TimeDisplayFormat::Seconds,
waveform_upload_progress: std::collections::HashMap::new(),
}
}
@ -1497,7 +1502,7 @@ impl TimelinePane {
/// Render layer rows (timeline content area)
/// Returns video clip hover data for processing after input handling
fn render_layers(
&self,
&mut self,
ui: &mut egui::Ui,
rect: egui::Rect,
theme: &crate::theme::Theme,
@ -2128,11 +2133,27 @@ impl TimelinePane {
let screen_size = ui.ctx().content_rect().size();
let pending_upload = if waveform_gpu_dirty.contains(audio_pool_index) {
waveform_gpu_dirty.remove(audio_pool_index);
// Chunked upload: track progress across frames
let chunk = crate::waveform_gpu::UPLOAD_CHUNK_FRAMES;
let progress = self.waveform_upload_progress.get(audio_pool_index).copied().unwrap_or(0);
let next_end = (progress + chunk).min(total_frames);
let frame_limit = Some(next_end);
if next_end >= total_frames {
// Final chunk — done
waveform_gpu_dirty.remove(audio_pool_index);
self.waveform_upload_progress.remove(audio_pool_index);
} else {
// More chunks needed
self.waveform_upload_progress.insert(*audio_pool_index, next_end);
ui.ctx().request_repaint();
}
Some(crate::waveform_gpu::PendingUpload {
samples: samples.clone(),
sample_rate: *sr,
channels: *ch,
frame_limit,
})
} else {
None
@ -2239,6 +2260,7 @@ impl TimelinePane {
samples: samples.clone(),
sample_rate: *sr,
channels: *ch,
frame_limit: None, // recording uses incremental path
})
} else {
None

50
lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs
View File

@ -108,8 +108,16 @@ pub struct PendingUpload {
pub samples: std::sync::Arc<Vec<f32>>,
pub sample_rate: u32,
pub channels: u32,
/// If set, only upload up to this many frames (for chunked uploads).
/// The texture is allocated at full size, but total_frames is set to
/// the limited count so subsequent calls use the incremental path.
pub frame_limit: Option<usize>,
}
/// Maximum frames to convert and upload per frame (~250K frames ≈ 5.6s at 44.1kHz).
/// Keeps the CPU f32→f16 conversion under ~2-3ms per frame.
pub const UPLOAD_CHUNK_FRAMES: usize = 250_000;
impl WaveformGpuResources {
pub fn new(device: &wgpu::Device, target_format: wgpu::TextureFormat) -> Self {
// Render shader
@ -282,18 +290,22 @@ impl WaveformGpuResources {
samples: &[f32],
sample_rate: u32,
channels: u32,
frame_limit: Option<usize>,
) -> Vec<wgpu::CommandBuffer> {
let new_total_frames = samples.len() / channels.max(1) as usize;
if new_total_frames == 0 {
return Vec::new();
}
// For incremental path, also respect frame_limit
let effective_frames = frame_limit.map_or(new_total_frames, |lim| lim.min(new_total_frames));
// If entry exists and texture is large enough, do an incremental update
let incremental = if let Some(entry) = self.entries.get(&pool_index) {
let new_tex_height = (new_total_frames as u32 + TEX_WIDTH - 1) / TEX_WIDTH;
if new_tex_height <= entry.tex_height && new_total_frames > entry.total_frames as usize {
if new_tex_height <= entry.tex_height && effective_frames > entry.total_frames as usize {
Some((entry.total_frames as usize, entry.tex_height))
} else if new_total_frames <= entry.total_frames as usize {
} else if effective_frames <= entry.total_frames as usize {
return Vec::new(); // No new data
} else {
None // Texture too small, need full recreate
@ -305,7 +317,7 @@ impl WaveformGpuResources {
if let Some((old_frames, tex_height)) = incremental {
// Write only the NEW rows into the existing texture
let start_row = old_frames as u32 / TEX_WIDTH;
let end_row = (new_total_frames as u32 + TEX_WIDTH - 1) / TEX_WIDTH;
let end_row = (effective_frames as u32 + TEX_WIDTH - 1) / TEX_WIDTH;
let rows_to_write = end_row - start_row;
let row_texel_count = (TEX_WIDTH * rows_to_write) as usize;
@ -314,7 +326,7 @@ impl WaveformGpuResources {
let row_start_frame = start_row as usize * TEX_WIDTH as usize;
for frame in 0..(rows_to_write as usize * TEX_WIDTH as usize) {
let global_frame = row_start_frame + frame;
if global_frame >= new_total_frames {
if global_frame >= effective_frames {
break;
}
let sample_offset = global_frame * channels as usize;
@ -364,11 +376,11 @@ impl WaveformGpuResources {
TEX_WIDTH,
tex_height,
mip_count,
new_total_frames as u32,
effective_frames as u32,
);
// Update total_frames after borrow of entry is done
self.entries.get_mut(&pool_index).unwrap().total_frames = new_total_frames as u64;
self.entries.get_mut(&pool_index).unwrap().total_frames = effective_frames as u64;
return cmds;
}
@ -378,12 +390,15 @@ impl WaveformGpuResources {
self.per_instance.retain(|&(pi, _), _| pi != pool_index);
let total_frames = new_total_frames;
// Upload only effective_frames worth of data on this call
let upload_frames = effective_frames;
// For live recording (pool_index == usize::MAX), pre-allocate extra texture
// height to avoid frequent full recreates as recording grows.
// Allocate 60 seconds ahead so incremental updates can fill without recreating.
// When chunking, always allocate for the full total so incremental updates fit.
let alloc_frames = if pool_index == usize::MAX {
let extra = sample_rate as usize * 60; // 60s of mono frames (texture is per-frame, not per-sample)
let extra = sample_rate as usize * 60;
total_frames + extra
} else {
total_frames
@ -411,12 +426,16 @@ impl WaveformGpuResources {
let seg_end_frame = ((seg + 1) as u64 * frames_per_segment as u64)
.min(total_frames as u64);
let seg_frame_count = (seg_end_frame - seg_start_frame) as u32;
// Limit actual data processing to upload_frames (for chunked uploads)
let seg_upload_end = ((seg + 1) as u64 * frames_per_segment as u64)
.min(upload_frames as u64);
let seg_upload_count = seg_upload_end.saturating_sub(seg_start_frame) as u32;
// Allocate texture large enough for future growth (recording) or exact fit (normal)
// Allocate texture large enough for the FULL data (not just this chunk)
let alloc_seg_frames = if pool_index == usize::MAX {
(alloc_frames as u32).min(seg_frame_count + sample_rate * 60)
} else {
seg_frame_count
seg_frame_count // full size so incremental updates fit
};
let tex_height = (alloc_seg_frames + TEX_WIDTH - 1) / TEX_WIDTH;
let mip_count = compute_mip_count(TEX_WIDTH, tex_height);
@ -440,12 +459,12 @@ impl WaveformGpuResources {
});
// Pack raw samples into Rgba16Float data for mip 0
// Only pack rows containing actual data (not the pre-allocated empty region)
let data_height = (seg_frame_count + TEX_WIDTH - 1) / TEX_WIDTH;
// Only pack rows containing data uploaded this chunk
let data_height = (seg_upload_count + TEX_WIDTH - 1) / TEX_WIDTH;
let data_texel_count = (TEX_WIDTH * data_height) as usize;
let mut mip0_data: Vec<half::f16> = vec![half::f16::ZERO; data_texel_count * 4];
for frame in 0..seg_frame_count as usize {
for frame in 0..seg_upload_count as usize {
let global_frame = seg_start_frame as usize + frame;
let sample_offset = global_frame * channels as usize;
@ -490,14 +509,14 @@ impl WaveformGpuResources {
);
}
// Generate mipmaps via compute shader
// Generate mipmaps via compute shader (only for uploaded data)
let cmds = self.generate_mipmaps(
device,
&texture,
TEX_WIDTH,
tex_height,
mip_count,
seg_frame_count,
seg_upload_count,
);
all_command_buffers.extend(cmds);
@ -549,7 +568,7 @@ impl WaveformGpuResources {
render_bind_groups,
uniform_buffers,
frames_per_segment,
total_frames: total_frames as u64,
total_frames: upload_frames as u64, // only what was uploaded this chunk
tex_height: (alloc_frames as u32 + TEX_WIDTH - 1) / TEX_WIDTH,
sample_rate,
channels,
@ -681,6 +700,7 @@ impl egui_wgpu::CallbackTrait for WaveformCallback {
&upload.samples,
upload.sample_rate,
upload.channels,
upload.frame_limit,
);
cmds.extend(new_cmds);
}