Add spectrogram
This commit is contained in:
Skyler Lehmkuhl
parent
ad81cce0c6
commit
c11dab928c
|
|
@ -24,6 +24,7 @@ pub mod create_folder;
|
|||
pub mod rename_folder;
|
||||
pub mod delete_folder;
|
||||
pub mod move_asset_to_folder;
|
||||
pub mod update_midi_notes;
|
||||
|
||||
pub use add_clip_instance::AddClipInstanceAction;
|
||||
pub use add_effect::AddEffectAction;
|
||||
|
|
@ -46,3 +47,4 @@ pub use create_folder::CreateFolderAction;
|
|||
pub use rename_folder::RenameFolderAction;
|
||||
pub use delete_folder::{DeleteFolderAction, DeleteStrategy};
|
||||
pub use move_asset_to_folder::MoveAssetToFolderAction;
|
||||
pub use update_midi_notes::UpdateMidiNotesAction;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
use crate::action::Action;
|
||||
use crate::document::Document;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Action to update MIDI notes in a clip (supports undo/redo)
|
||||
///
|
||||
/// Stores the before and after note states. MIDI note data lives in the backend,
|
||||
/// so execute/rollback are no-ops on the document — all changes go through
|
||||
/// execute_backend/rollback_backend.
|
||||
pub struct UpdateMidiNotesAction {
|
||||
/// Layer containing the MIDI clip
|
||||
pub layer_id: Uuid,
|
||||
/// Backend MIDI clip ID
|
||||
pub midi_clip_id: u32,
|
||||
/// Notes before the edit: (start_time, note, velocity, duration)
|
||||
pub old_notes: Vec<(f64, u8, u8, f64)>,
|
||||
/// Notes after the edit: (start_time, note, velocity, duration)
|
||||
pub new_notes: Vec<(f64, u8, u8, f64)>,
|
||||
/// Human-readable description
|
||||
pub description_text: String,
|
||||
}
|
||||
|
||||
impl Action for UpdateMidiNotesAction {
|
||||
fn execute(&mut self, _document: &mut Document) -> Result<(), String> {
|
||||
// MIDI note data lives in the backend, not the document
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rollback(&mut self, _document: &mut Document) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn description(&self) -> String {
|
||||
self.description_text.clone()
|
||||
}
|
||||
|
||||
fn execute_backend(
|
||||
&mut self,
|
||||
backend: &mut crate::action::BackendContext,
|
||||
_document: &Document,
|
||||
) -> Result<(), String> {
|
||||
let controller = match backend.audio_controller.as_mut() {
|
||||
Some(c) => c,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
let track_id = backend
|
||||
.layer_to_track_map
|
||||
.get(&self.layer_id)
|
||||
.ok_or_else(|| format!("Layer {} not mapped to backend track", self.layer_id))?;
|
||||
|
||||
controller.update_midi_clip_notes(*track_id, self.midi_clip_id, self.new_notes.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn rollback_backend(
|
||||
&mut self,
|
||||
backend: &mut crate::action::BackendContext,
|
||||
_document: &Document,
|
||||
) -> Result<(), String> {
|
||||
let controller = match backend.audio_controller.as_mut() {
|
||||
Some(c) => c,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
let track_id = backend
|
||||
.layer_to_track_map
|
||||
.get(&self.layer_id)
|
||||
.ok_or_else(|| format!("Layer {} not mapped to backend track", self.layer_id))?;
|
||||
|
||||
controller.update_midi_clip_notes(*track_id, self.midi_clip_id, self.old_notes.clone());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,8 @@ mod theme;
|
|||
use theme::{Theme, ThemeMode};
|
||||
|
||||
mod waveform_gpu;
|
||||
mod spectrogram_gpu;
|
||||
mod spectrogram_compute;
|
||||
|
||||
mod config;
|
||||
use config::AppConfig;
|
||||
|
|
@ -687,8 +689,8 @@ struct EditorApp {
|
|||
|
||||
/// Cache for MIDI event data (keyed by backend midi_clip_id)
|
||||
/// Prevents repeated backend queries for the same MIDI clip
|
||||
/// Format: (timestamp, note_number, is_note_on)
|
||||
midi_event_cache: HashMap<u32, Vec<(f64, u8, bool)>>,
|
||||
/// Format: (timestamp, note_number, velocity, is_note_on)
|
||||
midi_event_cache: HashMap<u32, Vec<(f64, u8, u8, bool)>>,
|
||||
/// Cache for audio file durations to avoid repeated queries
|
||||
/// Format: pool_index -> duration in seconds
|
||||
audio_duration_cache: HashMap<usize, f64>,
|
||||
|
|
@ -2395,15 +2397,15 @@ impl EditorApp {
|
|||
let duration = midi_clip.duration;
|
||||
let event_count = midi_clip.events.len();
|
||||
|
||||
// Process MIDI events to cache format: (timestamp, note_number, is_note_on)
|
||||
// Process MIDI events to cache format: (timestamp, note_number, velocity, is_note_on)
|
||||
// Filter to note events only (status 0x90 = note-on, 0x80 = note-off)
|
||||
let processed_events: Vec<(f64, u8, bool)> = midi_clip.events.iter()
|
||||
let processed_events: Vec<(f64, u8, u8, bool)> = midi_clip.events.iter()
|
||||
.filter_map(|event| {
|
||||
let status_type = event.status & 0xF0;
|
||||
if status_type == 0x90 || status_type == 0x80 {
|
||||
// Note-on is 0x90 with velocity > 0, Note-off is 0x80 or velocity = 0
|
||||
let is_note_on = status_type == 0x90 && event.data2 > 0;
|
||||
Some((event.timestamp, event.data1, is_note_on))
|
||||
Some((event.timestamp, event.data1, event.data2, is_note_on))
|
||||
} else {
|
||||
None // Ignore non-note events (CC, pitch bend, etc.)
|
||||
}
|
||||
|
|
@ -4026,7 +4028,7 @@ struct RenderContext<'a> {
|
|||
/// Mapping from Document layer UUIDs to daw-backend TrackIds
|
||||
layer_to_track_map: &'a std::collections::HashMap<Uuid, daw_backend::TrackId>,
|
||||
/// Cache of MIDI events for rendering (keyed by backend midi_clip_id)
|
||||
midi_event_cache: &'a HashMap<u32, Vec<(f64, u8, bool)>>,
|
||||
midi_event_cache: &'a HashMap<u32, Vec<(f64, u8, u8, bool)>>,
|
||||
/// Audio pool indices with new raw audio data this frame (for thumbnail invalidation)
|
||||
audio_pools_with_new_waveforms: &'a HashSet<usize>,
|
||||
/// Raw audio samples for GPU waveform rendering (pool_index -> (samples, sample_rate, channels))
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ fn generate_video_thumbnail(
|
|||
/// Generate a piano roll thumbnail for MIDI clips
|
||||
/// Shows notes as horizontal bars with Y position = note % 12 (one octave)
|
||||
fn generate_midi_thumbnail(
|
||||
events: &[(f64, u8, bool)], // (timestamp, note_number, is_note_on)
|
||||
events: &[(f64, u8, u8, bool)], // (timestamp, note_number, velocity, is_note_on)
|
||||
duration: f64,
|
||||
bg_color: egui::Color32,
|
||||
note_color: egui::Color32,
|
||||
|
|
@ -394,7 +394,7 @@ fn generate_midi_thumbnail(
|
|||
}
|
||||
|
||||
// Draw note events
|
||||
for &(timestamp, note_number, is_note_on) in events {
|
||||
for &(timestamp, note_number, _velocity, is_note_on) in events {
|
||||
if !is_note_on || timestamp > preview_duration {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ pub struct SharedPaneState<'a> {
|
|||
/// Number of sides for polygon tool
|
||||
pub polygon_sides: &'a mut u32,
|
||||
/// Cache of MIDI events for rendering (keyed by backend midi_clip_id)
|
||||
pub midi_event_cache: &'a std::collections::HashMap<u32, Vec<(f64, u8, bool)>>,
|
||||
pub midi_event_cache: &'a std::collections::HashMap<u32, Vec<(f64, u8, u8, bool)>>,
|
||||
/// Audio pool indices that got new raw audio data this frame (for thumbnail invalidation)
|
||||
pub audio_pools_with_new_waveforms: &'a std::collections::HashSet<usize>,
|
||||
/// Raw audio samples for GPU waveform rendering (pool_index -> (samples, sample_rate, channels))
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,153 @@
|
|||
// Spectrogram rendering shader for FFT magnitude data.
|
||||
// Texture layout: X = frequency bin, Y = time bin
|
||||
// Values: normalized magnitude (0.0 = silence, 1.0 = peak)
|
||||
// Vertical axis maps MIDI notes to frequency bins (matching piano roll)
|
||||
|
||||
struct Params {
|
||||
clip_rect: vec4<f32>,
|
||||
viewport_start_time: f32,
|
||||
pixels_per_second: f32,
|
||||
audio_duration: f32,
|
||||
sample_rate: f32,
|
||||
clip_start_time: f32,
|
||||
trim_start: f32,
|
||||
time_bins: f32,
|
||||
freq_bins: f32,
|
||||
hop_size: f32,
|
||||
fft_size: f32,
|
||||
scroll_y: f32,
|
||||
note_height: f32,
|
||||
screen_size: vec2<f32>,
|
||||
min_note: f32,
|
||||
max_note: f32,
|
||||
gamma: f32,
|
||||
}
|
||||
|
||||
@group(0) @binding(0) var spec_tex: texture_2d<f32>;
|
||||
@group(0) @binding(1) var spec_sampler: sampler;
|
||||
@group(0) @binding(2) var<uniform> params: Params;
|
||||
|
||||
struct VertexOutput {
|
||||
@builtin(position) position: vec4<f32>,
|
||||
@location(0) uv: vec2<f32>,
|
||||
}
|
||||
|
||||
@vertex
|
||||
fn vs_main(@builtin(vertex_index) vi: u32) -> VertexOutput {
|
||||
var out: VertexOutput;
|
||||
let x = f32(i32(vi) / 2) * 4.0 - 1.0;
|
||||
let y = f32(i32(vi) % 2) * 4.0 - 1.0;
|
||||
out.position = vec4(x, y, 0.0, 1.0);
|
||||
out.uv = vec2((x + 1.0) * 0.5, (1.0 - y) * 0.5);
|
||||
return out;
|
||||
}
|
||||
|
||||
// Signed distance from point to rounded rectangle boundary
|
||||
fn rounded_rect_sdf(pos: vec2<f32>, rect_min: vec2<f32>, rect_max: vec2<f32>, r: f32) -> f32 {
|
||||
let center = (rect_min + rect_max) * 0.5;
|
||||
let half_size = (rect_max - rect_min) * 0.5;
|
||||
let q = abs(pos - center) - half_size + vec2(r);
|
||||
return length(max(q, vec2(0.0))) - r;
|
||||
}
|
||||
|
||||
// Colormap: black -> blue -> purple -> red -> orange -> yellow -> white
|
||||
fn colormap(v: f32, gamma: f32) -> vec4<f32> {
|
||||
let t = pow(clamp(v, 0.0, 1.0), gamma);
|
||||
|
||||
if t < 1.0 / 6.0 {
|
||||
// Black -> blue
|
||||
let s = t * 6.0;
|
||||
return vec4(0.0, 0.0, s, 1.0);
|
||||
} else if t < 2.0 / 6.0 {
|
||||
// Blue -> purple
|
||||
let s = (t - 1.0 / 6.0) * 6.0;
|
||||
return vec4(s * 0.6, 0.0, 1.0 - s * 0.2, 1.0);
|
||||
} else if t < 3.0 / 6.0 {
|
||||
// Purple -> red
|
||||
let s = (t - 2.0 / 6.0) * 6.0;
|
||||
return vec4(0.6 + s * 0.4, 0.0, 0.8 - s * 0.8, 1.0);
|
||||
} else if t < 4.0 / 6.0 {
|
||||
// Red -> orange
|
||||
let s = (t - 3.0 / 6.0) * 6.0;
|
||||
return vec4(1.0, s * 0.5, 0.0, 1.0);
|
||||
} else if t < 5.0 / 6.0 {
|
||||
// Orange -> yellow
|
||||
let s = (t - 4.0 / 6.0) * 6.0;
|
||||
return vec4(1.0, 0.5 + s * 0.5, 0.0, 1.0);
|
||||
} else {
|
||||
// Yellow -> white
|
||||
let s = (t - 5.0 / 6.0) * 6.0;
|
||||
return vec4(1.0, 1.0, s, 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
@fragment
|
||||
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
|
||||
let frag_x = in.position.x;
|
||||
let frag_y = in.position.y;
|
||||
|
||||
// Clip to view rectangle
|
||||
if frag_x < params.clip_rect.x || frag_x > params.clip_rect.z ||
|
||||
frag_y < params.clip_rect.y || frag_y > params.clip_rect.w {
|
||||
discard;
|
||||
}
|
||||
|
||||
// Compute the content rect in screen space
|
||||
let content_left = params.clip_rect.x + (params.clip_start_time - params.trim_start - params.viewport_start_time) * params.pixels_per_second;
|
||||
let content_right = content_left + params.audio_duration * params.pixels_per_second;
|
||||
let content_top = params.clip_rect.y - params.scroll_y;
|
||||
let content_bottom = params.clip_rect.y + (params.max_note - params.min_note + 1.0) * params.note_height - params.scroll_y;
|
||||
|
||||
// Rounded corners: content edges on X, visible viewport edges on Y.
|
||||
// This rounds left/right where the clip starts/ends, and top/bottom at the view boundary.
|
||||
let vis_top = max(content_top, params.clip_rect.y);
|
||||
let vis_bottom = min(content_bottom, params.clip_rect.w);
|
||||
|
||||
let corner_radius = 6.0;
|
||||
let dist = rounded_rect_sdf(
|
||||
vec2(frag_x, frag_y),
|
||||
vec2(content_left, vis_top),
|
||||
vec2(content_right, vis_bottom),
|
||||
corner_radius
|
||||
);
|
||||
if dist > 0.0 {
|
||||
discard;
|
||||
}
|
||||
|
||||
// Fragment X -> audio time -> time bin
|
||||
let timeline_time = params.viewport_start_time + (frag_x - params.clip_rect.x) / params.pixels_per_second;
|
||||
let audio_time = timeline_time - params.clip_start_time + params.trim_start;
|
||||
|
||||
if audio_time < 0.0 || audio_time > params.audio_duration {
|
||||
discard;
|
||||
}
|
||||
|
||||
let time_bin = audio_time * params.sample_rate / params.hop_size;
|
||||
if time_bin < 0.0 || time_bin >= params.time_bins {
|
||||
discard;
|
||||
}
|
||||
|
||||
// Fragment Y -> MIDI note -> frequency -> frequency bin
|
||||
let note = params.max_note - ((frag_y - params.clip_rect.y + params.scroll_y) / params.note_height);
|
||||
|
||||
if note < params.min_note || note > params.max_note {
|
||||
discard;
|
||||
}
|
||||
|
||||
// MIDI note -> frequency: freq = 440 * 2^((note - 69) / 12)
|
||||
let freq = 440.0 * pow(2.0, (note - 69.0) / 12.0);
|
||||
|
||||
// Frequency -> FFT bin index
|
||||
let freq_bin = freq * params.fft_size / params.sample_rate;
|
||||
|
||||
if freq_bin < 0.0 || freq_bin >= params.freq_bins {
|
||||
discard;
|
||||
}
|
||||
|
||||
// Sample texture with bilinear filtering
|
||||
let u = freq_bin / params.freq_bins;
|
||||
let v = time_bin / params.time_bins;
|
||||
let magnitude = textureSampleLevel(spec_tex, spec_sampler, vec2(u, v), 0.0).r;
|
||||
|
||||
return colormap(magnitude, params.gamma);
|
||||
}
|
||||
|
|
@ -505,7 +505,7 @@ impl TimelinePane {
|
|||
painter: &egui::Painter,
|
||||
clip_rect: egui::Rect,
|
||||
rect_min_x: f32, // Timeline panel left edge (for proper viewport-relative positioning)
|
||||
events: &[(f64, u8, bool)], // (timestamp, note_number, is_note_on)
|
||||
events: &[(f64, u8, u8, bool)], // (timestamp, note_number, velocity, is_note_on)
|
||||
trim_start: f64,
|
||||
visible_duration: f64,
|
||||
timeline_start: f64,
|
||||
|
|
@ -527,7 +527,7 @@ impl TimelinePane {
|
|||
let mut note_rectangles: Vec<(egui::Rect, u8)> = Vec::new();
|
||||
|
||||
// First pass: pair note-ons with note-offs to calculate durations
|
||||
for &(timestamp, note_number, is_note_on) in events {
|
||||
for &(timestamp, note_number, _velocity, is_note_on) in events {
|
||||
if is_note_on {
|
||||
// Store note-on timestamp
|
||||
active_notes.insert(note_number, timestamp);
|
||||
|
|
@ -892,7 +892,7 @@ impl TimelinePane {
|
|||
document: &lightningbeam_core::document::Document,
|
||||
active_layer_id: &Option<uuid::Uuid>,
|
||||
selection: &lightningbeam_core::selection::Selection,
|
||||
midi_event_cache: &std::collections::HashMap<u32, Vec<(f64, u8, bool)>>,
|
||||
midi_event_cache: &std::collections::HashMap<u32, Vec<(f64, u8, u8, bool)>>,
|
||||
raw_audio_cache: &std::collections::HashMap<usize, (Vec<f32>, u32, u32)>,
|
||||
waveform_gpu_dirty: &mut std::collections::HashSet<usize>,
|
||||
target_format: wgpu::TextureFormat,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,144 @@
|
|||
/// CPU-side FFT computation for spectrogram visualization.
|
||||
///
|
||||
/// Uses rayon to parallelize FFT across time slices on all CPU cores.
|
||||
/// Produces a 2D magnitude grid (time bins x frequency bins) for GPU texture upload.
|
||||
|
||||
use rayon::prelude::*;
|
||||
use std::f32::consts::PI;
|
||||
|
||||
/// Pre-computed spectrogram data ready for GPU upload
|
||||
pub struct SpectrogramData {
|
||||
/// Flattened 2D array of normalized magnitudes [time_bins * freq_bins], row-major
|
||||
/// Each value is 0.0 (silence) to 1.0 (peak), log-scale normalized
|
||||
pub magnitudes: Vec<f32>,
|
||||
pub time_bins: usize,
|
||||
pub freq_bins: usize,
|
||||
pub sample_rate: u32,
|
||||
pub hop_size: usize,
|
||||
pub fft_size: usize,
|
||||
pub duration: f64,
|
||||
}
|
||||
|
||||
/// Compute a spectrogram from raw audio samples using parallel FFT.
|
||||
///
|
||||
/// Each time slice is processed independently via rayon, making this
|
||||
/// scale well across all CPU cores.
|
||||
pub fn compute_spectrogram(
|
||||
samples: &[f32],
|
||||
sample_rate: u32,
|
||||
channels: u32,
|
||||
fft_size: usize,
|
||||
hop_size: usize,
|
||||
) -> SpectrogramData {
|
||||
// Mix to mono
|
||||
let mono: Vec<f32> = if channels >= 2 {
|
||||
samples
|
||||
.chunks(channels as usize)
|
||||
.map(|frame| frame.iter().sum::<f32>() / channels as f32)
|
||||
.collect()
|
||||
} else {
|
||||
samples.to_vec()
|
||||
};
|
||||
|
||||
let freq_bins = fft_size / 2 + 1;
|
||||
let duration = mono.len() as f64 / sample_rate as f64;
|
||||
|
||||
if mono.len() < fft_size {
|
||||
return SpectrogramData {
|
||||
magnitudes: Vec::new(),
|
||||
time_bins: 0,
|
||||
freq_bins,
|
||||
sample_rate,
|
||||
hop_size,
|
||||
fft_size,
|
||||
duration,
|
||||
};
|
||||
}
|
||||
|
||||
let time_bins = (mono.len().saturating_sub(fft_size)) / hop_size + 1;
|
||||
|
||||
// Precompute Hann window
|
||||
let window: Vec<f32> = (0..fft_size)
|
||||
.map(|i| 0.5 * (1.0 - (2.0 * PI * i as f32 / fft_size as f32).cos()))
|
||||
.collect();
|
||||
|
||||
// Precompute twiddle factors for Cooley-Tukey radix-2 FFT
|
||||
let twiddles: Vec<(f32, f32)> = (0..fft_size / 2)
|
||||
.map(|k| {
|
||||
let angle = -2.0 * PI * k as f32 / fft_size as f32;
|
||||
(angle.cos(), angle.sin())
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Bit-reversal permutation table
|
||||
let bits = (fft_size as f32).log2() as u32;
|
||||
let bit_rev: Vec<usize> = (0..fft_size)
|
||||
.map(|i| (i as u32).reverse_bits().wrapping_shr(32 - bits) as usize)
|
||||
.collect();
|
||||
|
||||
// Process all time slices in parallel
|
||||
let magnitudes: Vec<f32> = (0..time_bins)
|
||||
.into_par_iter()
|
||||
.flat_map(|t| {
|
||||
let offset = t * hop_size;
|
||||
let mut re = vec![0.0f32; fft_size];
|
||||
let mut im = vec![0.0f32; fft_size];
|
||||
|
||||
// Load windowed samples in bit-reversed order
|
||||
for i in 0..fft_size {
|
||||
let sample = if offset + i < mono.len() {
|
||||
mono[offset + i]
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
re[bit_rev[i]] = sample * window[i];
|
||||
}
|
||||
|
||||
// Cooley-Tukey radix-2 DIT FFT
|
||||
let mut half_size = 1;
|
||||
while half_size < fft_size {
|
||||
let step = half_size * 2;
|
||||
let twiddle_step = fft_size / step;
|
||||
|
||||
for k in (0..fft_size).step_by(step) {
|
||||
for j in 0..half_size {
|
||||
let tw_idx = j * twiddle_step;
|
||||
let (tw_re, tw_im) = twiddles[tw_idx];
|
||||
|
||||
let a = k + j;
|
||||
let b = a + half_size;
|
||||
|
||||
let t_re = tw_re * re[b] - tw_im * im[b];
|
||||
let t_im = tw_re * im[b] + tw_im * re[b];
|
||||
|
||||
re[b] = re[a] - t_re;
|
||||
im[b] = im[a] - t_im;
|
||||
re[a] += t_re;
|
||||
im[a] += t_im;
|
||||
}
|
||||
}
|
||||
half_size = step;
|
||||
}
|
||||
|
||||
// Extract magnitudes for positive frequencies
|
||||
let mut mags = Vec::with_capacity(freq_bins);
|
||||
for f in 0..freq_bins {
|
||||
let mag = (re[f] * re[f] + im[f] * im[f]).sqrt();
|
||||
// dB normalization: -80dB floor to 0dB ceiling → 0.0 to 1.0
|
||||
let db = 20.0 * (mag + 1e-10).log10();
|
||||
mags.push(((db + 80.0) / 80.0).clamp(0.0, 1.0));
|
||||
}
|
||||
mags
|
||||
})
|
||||
.collect();
|
||||
|
||||
SpectrogramData {
|
||||
magnitudes,
|
||||
time_bins,
|
||||
freq_bins,
|
||||
sample_rate,
|
||||
hop_size,
|
||||
fft_size,
|
||||
duration,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,350 @@
|
|||
/// GPU resources for spectrogram rendering.
|
||||
///
|
||||
/// Follows the same pattern as waveform_gpu.rs:
|
||||
/// - SpectrogramGpuResources stored in CallbackResources (long-lived)
|
||||
/// - SpectrogramCallback implements egui_wgpu::CallbackTrait (per-frame)
|
||||
/// - R32Float texture holds magnitude data (time bins × freq bins)
|
||||
/// - Fragment shader applies colormap and frequency mapping
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// GPU resources for all spectrograms (stored in egui_wgpu::CallbackResources)
|
||||
pub struct SpectrogramGpuResources {
|
||||
pub entries: HashMap<usize, SpectrogramGpuEntry>,
|
||||
render_pipeline: wgpu::RenderPipeline,
|
||||
render_bind_group_layout: wgpu::BindGroupLayout,
|
||||
sampler: wgpu::Sampler,
|
||||
}
|
||||
|
||||
/// Per-audio-pool GPU data for one spectrogram
|
||||
#[allow(dead_code)]
|
||||
pub struct SpectrogramGpuEntry {
|
||||
pub texture: wgpu::Texture,
|
||||
pub texture_view: wgpu::TextureView,
|
||||
pub render_bind_group: wgpu::BindGroup,
|
||||
pub uniform_buffer: wgpu::Buffer,
|
||||
pub time_bins: u32,
|
||||
pub freq_bins: u32,
|
||||
pub sample_rate: u32,
|
||||
pub hop_size: u32,
|
||||
pub fft_size: u32,
|
||||
pub duration: f32,
|
||||
}
|
||||
|
||||
/// Uniform buffer struct — must match spectrogram.wgsl Params exactly
|
||||
#[repr(C)]
|
||||
#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)]
|
||||
pub struct SpectrogramParams {
|
||||
pub clip_rect: [f32; 4], // 16 bytes @ offset 0
|
||||
pub viewport_start_time: f32, // 4 bytes @ offset 16
|
||||
pub pixels_per_second: f32, // 4 bytes @ offset 20
|
||||
pub audio_duration: f32, // 4 bytes @ offset 24
|
||||
pub sample_rate: f32, // 4 bytes @ offset 28
|
||||
pub clip_start_time: f32, // 4 bytes @ offset 32
|
||||
pub trim_start: f32, // 4 bytes @ offset 36
|
||||
pub time_bins: f32, // 4 bytes @ offset 40
|
||||
pub freq_bins: f32, // 4 bytes @ offset 44
|
||||
pub hop_size: f32, // 4 bytes @ offset 48
|
||||
pub fft_size: f32, // 4 bytes @ offset 52
|
||||
pub scroll_y: f32, // 4 bytes @ offset 56
|
||||
pub note_height: f32, // 4 bytes @ offset 60
|
||||
pub screen_size: [f32; 2], // 8 bytes @ offset 64
|
||||
pub min_note: f32, // 4 bytes @ offset 72
|
||||
pub max_note: f32, // 4 bytes @ offset 76
|
||||
pub gamma: f32, // 4 bytes @ offset 80
|
||||
pub _pad: [f32; 3], // 12 bytes @ offset 84 (pad to 96 for WGSL struct alignment)
|
||||
}
|
||||
// Total: 96 bytes (multiple of 16 for vec4 alignment)
|
||||
|
||||
/// Data for a pending spectrogram texture upload
|
||||
pub struct SpectrogramUpload {
|
||||
pub magnitudes: Vec<f32>,
|
||||
pub time_bins: u32,
|
||||
pub freq_bins: u32,
|
||||
pub sample_rate: u32,
|
||||
pub hop_size: u32,
|
||||
pub fft_size: u32,
|
||||
pub duration: f32,
|
||||
}
|
||||
|
||||
/// Per-frame callback for rendering one spectrogram instance
|
||||
pub struct SpectrogramCallback {
|
||||
pub pool_index: usize,
|
||||
pub params: SpectrogramParams,
|
||||
pub target_format: wgpu::TextureFormat,
|
||||
pub pending_upload: Option<SpectrogramUpload>,
|
||||
}
|
||||
|
||||
impl SpectrogramGpuResources {
|
||||
pub fn new(device: &wgpu::Device, target_format: wgpu::TextureFormat) -> Self {
|
||||
// Shader
|
||||
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: Some("spectrogram_render_shader"),
|
||||
source: wgpu::ShaderSource::Wgsl(
|
||||
include_str!("panes/shaders/spectrogram.wgsl").into(),
|
||||
),
|
||||
});
|
||||
|
||||
// Bind group layout: texture + sampler + uniforms
|
||||
let render_bind_group_layout =
|
||||
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
|
||||
label: Some("spectrogram_render_bgl"),
|
||||
entries: &[
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 0,
|
||||
visibility: wgpu::ShaderStages::FRAGMENT,
|
||||
ty: wgpu::BindingType::Texture {
|
||||
sample_type: wgpu::TextureSampleType::Float { filterable: true },
|
||||
view_dimension: wgpu::TextureViewDimension::D2,
|
||||
multisampled: false,
|
||||
},
|
||||
count: None,
|
||||
},
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 1,
|
||||
visibility: wgpu::ShaderStages::FRAGMENT,
|
||||
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
|
||||
count: None,
|
||||
},
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 2,
|
||||
visibility: wgpu::ShaderStages::FRAGMENT,
|
||||
ty: wgpu::BindingType::Buffer {
|
||||
ty: wgpu::BufferBindingType::Uniform,
|
||||
has_dynamic_offset: false,
|
||||
min_binding_size: None,
|
||||
},
|
||||
count: None,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// Render pipeline
|
||||
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("spectrogram_pipeline_layout"),
|
||||
bind_group_layouts: &[&render_bind_group_layout],
|
||||
push_constant_ranges: &[],
|
||||
});
|
||||
|
||||
let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
|
||||
label: Some("spectrogram_render_pipeline"),
|
||||
layout: Some(&pipeline_layout),
|
||||
vertex: wgpu::VertexState {
|
||||
module: &shader,
|
||||
entry_point: Some("vs_main"),
|
||||
buffers: &[],
|
||||
compilation_options: Default::default(),
|
||||
},
|
||||
fragment: Some(wgpu::FragmentState {
|
||||
module: &shader,
|
||||
entry_point: Some("fs_main"),
|
||||
targets: &[Some(wgpu::ColorTargetState {
|
||||
format: target_format,
|
||||
blend: Some(wgpu::BlendState::ALPHA_BLENDING),
|
||||
write_mask: wgpu::ColorWrites::ALL,
|
||||
})],
|
||||
compilation_options: Default::default(),
|
||||
}),
|
||||
primitive: wgpu::PrimitiveState {
|
||||
topology: wgpu::PrimitiveTopology::TriangleList,
|
||||
..Default::default()
|
||||
},
|
||||
depth_stencil: None,
|
||||
multisample: wgpu::MultisampleState::default(),
|
||||
multiview: None,
|
||||
cache: None,
|
||||
});
|
||||
|
||||
// Bilinear sampler for smooth frequency interpolation
|
||||
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
|
||||
label: Some("spectrogram_sampler"),
|
||||
mag_filter: wgpu::FilterMode::Linear,
|
||||
min_filter: wgpu::FilterMode::Linear,
|
||||
mipmap_filter: wgpu::FilterMode::Nearest,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
Self {
|
||||
entries: HashMap::new(),
|
||||
render_pipeline,
|
||||
render_bind_group_layout,
|
||||
sampler,
|
||||
}
|
||||
}
|
||||
|
||||
/// Upload pre-computed spectrogram magnitude data as a GPU texture
|
||||
pub fn upload_spectrogram(
|
||||
&mut self,
|
||||
device: &wgpu::Device,
|
||||
queue: &wgpu::Queue,
|
||||
pool_index: usize,
|
||||
upload: &SpectrogramUpload,
|
||||
) {
|
||||
// Remove old entry
|
||||
self.entries.remove(&pool_index);
|
||||
|
||||
if upload.time_bins == 0 || upload.freq_bins == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
// Data layout: magnitudes[t * freq_bins + f] — each row is one time slice
|
||||
// with freq_bins values. So texture width = freq_bins, height = time_bins.
|
||||
// R8Unorm is filterable (unlike R32Float) for bilinear interpolation.
|
||||
let tex_width = upload.freq_bins;
|
||||
let tex_height = upload.time_bins;
|
||||
|
||||
let texture = device.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some(&format!("spectrogram_{}", pool_index)),
|
||||
size: wgpu::Extent3d {
|
||||
width: tex_width,
|
||||
height: tex_height,
|
||||
depth_or_array_layers: 1,
|
||||
},
|
||||
mip_level_count: 1,
|
||||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: wgpu::TextureFormat::R8Unorm,
|
||||
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
||||
view_formats: &[],
|
||||
});
|
||||
|
||||
// Convert f32 magnitudes to u8 for R8Unorm, with row padding for alignment.
|
||||
// wgpu requires bytes_per_row to be a multiple of COPY_BYTES_PER_ROW_ALIGNMENT (256).
|
||||
let align = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
|
||||
let unpadded_row = tex_width; // 1 byte per texel for R8Unorm
|
||||
let padded_row = (unpadded_row + align - 1) / align * align;
|
||||
|
||||
let mut texel_data = vec![0u8; padded_row as usize * tex_height as usize];
|
||||
for row in 0..tex_height as usize {
|
||||
let src_offset = row * tex_width as usize;
|
||||
let dst_offset = row * padded_row as usize;
|
||||
for col in 0..tex_width as usize {
|
||||
let m = upload.magnitudes[src_offset + col];
|
||||
texel_data[dst_offset + col] = (m.clamp(0.0, 1.0) * 255.0) as u8;
|
||||
}
|
||||
}
|
||||
|
||||
// Upload magnitude data
|
||||
queue.write_texture(
|
||||
wgpu::TexelCopyTextureInfo {
|
||||
texture: &texture,
|
||||
mip_level: 0,
|
||||
origin: wgpu::Origin3d::ZERO,
|
||||
aspect: wgpu::TextureAspect::All,
|
||||
},
|
||||
&texel_data,
|
||||
wgpu::TexelCopyBufferLayout {
|
||||
offset: 0,
|
||||
bytes_per_row: Some(padded_row),
|
||||
rows_per_image: Some(tex_height),
|
||||
},
|
||||
wgpu::Extent3d {
|
||||
width: tex_width,
|
||||
height: tex_height,
|
||||
depth_or_array_layers: 1,
|
||||
},
|
||||
);
|
||||
|
||||
let texture_view = texture.create_view(&wgpu::TextureViewDescriptor {
|
||||
label: Some(&format!("spectrogram_{}_view", pool_index)),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let uniform_buffer = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some(&format!("spectrogram_{}_uniforms", pool_index)),
|
||||
size: std::mem::size_of::<SpectrogramParams>() as u64,
|
||||
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
|
||||
let render_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some(&format!("spectrogram_{}_bg", pool_index)),
|
||||
layout: &self.render_bind_group_layout,
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 0,
|
||||
resource: wgpu::BindingResource::TextureView(&texture_view),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 1,
|
||||
resource: wgpu::BindingResource::Sampler(&self.sampler),
|
||||
},
|
||||
wgpu::BindGroupEntry {
|
||||
binding: 2,
|
||||
resource: uniform_buffer.as_entire_binding(),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
self.entries.insert(
|
||||
pool_index,
|
||||
SpectrogramGpuEntry {
|
||||
texture,
|
||||
texture_view,
|
||||
render_bind_group,
|
||||
uniform_buffer,
|
||||
time_bins: upload.time_bins,
|
||||
freq_bins: upload.freq_bins,
|
||||
sample_rate: upload.sample_rate,
|
||||
hop_size: upload.hop_size,
|
||||
fft_size: upload.fft_size,
|
||||
duration: upload.duration,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl egui_wgpu::CallbackTrait for SpectrogramCallback {
|
||||
fn prepare(
|
||||
&self,
|
||||
device: &wgpu::Device,
|
||||
queue: &wgpu::Queue,
|
||||
_screen_descriptor: &egui_wgpu::ScreenDescriptor,
|
||||
_egui_encoder: &mut wgpu::CommandEncoder,
|
||||
resources: &mut egui_wgpu::CallbackResources,
|
||||
) -> Vec<wgpu::CommandBuffer> {
|
||||
// Initialize global resources on first use
|
||||
if !resources.contains::<SpectrogramGpuResources>() {
|
||||
resources.insert(SpectrogramGpuResources::new(device, self.target_format));
|
||||
}
|
||||
|
||||
let gpu: &mut SpectrogramGpuResources = resources.get_mut().unwrap();
|
||||
|
||||
// Handle pending upload
|
||||
if let Some(ref upload) = self.pending_upload {
|
||||
gpu.upload_spectrogram(device, queue, self.pool_index, upload);
|
||||
}
|
||||
|
||||
// Update uniform buffer
|
||||
if let Some(entry) = gpu.entries.get(&self.pool_index) {
|
||||
queue.write_buffer(
|
||||
&entry.uniform_buffer,
|
||||
0,
|
||||
bytemuck::cast_slice(&[self.params]),
|
||||
);
|
||||
}
|
||||
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn paint(
|
||||
&self,
|
||||
_info: eframe::egui::PaintCallbackInfo,
|
||||
render_pass: &mut wgpu::RenderPass<'static>,
|
||||
resources: &egui_wgpu::CallbackResources,
|
||||
) {
|
||||
let gpu: &SpectrogramGpuResources = match resources.get() {
|
||||
Some(r) => r,
|
||||
None => return,
|
||||
};
|
||||
|
||||
let entry = match gpu.entries.get(&self.pool_index) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
};
|
||||
|
||||
render_pass.set_pipeline(&gpu.render_pipeline);
|
||||
render_pass.set_bind_group(0, &entry.render_bind_group, &[]);
|
||||
render_pass.draw(0..3, 0..1); // Fullscreen triangle
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue