From 053a77cfa1b3563d8c2fda3481da3a546cd0f81d Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 10 Jul 2026 09:35:48 -0400 Subject: [PATCH 1/8] Timeline types stage 1: make lightningbeam-core Beats/Seconds-typed Type ClipInstance.timeline_start/timeline_duration/loop_before as Beats and thread Beats/Seconds through core so the compiler catches the seconds-vs-beats mismatches behind the audio-clip placement/drag/trim bugs. Fixes latent mixups surfaced by the types: - add/remove/split clip instance: the audio "effective duration" fallback was the content-seconds span treated as beats (clips stopped early off 60 BPM); now converted via the tempo map at the clip's start. - trim validation: extend-left/right clamped a content-seconds delta against a timeline-beats gap (and moved trim_start + timeline_start by the same raw amount, assuming 1:1). Now the gap is converted to content seconds and the timeline moves by the beats-equivalent. - split content-split point mixed beats into a seconds trim value. - set_keyframe: visibility-start fallback returned beats where seconds expected. - hit_test: timeline_time (s) compared against beats without conversion. Typed backend boundaries that were passing beats/seconds as bare f64: add_audio_clip (start/dur Beats, offset Seconds), move_clip/extend_clip (Beats), set_offset (Seconds). Command enum transport stays f64. Serialization is unchanged (Beats/Seconds are #[serde(transparent)]). lightningbeam-core builds; 299 core tests pass. The editor call sites (recording/drop/paste/drag placement) are updated in stage 2. --- daw-backend/src/audio/engine.rs | 30 ++--- daw-backend/src/command/types.rs | 3 +- .../src/actions/add_clip_instance.rs | 26 ++-- .../src/actions/add_effect.rs | 10 +- .../src/actions/loop_clip_instances.rs | 18 ++- .../src/actions/move_clip_instances.rs | 34 ++--- .../src/actions/remove_clip_instances.rs | 23 ++-- .../src/actions/remove_effect.rs | 10 +- .../src/actions/set_keyframe.rs | 4 +- .../src/actions/split_clip_instance.rs | 65 +++++---- .../src/actions/trim_clip_instances.rs | 71 ++++++---- .../lightningbeam-core/src/clip.rs | 84 ++++++------ .../lightningbeam-core/src/document.rs | 127 +++++++++--------- .../lightningbeam-core/src/effect.rs | 2 +- .../lightningbeam-core/src/effect_layer.rs | 14 +- .../lightningbeam-core/src/hit_test.rs | 12 +- .../lightningbeam-core/src/renderer.rs | 29 ++-- 17 files changed, 313 insertions(+), 249 deletions(-) diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index ac6a6c0..a7dcab5 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -1001,20 +1001,20 @@ impl Engine { let _ = self.event_tx.push(AudioEvent::AudioFileAdded(pool_index, path)); } Command::AddAudioClip(track_id, clip_id, pool_index, start_time, duration, offset) => { - // Create a new clip instance with the pre-assigned clip_id - // start_time and duration are in beats; offset (internal_start) is seconds - let start_beats = Beats(start_time); - let end_beats = Beats(start_time + duration); + // Create a new clip instance with the pre-assigned clip_id. + // start_time/duration are beats; offset (internal_start) is seconds. + let start_beats = start_time; + let end_beats = start_time + duration; let start_secs = self.tempo_map.beats_to_seconds(start_beats); let end_secs = self.tempo_map.beats_to_seconds(end_beats); let content_dur_secs = (end_secs - start_secs).seconds_to_f64(); let mut clip = AudioClipInstance::new( clip_id, pool_index, - Seconds(offset), - Seconds(offset + content_dur_secs), + offset, + offset + Seconds(content_dur_secs), start_beats, - Beats(duration), + duration, ); // If the source is streamed (a compressed audio file, or a video's @@ -3380,8 +3380,8 @@ impl EngineController { } /// Move a clip to a new timeline position (changes external_start) - pub fn move_clip(&mut self, track_id: TrackId, clip_id: ClipId, new_start_time: f64) { - let _ = self.command_tx.push(Command::MoveClip(track_id, clip_id, new_start_time)); + pub fn move_clip(&mut self, track_id: TrackId, clip_id: ClipId, new_start_time: Beats) { + let _ = self.command_tx.push(Command::MoveClip(track_id, clip_id, new_start_time.beats_to_f64())); } /// Trim a clip's internal boundaries (changes which portion of source content is used) @@ -3391,8 +3391,8 @@ impl EngineController { } /// Extend or shrink a clip's external duration (enables looping if > internal duration) - pub fn extend_clip(&mut self, track_id: TrackId, clip_id: ClipId, new_external_duration: f64) { - let _ = self.command_tx.push(Command::ExtendClip(track_id, clip_id, new_external_duration)); + pub fn extend_clip(&mut self, track_id: TrackId, clip_id: ClipId, new_external_duration: Beats) { + let _ = self.command_tx.push(Command::ExtendClip(track_id, clip_id, new_external_duration.beats_to_f64())); } /// Send a generic command to the audio thread @@ -3440,8 +3440,8 @@ impl EngineController { /// Set metatrack time offset in seconds /// Positive = shift content later, negative = shift earlier - pub fn set_offset(&mut self, track_id: TrackId, offset: f64) { - let _ = self.command_tx.push(Command::SetOffset(track_id, offset)); + pub fn set_offset(&mut self, track_id: TrackId, offset: Seconds) { + let _ = self.command_tx.push(Command::SetOffset(track_id, offset.seconds_to_f64())); } /// Set metatrack pitch shift in semitones (for future use) @@ -3525,14 +3525,14 @@ impl EngineController { /// Add a clip to an audio track (async, fire-and-forget) /// Returns the pre-assigned clip instance ID so callers can track the clip without a sync round-trip - pub fn add_audio_clip(&mut self, track_id: TrackId, pool_index: usize, start_time: f64, duration: f64, offset: f64) -> AudioClipInstanceId { + pub fn add_audio_clip(&mut self, track_id: TrackId, pool_index: usize, start_time: Beats, duration: Beats, offset: Seconds) -> AudioClipInstanceId { let clip_id = self.next_audio_clip_id.fetch_add(1, Ordering::Relaxed); let _ = self.command_tx.push(Command::AddAudioClip(track_id, clip_id, pool_index, start_time, duration, offset)); clip_id } /// Add a clip to an audio track with a pre-assigned ID (for undo/redo, restoring deleted clips) - pub fn add_audio_clip_with_id(&mut self, track_id: TrackId, clip_id: AudioClipInstanceId, pool_index: usize, start_time: f64, duration: f64, offset: f64) { + pub fn add_audio_clip_with_id(&mut self, track_id: TrackId, clip_id: AudioClipInstanceId, pool_index: usize, start_time: Beats, duration: Beats, offset: Seconds) { let _ = self.command_tx.push(Command::AddAudioClip(track_id, clip_id, pool_index, start_time, duration, offset)); } diff --git a/daw-backend/src/command/types.rs b/daw-backend/src/command/types.rs index cf29ccc..98cce70 100644 --- a/daw-backend/src/command/types.rs +++ b/daw-backend/src/command/types.rs @@ -71,7 +71,8 @@ pub enum Command { AddAudioFile(String, Vec, u32, u32), /// Add a clip to an audio track (track_id, clip_id, pool_index, start_time, duration, offset) /// The clip_id is pre-assigned by the caller (via EngineController::next_audio_clip_id()) - AddAudioClip(TrackId, AudioClipInstanceId, usize, f64, f64, f64), + /// (track, clip_id, pool_index, start_time [beats], duration [beats], offset [seconds]) + AddAudioClip(TrackId, AudioClipInstanceId, usize, Beats, Beats, Seconds), // MIDI commands /// Create a new MIDI track with a name and optional parent group diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs index 326550a..705c9d5 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs @@ -60,13 +60,14 @@ impl AddClipInstanceAction { impl Action for AddClipInstanceAction { fn execute(&mut self, document: &mut Document) -> Result<(), String> { - // Calculate the clip's effective duration + // Calculate the clip's effective duration in BEATS for overlap testing. + // `get_clip_duration` is the content length in seconds; the placement span + // must be beats (the timeline is beats-domain), so convert via the clip's + // typed helper rather than treating the seconds span as beats. let clip_duration = document.get_clip_duration(&self.clip_instance.clip_id) .ok_or_else(|| format!("Clip {} not found", self.clip_instance.clip_id))?; - - let trim_start = self.clip_instance.trim_start; - let trim_end = self.clip_instance.trim_end.unwrap_or(clip_duration); - let effective_duration = trim_end - trim_start; + let effective_duration = self.clip_instance + .effective_duration_beats(clip_duration, document.tempo_map()); // Auto-adjust position for audio/video layers to avoid overlaps let adjusted_start = document.find_nearest_valid_position( @@ -200,9 +201,10 @@ impl Action for AddClipInstanceAction { let internal_end = self.clip_instance.trim_end.unwrap_or(clip.duration); let external_start = self.clip_instance.timeline_start; - // Calculate external duration (for looping if timeline_duration is set) + // Calculate external duration (for looping if timeline_duration is set). + // MIDI trims are beats-domain, so the fallback span is beats too. let external_duration = self.clip_instance.timeline_duration - .unwrap_or(internal_end - internal_start); + .unwrap_or(daw_backend::Beats(internal_end - internal_start)); // Create MidiClipInstance let instance = daw_backend::MidiClipInstance::new( @@ -210,8 +212,8 @@ impl Action for AddClipInstanceAction { *midi_clip_id, daw_backend::Beats(internal_start), daw_backend::Beats(internal_end), - daw_backend::Beats(external_start), - daw_backend::Beats(external_duration), + external_start, + external_duration, ); // Send query to add instance and get instance ID @@ -247,8 +249,8 @@ impl Action for AddClipInstanceAction { // the seconds-as-beats bug that made clips stop early off 60 BPM). let effective_duration = self.clip_instance.timeline_duration.unwrap_or_else(|| { let tempo_map = document.tempo_map(); - let content_secs = internal_end - internal_start; - tempo_map.inverse_transform(tempo_map.transform(start_time) + content_secs) + let content_secs = daw_backend::Seconds(internal_end - internal_start); + tempo_map.seconds_to_beats(tempo_map.beats_to_seconds(start_time) + content_secs) - start_time }); @@ -257,7 +259,7 @@ impl Action for AddClipInstanceAction { *audio_pool_index, start_time, effective_duration, - internal_start, + daw_backend::Seconds(internal_start), ); self.backend_track_id = Some(*backend_track_id); diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/add_effect.rs b/lightningbeam-ui/lightningbeam-core/src/actions/add_effect.rs index 9df7fca..cf89b9c 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/add_effect.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/add_effect.rs @@ -159,7 +159,7 @@ mod tests { fn test_add_effect() { let (mut document, layer_id, def) = create_test_setup(); - let instance = def.create_instance(0.0, 10.0); + let instance = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let instance_id = instance.id; let mut action = AddEffectAction::new(layer_id, instance); @@ -181,7 +181,7 @@ mod tests { fn test_add_effect_rollback() { let (mut document, layer_id, def) = create_test_setup(); - let instance = def.create_instance(0.0, 10.0); + let instance = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let mut action = AddEffectAction::new(layer_id, instance); action.execute(&mut document).unwrap(); @@ -201,19 +201,19 @@ mod tests { let (mut document, layer_id, def) = create_test_setup(); // Add first effect - let instance1 = def.create_instance(0.0, 10.0); + let instance1 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id1 = instance1.id; let mut action1 = AddEffectAction::new(layer_id, instance1); action1.execute(&mut document).unwrap(); // Add second effect - let instance2 = def.create_instance(0.0, 10.0); + let instance2 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id2 = instance2.id; let mut action2 = AddEffectAction::new(layer_id, instance2); action2.execute(&mut document).unwrap(); // Insert third effect at index 1 (between first and second) - let instance3 = def.create_instance(0.0, 10.0); + let instance3 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id3 = instance3.id; let mut action3 = AddEffectAction::at_index(layer_id, instance3, 1); action3.execute(&mut document).unwrap(); diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs index 01fd71e..b275b22 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs @@ -9,8 +9,9 @@ use crate::layer::AnyLayer; use std::collections::HashMap; use uuid::Uuid; -/// Per-instance loop change: (instance_id, old_timeline_duration, new_timeline_duration, old_loop_before, new_loop_before) -pub type LoopEntry = (Uuid, Option, Option, Option, Option); +/// Per-instance loop change: (instance_id, old_timeline_duration, new_timeline_duration, old_loop_before, new_loop_before). +/// All durations/offsets are in beats. +pub type LoopEntry = (Uuid, Option, Option, Option, Option); /// Action that changes the loop duration of clip instances pub struct LoopClipInstancesAction { @@ -129,10 +130,17 @@ impl LoopClipInstancesAction { let content_window = { let trim_end = instance.trim_end.unwrap_or(clip.duration); - (trim_end - instance.trim_start).max(0.0) + (trim_end - instance.trim_start).max(0.0) // seconds }; - let right_duration = target_duration.unwrap_or(content_window); - let left_duration = target_loop_before.unwrap_or(0.0); + // Natural content length as a beats span at the clip's start (the + // fallback when no explicit timeline_duration is set). + let tempo_map = document.tempo_map(); + let content_window_beats = tempo_map.seconds_to_beats( + tempo_map.beats_to_seconds(instance.timeline_start) + + daw_backend::Seconds(content_window), + ) - instance.timeline_start; + let right_duration = target_duration.unwrap_or(content_window_beats); + let left_duration = target_loop_before.unwrap_or(daw_backend::Beats::ZERO); let external_duration = left_duration + right_duration; let external_start = instance.timeline_start - left_duration; diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs index dd70075..6664d3d 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs @@ -6,13 +6,15 @@ use crate::action::Action; use crate::clip::ClipInstance; use crate::document::Document; use crate::layer::AnyLayer; +use daw_backend::Beats; use std::collections::HashMap; use uuid::Uuid; /// Action that moves clip instances to new timeline positions pub struct MoveClipInstancesAction { - /// Map of layer IDs to vectors of (clip_instance_id, old_timeline_start, new_timeline_start) - layer_moves: HashMap>, + /// Map of layer IDs to vectors of (clip_instance_id, old_timeline_start, new_timeline_start). + /// Timeline positions are in beats. + layer_moves: HashMap>, } impl MoveClipInstancesAction { @@ -20,8 +22,8 @@ impl MoveClipInstancesAction { /// /// # Arguments /// - /// * `layer_moves` - Map of layer IDs to vectors of (clip_instance_id, old_timeline_start, new_timeline_start) - pub fn new(layer_moves: HashMap>) -> Self { + /// * `layer_moves` - Map of layer IDs to vectors of (clip_instance_id, old_timeline_start, new_timeline_start) in beats + pub fn new(layer_moves: HashMap>) -> Self { Self { layer_moves } } } @@ -42,7 +44,7 @@ impl Action for MoveClipInstancesAction { // Check if this instance is in a group if let Some(group) = document.find_group_for_instance(instance_id) { - let offset = new_start - old_start; + let offset = *new_start - *old_start; // Add all group members to the move list for (member_layer_id, member_instance_id) in group.get_members() { @@ -77,7 +79,7 @@ impl Action for MoveClipInstancesAction { } // Auto-adjust moves to avoid overlaps - let mut adjusted_moves: HashMap> = HashMap::new(); + let mut adjusted_moves: HashMap> = HashMap::new(); for (layer_id, moves) in &expanded_moves { let layer = document.get_layer(layer_id) @@ -101,10 +103,10 @@ impl Action for MoveClipInstancesAction { AnyLayer::Text(_) => &[], }; - let group: Vec<(Uuid, f64, f64)> = moves.iter().filter_map(|(id, old_start, _)| { + let group: Vec<(Uuid, Beats, Beats)> = moves.iter().filter_map(|(id, old_start, _)| { let inst = clip_instances.iter().find(|ci| &ci.id == id)?; let dur = document.get_clip_duration(&inst.clip_id)?; - let eff = inst.trim_end.unwrap_or(dur) - inst.trim_start; + let eff = inst.effective_duration_beats(dur, document.tempo_map()); Some((*id, *old_start, eff)) }).collect(); @@ -112,7 +114,7 @@ impl Action for MoveClipInstancesAction { let clamped = document.clamp_group_move_offset(layer_id, &group, desired_offset); for (instance_id, old_start, _) in moves { - adjusted_layer_moves.push((*instance_id, *old_start, (*old_start + clamped).max(0.0))); + adjusted_layer_moves.push((*instance_id, *old_start, (*old_start + clamped).max(Beats::ZERO))); } adjusted_moves.insert(*layer_id, adjusted_layer_moves); @@ -208,7 +210,7 @@ impl Action for MoveClipInstancesAction { if let Some(instance) = vl.clip_instances.iter().find(|ci| ci.id == *instance_id) { // Check if this clip has a metatrack if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { - controller.set_offset(metatrack_id, *new_start); + controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(*new_start)); controller.set_trim_start(metatrack_id, instance.trim_start); controller.set_trim_end(metatrack_id, instance.trim_end); } @@ -292,7 +294,7 @@ impl Action for MoveClipInstancesAction { for (instance_id, old_start, _new_start) in moves { if let Some(instance) = vl.clip_instances.iter().find(|ci| ci.id == *instance_id) { if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { - controller.set_offset(metatrack_id, *old_start); + controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(*old_start)); controller.set_trim_start(metatrack_id, instance.trim_start); controller.set_trim_end(metatrack_id, instance.trim_end); } @@ -373,15 +375,15 @@ mod tests { let mut vector_layer = VectorLayer::new("Layer 1"); let mut clip_instance = ClipInstance::new(clip_id); - clip_instance.timeline_start = 1.0; // Start at 1 second + clip_instance.timeline_start = Beats(1.0); // Start at beat 1 let instance_id = clip_instance.id; vector_layer.clip_instances.push(clip_instance); let layer_id = document.root.add_child(AnyLayer::Vector(vector_layer)); - // Create move action: move from 1.0 to 5.0 seconds + // Create move action: move from beat 1 to beat 5 let mut layer_moves = HashMap::new(); - layer_moves.insert(layer_id, vec![(instance_id, 1.0, 5.0)]); + layer_moves.insert(layer_id, vec![(instance_id, Beats(1.0), Beats(5.0))]); let mut action = MoveClipInstancesAction::new(layer_moves); @@ -395,7 +397,7 @@ mod tests { .iter() .find(|ci| ci.id == instance_id) .unwrap(); - assert_eq!(instance.timeline_start, 5.0); + assert_eq!(instance.timeline_start, Beats(5.0)); } // Rollback @@ -408,7 +410,7 @@ mod tests { .iter() .find(|ci| ci.id == instance_id) .unwrap(); - assert_eq!(instance.timeline_start, 1.0); + assert_eq!(instance.timeline_start, Beats(1.0)); } } } diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs index a043c7b..76af639 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs @@ -172,17 +172,18 @@ impl Action for RemoveClipInstancesAction { let internal_start = instance.trim_start; let internal_end = instance.trim_end.unwrap_or(clip.duration); let external_start = instance.timeline_start; + // MIDI trims are beats-domain, so the fallback span is beats too. let external_duration = instance .timeline_duration - .unwrap_or(internal_end - internal_start); + .unwrap_or(daw_backend::Beats(internal_end - internal_start)); let midi_instance = daw_backend::MidiClipInstance::new( 0, *midi_clip_id, daw_backend::Beats(internal_start), daw_backend::Beats(internal_end), - daw_backend::Beats(external_start), - daw_backend::Beats(external_duration), + external_start, + external_duration, ); let query = Query::AddMidiClipInstanceSync(track_id, midi_instance); @@ -198,16 +199,22 @@ impl Action for RemoveClipInstancesAction { AudioClipType::Sampled { audio_pool_index } => { let internal_start = instance.trim_start; let internal_end = instance.trim_end.unwrap_or(clip.duration); - let effective_duration = instance.timeline_duration - .unwrap_or(internal_end - internal_start); let start_time = instance.timeline_start; + // Fallback span is the content seconds converted to beats at the + // clip's start (not the seconds span treated as beats). + let effective_duration = instance.timeline_duration.unwrap_or_else(|| { + let tempo_map = document.tempo_map(); + let content_secs = daw_backend::Seconds(internal_end - internal_start); + tempo_map.seconds_to_beats(tempo_map.beats_to_seconds(start_time) + content_secs) + - start_time + }); let new_id = controller.add_audio_clip( track_id, *audio_pool_index, start_time, effective_duration, - internal_start, + daw_backend::Seconds(internal_start), ); backend.clip_instance_to_backend_map.insert( instance.id, @@ -238,11 +245,11 @@ mod tests { let mut vector_layer = VectorLayer::new("Layer 1"); let mut ci1 = ClipInstance::new(clip_id); - ci1.timeline_start = 0.0; + ci1.timeline_start = daw_backend::Beats::ZERO; let id1 = ci1.id; let mut ci2 = ClipInstance::new(clip_id); - ci2.timeline_start = 5.0; + ci2.timeline_start = daw_backend::Beats(5.0); let id2 = ci2.id; vector_layer.clip_instances.push(ci1); diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/remove_effect.rs b/lightningbeam-ui/lightningbeam-core/src/actions/remove_effect.rs index 4aeb5fb..7b83df5 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/remove_effect.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/remove_effect.rs @@ -134,7 +134,7 @@ mod tests { let (mut document, layer_id, def) = create_test_setup(); // Add an effect first - let instance = def.create_instance(0.0, 10.0); + let instance = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let instance_id = instance.id; if let Some(AnyLayer::Effect(el)) = document.get_layer_mut(&layer_id) { @@ -161,7 +161,7 @@ mod tests { let (mut document, layer_id, def) = create_test_setup(); // Add an effect first - let instance = def.create_instance(0.0, 10.0); + let instance = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let instance_id = instance.id; if let Some(AnyLayer::Effect(el)) = document.get_layer_mut(&layer_id) { @@ -185,11 +185,11 @@ mod tests { let (mut document, layer_id, def) = create_test_setup(); // Add three effects - let instance1 = def.create_instance(0.0, 10.0); + let instance1 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id1 = instance1.id; - let instance2 = def.create_instance(0.0, 10.0); + let instance2 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id2 = instance2.id; - let instance3 = def.create_instance(0.0, 10.0); + let instance3 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id3 = instance3.id; if let Some(AnyLayer::Effect(el)) = document.get_layer_mut(&layer_id) { diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/set_keyframe.rs b/lightningbeam-ui/lightningbeam-core/src/actions/set_keyframe.rs index 8f10b6f..961e304 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/set_keyframe.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/set_keyframe.rs @@ -81,9 +81,11 @@ impl Action for SetKeyframeAction { if let Some(AnyLayer::Vector(vl)) = document.get_layer(&self.layer_id) { for clip_id in &self.clip_instance_ids { if let Some(ci) = vl.clip_instances.iter().find(|c| c.id == *clip_id) { + // `start` is a keyframe time in seconds; group_visibility_start returns seconds, + // so the fallback must convert the clip's beats start to seconds too. let start = vl .group_visibility_start(clip_id, self.time) - .unwrap_or(ci.timeline_start); + .unwrap_or_else(|| document.tempo_map().beats_to_seconds(ci.timeline_start).seconds_to_f64()); clip_info.insert(*clip_id, (ci.transform.clone(), ci.opacity, start)); } } diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs b/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs index 5e2101d..fb3095e 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs @@ -17,8 +17,8 @@ pub struct SplitClipInstanceAction { /// The clip instance to split instance_id: Uuid, - /// Timeline time where to split (in seconds) - split_time: f64, + /// Timeline position where to split (in beats) + split_time: daw_backend::Beats, /// Whether the action has been executed (for rollback) executed: bool, @@ -26,8 +26,8 @@ pub struct SplitClipInstanceAction { // Stored during execute for rollback /// Original trim_end value of the left (original) instance original_trim_end: Option, - /// Original timeline_duration value of the left (original) instance - original_timeline_duration: Option, + /// Original timeline_duration value of the left (original) instance (beats) + original_timeline_duration: Option, /// ID of the new (right) instance created by the split new_instance_id: Option, @@ -47,8 +47,8 @@ impl SplitClipInstanceAction { /// /// * `layer_id` - The ID of the layer containing the clip instance /// * `instance_id` - The ID of the clip instance to split - /// * `split_time` - The timeline time (in seconds) where to split - pub fn new(layer_id: Uuid, instance_id: Uuid, split_time: f64) -> Self { + /// * `split_time` - The timeline position (in beats) where to split + pub fn new(layer_id: Uuid, instance_id: Uuid, split_time: daw_backend::Beats) -> Self { Self { layer_id, instance_id, @@ -72,9 +72,9 @@ impl SplitClipInstanceAction { /// /// * `layer_id` - The ID of the layer containing the clip instance /// * `instance_id` - The ID of the clip instance to split - /// * `split_time` - The timeline time (in seconds) where to split + /// * `split_time` - The timeline position (in beats) where to split /// * `new_instance_id` - The UUID to use for the new (right) clip instance - pub fn with_new_instance_id(layer_id: Uuid, instance_id: Uuid, split_time: f64, new_instance_id: Uuid) -> Self { + pub fn with_new_instance_id(layer_id: Uuid, instance_id: Uuid, split_time: daw_backend::Beats, new_instance_id: Uuid) -> Self { Self { layer_id, instance_id, @@ -132,9 +132,9 @@ impl Action for SplitClipInstanceAction { let timeline_end = instance.timeline_start + effective_duration; // Validate: split_time must be strictly within the clip's timeline span - const EPSILON: f64 = 0.001; // 1ms tolerance - if self.split_time <= instance.timeline_start + EPSILON - || self.split_time >= timeline_end - EPSILON + let epsilon = daw_backend::Beats(0.001); // ~1ms tolerance + if self.split_time <= instance.timeline_start + epsilon + || self.split_time >= timeline_end - epsilon { return Err(format!( "Split time {} must be within clip bounds ({} to {})", @@ -146,21 +146,27 @@ impl Action for SplitClipInstanceAction { self.original_trim_end = instance.trim_end; self.original_timeline_duration = instance.timeline_duration; - // Check if this is a looping clip + // Check if this is a looping clip. `content_duration` is a trim-domain + // span (seconds), so `clip_duration` must be unwrapped as seconds. let is_looping = instance.timeline_duration.is_some(); - let content_duration = instance.trim_end.unwrap_or(clip_duration) - instance.trim_start; + let content_duration = instance.trim_end.unwrap_or(clip_duration.seconds_to_f64()) - instance.trim_start; - // Calculate the split point + // Timeline split point (beats). let time_into_clip = self.split_time - instance.timeline_start; let left_duration = time_into_clip; let right_duration = effective_duration - left_duration; - // Calculate content split time + // How far the split lands into the clip's *content* (seconds, trim domain). + let tempo_map = document.tempo_map(); + let time_into_clip_secs = (tempo_map.beats_to_seconds(self.split_time) + - tempo_map.beats_to_seconds(instance.timeline_start)).seconds_to_f64(); + + // Calculate content split time (seconds) let content_split_time = if is_looping { // For looping clips, wrap around content - instance.trim_start + (time_into_clip % content_duration) + instance.trim_start + (time_into_clip_secs % content_duration) } else { - instance.trim_start + time_into_clip + instance.trim_start + time_into_clip_secs }; // Clone the instance for the right side @@ -384,17 +390,18 @@ impl Action for SplitClipInstanceAction { let internal_start = new_instance.trim_start; let internal_end = new_instance.trim_end.unwrap_or(clip.duration); let external_start = new_instance.timeline_start; + // MIDI trims are beats-domain, so the fallback span is beats too. let external_duration = new_instance .timeline_duration - .unwrap_or(internal_end - internal_start); + .unwrap_or(daw_backend::Beats(internal_end - internal_start)); let instance = daw_backend::MidiClipInstance::new( 0, *midi_clip_id, daw_backend::Beats(internal_start), daw_backend::Beats(internal_end), - daw_backend::Beats(external_start), - daw_backend::Beats(external_duration), + external_start, + external_duration, ); let query = Query::AddMidiClipInstanceSync(*backend_track_id, instance); @@ -430,16 +437,22 @@ impl Action for SplitClipInstanceAction { // 2. Add the new (right) instance let internal_start = new_instance.trim_start; let internal_end = new_instance.trim_end.unwrap_or(clip.duration); - let effective_duration = new_instance.timeline_duration - .unwrap_or(internal_end - internal_start); let start_time = new_instance.timeline_start; + // Fallback span is the content seconds converted to beats at the + // clip's start (not the seconds span treated as beats). + let effective_duration = new_instance.timeline_duration.unwrap_or_else(|| { + let tempo_map = document.tempo_map(); + let content_secs = daw_backend::Seconds(internal_end - internal_start); + tempo_map.seconds_to_beats(tempo_map.beats_to_seconds(start_time) + content_secs) + - start_time + }); let instance_id = controller.add_audio_clip( *backend_track_id, *audio_pool_index, start_time, effective_duration, - internal_start, + daw_backend::Seconds(internal_start), ); self.backend_track_id = Some(*backend_track_id); @@ -540,7 +553,7 @@ mod tests { // Create a clip instance at timeline 0, with trim 0-10 (10 seconds) let mut clip_instance = ClipInstance::new(clip_id); - clip_instance.timeline_start = 0.0; + clip_instance.timeline_start = daw_backend::Beats::ZERO; clip_instance.trim_start = 0.0; clip_instance.trim_end = Some(10.0); let instance_id = clip_instance.id; @@ -549,7 +562,7 @@ mod tests { let layer_id = document.root.add_child(AnyLayer::Vector(vector_layer)); // Split at timeline 5.0 - let mut action = SplitClipInstanceAction::new(layer_id, instance_id, 5.0); + let mut action = SplitClipInstanceAction::new(layer_id, instance_id, daw_backend::Beats(5.0)); // Execute - this will fail because we don't have a real clip in the document // In a real test, we'd need to add a VectorClip first @@ -559,7 +572,7 @@ mod tests { #[test] fn test_split_action_description() { - let action = SplitClipInstanceAction::new(Uuid::new_v4(), Uuid::new_v4(), 5.0); + let action = SplitClipInstanceAction::new(Uuid::new_v4(), Uuid::new_v4(), daw_backend::Beats(5.0)); assert_eq!(action.description(), "Split clip instance"); } } diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs index 60c1bd7..6d46a66 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs @@ -6,6 +6,7 @@ use crate::action::Action; use crate::clip::ClipInstance; use crate::document::Document; use crate::layer::AnyLayer; +use daw_backend::{Beats, Seconds}; use std::collections::HashMap; use uuid::Uuid; @@ -32,14 +33,14 @@ pub struct TrimData { /// For TrimLeft: trim_start value /// For TrimRight: trim_end value (Option because it can be None) pub trim_value: Option, - /// For TrimLeft: timeline_start value (where the clip appears on timeline) + /// For TrimLeft: timeline_start value (where the clip appears on timeline, beats) /// For TrimRight: unused (None) - pub timeline_start: Option, + pub timeline_start: Option, } impl TrimData { /// Create TrimData for left trim - pub fn left(trim_start: f64, timeline_start: f64) -> Self { + pub fn left(trim_start: f64, timeline_start: Beats) -> Self { Self { trim_value: Some(trim_start), timeline_start: Some(timeline_start), @@ -203,51 +204,73 @@ impl Action for TrimClipInstancesAction { { // If extending to the left (new_trim < old_trim) if should_validate && new_trim < old_trim { - // Find the maximum we can extend left - let max_extend = document.find_max_trim_extend_left( + // Max we can extend left is the timeline gap to the + // previous clip (beats). + let max_extend_beats = document.find_max_trim_extend_left( layer_id, instance_id, instance.timeline_start, ); + // Audio plays 1 content-second per timeline-second, so the + // revealable content equals that gap's wall-clock span. + let tempo_map = document.tempo_map(); + let old_timeline_secs = tempo_map.beats_to_seconds(old_timeline); + let max_extend_secs = (old_timeline_secs + - tempo_map.beats_to_seconds(old_timeline - max_extend_beats)) + .seconds_to_f64(); - // Calculate how much we want to extend + // Calculate how much we want to extend (content seconds) let desired_extend = old_trim - new_trim; // Clamp to max allowed - let actual_extend = desired_extend.min(max_extend); + let actual_extend = desired_extend.min(max_extend_secs); let clamped_trim_start = old_trim - actual_extend; - let clamped_timeline_start = (old_timeline - actual_extend).max(0.0); + // Move the timeline left by the same wall-clock seconds. + let clamped_timeline_start = tempo_map + .seconds_to_beats(old_timeline_secs - Seconds(actual_extend)) + .max(Beats::ZERO); clamped_new = TrimData::left(clamped_trim_start, clamped_timeline_start); } } } TrimType::TrimRight => { - let old_trim_end = old.trim_value.unwrap_or(clip_duration); - let new_trim_end = new.trim_value.unwrap_or(clip_duration); + let old_trim_end = old.trim_value.unwrap_or(clip_duration.seconds_to_f64()); + let new_trim_end = new.trim_value.unwrap_or(clip_duration.seconds_to_f64()); // If extending to the right (new_trim_end > old_trim_end) if should_validate && new_trim_end > old_trim_end { - // Calculate current effective duration - let current_effective_duration = old_trim_end - instance.trim_start; + let tempo_map = document.tempo_map(); + // Current effective duration in beats (content seconds + // converted to beats at the clip's start). + let content_secs = Seconds(old_trim_end - instance.trim_start); + let current_effective_duration = tempo_map.seconds_to_beats( + tempo_map.beats_to_seconds(instance.timeline_start) + content_secs, + ) - instance.timeline_start; - // Find the maximum we can extend right - let max_extend = document.find_max_trim_extend_right( + // Max we can extend right is the timeline gap to the next + // clip (beats). + let max_extend_beats = document.find_max_trim_extend_right( layer_id, instance_id, instance.timeline_start, current_effective_duration, ); + // Convert that gap to content seconds at the clip's right edge. + let right_edge = instance.timeline_start + current_effective_duration; + let max_extend_secs = (tempo_map.beats_to_seconds(right_edge + max_extend_beats) + - tempo_map.beats_to_seconds(right_edge)) + .seconds_to_f64(); - // Calculate how much we want to extend + // Calculate how much we want to extend (content seconds) let desired_extend = new_trim_end - old_trim_end; // Clamp to max allowed - let actual_extend = desired_extend.min(max_extend); + let actual_extend = desired_extend.min(max_extend_secs); let clamped_trim_end = old_trim_end + actual_extend; // Don't exceed clip duration - let final_trim_end = clamped_trim_end.min(clip_duration); + let final_trim_end = clamped_trim_end.min(clip_duration.seconds_to_f64()); clamped_new = TrimData::right(Some(final_trim_end)); } @@ -376,7 +399,7 @@ impl Action for TrimClipInstancesAction { if let Some(instance) = vl.clip_instances.iter().find(|ci| ci.id == *instance_id) { if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { // Instance already has new values after execute() - controller.set_offset(metatrack_id, instance.timeline_start); + controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(instance.timeline_start)); controller.set_trim_start(metatrack_id, instance.trim_start); controller.set_trim_end(metatrack_id, instance.trim_end); } @@ -466,7 +489,7 @@ impl Action for TrimClipInstancesAction { if let Some(instance) = vl.clip_instances.iter().find(|ci| ci.id == *instance_id) { if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { // Instance already has old values after rollback() - controller.set_offset(metatrack_id, instance.timeline_start); + controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(instance.timeline_start)); controller.set_trim_start(metatrack_id, instance.trim_start); controller.set_trim_end(metatrack_id, instance.trim_end); } @@ -558,7 +581,7 @@ mod tests { let mut vector_layer = VectorLayer::new("Layer 1"); let mut clip_instance = ClipInstance::new(clip_id); - clip_instance.timeline_start = 0.0; + clip_instance.timeline_start = Beats::ZERO; clip_instance.trim_start = 0.0; let instance_id = clip_instance.id; vector_layer.clip_instances.push(clip_instance); @@ -572,8 +595,8 @@ mod tests { vec![( instance_id, TrimType::TrimLeft, - TrimData::left(0.0, 0.0), - TrimData::left(2.0, 2.0), + TrimData::left(0.0, Beats::ZERO), + TrimData::left(2.0, Beats(2.0)), )], ); @@ -590,7 +613,7 @@ mod tests { .find(|ci| ci.id == instance_id) .unwrap(); assert_eq!(instance.trim_start, 2.0); - assert_eq!(instance.timeline_start, 2.0); + assert_eq!(instance.timeline_start, Beats(2.0)); } // Rollback @@ -604,7 +627,7 @@ mod tests { .find(|ci| ci.id == instance_id) .unwrap(); assert_eq!(instance.trim_start, 0.0); - assert_eq!(instance.timeline_start, 0.0); + assert_eq!(instance.timeline_start, Beats::ZERO); } } diff --git a/lightningbeam-ui/lightningbeam-core/src/clip.rs b/lightningbeam-ui/lightningbeam-core/src/clip.rs index 17a306a..2c216df 100644 --- a/lightningbeam-ui/lightningbeam-core/src/clip.rs +++ b/lightningbeam-ui/lightningbeam-core/src/clip.rs @@ -14,6 +14,7 @@ use crate::layer::AnyLayer; use crate::layer_tree::LayerTree; use crate::object::Transform; +use daw_backend::{Beats, Seconds}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; use uuid::Uuid; @@ -110,7 +111,7 @@ impl VectorClip { pub fn content_duration_with(&self, framerate: f64, tempo_map: &crate::tempo_map::TempoMap, clip_duration_fn: impl Fn(&Uuid) -> Option) -> f64 { let frame_duration = 1.0 / framerate; // Work in beats, convert to seconds at the end. - let mut last_beats: Option = None; + let mut last_beats: Option = None; let mut last_secs: Option = None; for layer_node in self.layers.iter() { @@ -126,18 +127,18 @@ impl VectorClip { }; for ci in clip_instances { // Compute end position of this clip instance in beats - let end_beats = if let Some(td_beats) = ci.timeline_duration { + let end_beats: Beats = if let Some(td_beats) = ci.timeline_duration { ci.timeline_start + td_beats } else if let Some(te) = ci.trim_end { let secs = (te - ci.trim_start).max(0.0); - ci.timeline_start + tempo_map.inverse_transform(tempo_map.transform(ci.timeline_start) + secs) - ci.timeline_start + tempo_map.seconds_to_beats(tempo_map.beats_to_seconds(ci.timeline_start) + Seconds(secs)) } else if let Some(clip_dur_secs) = clip_duration_fn(&ci.clip_id) { let secs = (clip_dur_secs - ci.trim_start).max(0.0); - ci.timeline_start + tempo_map.inverse_transform(tempo_map.transform(ci.timeline_start) + secs) - ci.timeline_start + tempo_map.seconds_to_beats(tempo_map.beats_to_seconds(ci.timeline_start) + Seconds(secs)) } else { continue; }; - last_beats = Some(last_beats.map_or(end_beats, |t: f64| t.max(end_beats))); + last_beats = Some(last_beats.map_or(end_beats, |t: Beats| t.max(end_beats))); } // Vector layer keyframes are in seconds @@ -148,7 +149,7 @@ impl VectorClip { } } - let from_clips = last_beats.map(|b| tempo_map.transform(b)); + let from_clips = last_beats.map(|b| tempo_map.beats_to_seconds(b).seconds_to_f64()); let combined = match (from_clips, last_secs) { (Some(a), Some(b)) => Some(a.max(b)), (Some(a), None) => Some(a), @@ -199,7 +200,7 @@ impl VectorClip { for clip_instance in &vector_layer.clip_instances { // Convert parent clip time (seconds) to nested clip local time (seconds). // timeline_start is in beats; convert to seconds using document BPM. - let start_secs = document.tempo_map().transform(clip_instance.timeline_start); + let start_secs = document.tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); let nested_clip_time = ((clip_time - start_secs) * clip_instance.playback_speed) + clip_instance.trim_start; // Look up the nested clip definition @@ -647,12 +648,12 @@ pub struct ClipInstance { /// When this instance starts on the timeline, in **beats**. /// Default: 0.0 - pub timeline_start: f64, + pub timeline_start: Beats, /// How long this instance appears on the timeline, in **beats**. /// If set and longer than the trimmed content, the content will loop. /// Default: None (use trimmed clip duration, no looping) - pub timeline_duration: Option, + pub timeline_duration: Option, /// Trim start: offset into the clip's internal content, in **seconds**. /// - For audio: byte-offset into the audio file @@ -679,7 +680,7 @@ pub struct ClipInstance { /// When set, loop iterations are drawn/played before the content start. /// Default: None (no pre-loop) #[serde(default, skip_serializing_if = "Option::is_none")] - pub loop_before: Option, + pub loop_before: Option, } /// High 64-bit sentinel used to identify UUIDs that encode a backend audio clip instance ID. @@ -731,7 +732,7 @@ impl ClipInstance { transform: Transform::default(), opacity: 1.0, name: None, - timeline_start: 0.0, + timeline_start: Beats::ZERO, timeline_duration: None, trim_start: 0.0, trim_end: None, @@ -749,7 +750,7 @@ impl ClipInstance { transform: Transform::default(), opacity: 1.0, name: None, - timeline_start: 0.0, + timeline_start: Beats::ZERO, timeline_duration: None, trim_start: 0.0, trim_end: None, @@ -784,8 +785,8 @@ impl ClipInstance { self } - /// Set timeline position - pub fn with_timeline_start(mut self, timeline_start: f64) -> Self { + /// Set timeline position (beats). + pub fn with_timeline_start(mut self, timeline_start: Beats) -> Self { self.timeline_start = timeline_start; self } @@ -810,79 +811,78 @@ impl ClipInstance { } /// Set explicit timeline duration (in beats) by directly setting `timeline_duration`. - pub fn with_timeline_duration(mut self, duration_beats: f64) -> Self { + pub fn with_timeline_duration(mut self, duration_beats: Beats) -> Self { self.timeline_duration = Some(duration_beats); self } /// Content window size in seconds: `trim_end - trim_start`. /// Used for internal looping calculations. - pub fn content_window_secs(&self, clip_duration_secs: f64) -> f64 { - let end = self.trim_end.unwrap_or(clip_duration_secs); - (end - self.trim_start).max(0.0) + pub fn content_window_secs(&self, clip_duration_secs: Seconds) -> Seconds { + let end = self.trim_end.unwrap_or(clip_duration_secs.seconds_to_f64()); + Seconds((end - self.trim_start).max(0.0)) } /// How long this instance appears on the timeline, in **beats**. /// /// If `timeline_duration` is set, returns that (enabling content looping). /// Otherwise converts the content window from seconds to beats using the tempo map. - pub fn effective_duration_beats(&self, clip_duration_secs: f64, tempo_map: &crate::tempo_map::TempoMap) -> f64 { + pub fn effective_duration_beats(&self, clip_duration_secs: Seconds, tempo_map: &crate::tempo_map::TempoMap) -> Beats { if let Some(td) = self.timeline_duration { return td; } - let window_secs = self.content_window_secs(clip_duration_secs); - let start_secs = tempo_map.transform(self.timeline_start); - tempo_map.inverse_transform(start_secs + window_secs) - self.timeline_start + let window = self.content_window_secs(clip_duration_secs); + let start_secs = tempo_map.beats_to_seconds(self.timeline_start); + tempo_map.seconds_to_beats(start_secs + window) - self.timeline_start } /// Left edge of the clip's visual extent on the timeline, in **beats**. - pub fn effective_start(&self) -> f64 { - self.timeline_start - self.loop_before.unwrap_or(0.0) + pub fn effective_start(&self) -> Beats { + self.timeline_start - self.loop_before.unwrap_or(Beats::ZERO) } /// Total visual duration (loop_before + effective_duration), in **beats**. - pub fn total_duration(&self, clip_duration_secs: f64, tempo_map: &crate::tempo_map::TempoMap) -> f64 { - self.loop_before.unwrap_or(0.0) + self.effective_duration_beats(clip_duration_secs, tempo_map) + pub fn total_duration(&self, clip_duration_secs: Seconds, tempo_map: &crate::tempo_map::TempoMap) -> Beats { + self.loop_before.unwrap_or(Beats::ZERO) + self.effective_duration_beats(clip_duration_secs, tempo_map) } /// Map a playback time (in **seconds**) to clip-local content time (in **seconds**). /// /// Returns `None` if the clip instance is not active at `time_secs`. - pub fn remap_time_secs(&self, time_secs: f64, clip_duration_secs: f64, tempo_map: &crate::tempo_map::TempoMap) -> Option { - let start_secs = tempo_map.transform(self.timeline_start); + pub fn remap_time_secs(&self, time: Seconds, clip_duration_secs: Seconds, tempo_map: &crate::tempo_map::TempoMap) -> Option { + let start_secs = tempo_map.beats_to_seconds(self.timeline_start); let dur_beats = self.effective_duration_beats(clip_duration_secs, tempo_map); - let end_secs = tempo_map.transform(self.timeline_start + dur_beats); + let end_secs = tempo_map.beats_to_seconds(self.timeline_start + dur_beats); - if time_secs < start_secs || time_secs >= end_secs { + if time < start_secs || time >= end_secs { return None; } - let relative_secs = time_secs - start_secs; - let content_time = relative_secs * self.playback_speed; + let content_time = (time - start_secs) * self.playback_speed; let content_window = self.content_window_secs(clip_duration_secs); - if content_window == 0.0 { - return Some(self.trim_start); + if content_window == Seconds::ZERO { + return Some(Seconds(self.trim_start)); } - let looped_time = if content_time > content_window { + let looped = if content_time > content_window { content_time % content_window } else { content_time }; - Some(self.trim_start + looped_time) + Some(Seconds(self.trim_start) + looped) } /// Alias for `remap_time_secs`. #[inline] - pub fn remap_time(&self, time_secs: f64, clip_duration_secs: f64, tempo_map: &crate::tempo_map::TempoMap) -> Option { - self.remap_time_secs(time_secs, clip_duration_secs, tempo_map) + pub fn remap_time(&self, time: Seconds, clip_duration_secs: Seconds, tempo_map: &crate::tempo_map::TempoMap) -> Option { + self.remap_time_secs(time, clip_duration_secs, tempo_map) } /// Alias for `effective_duration_beats`. #[inline] - pub fn effective_duration(&self, clip_duration_secs: f64, tempo_map: &crate::tempo_map::TempoMap) -> f64 { + pub fn effective_duration(&self, clip_duration_secs: Seconds, tempo_map: &crate::tempo_map::TempoMap) -> Beats { self.effective_duration_beats(clip_duration_secs, tempo_map) } @@ -959,7 +959,7 @@ mod tests { assert_eq!(instance.clip_id, clip_id); assert_eq!(instance.opacity, 1.0); - assert_eq!(instance.timeline_start, 0.0); + assert_eq!(instance.timeline_start, Beats::ZERO); assert_eq!(instance.trim_start, 0.0); assert_eq!(instance.trim_end, None); assert_eq!(instance.playback_speed, 1.0); @@ -977,7 +977,7 @@ mod tests { // At 60 BPM the tempo map is identity (1 beat == 1 second), so the // beats-domain effective duration equals the seconds content window. let tempo_map = crate::tempo_map::TempoMap::constant(60.0); - assert_eq!(instance.effective_duration(10.0, &tempo_map), 6.0); + assert_eq!(instance.effective_duration(Seconds(10.0), &tempo_map), Beats(6.0)); } #[test] @@ -990,7 +990,7 @@ mod tests { assert_eq!(instance.trim_end, None); // At 60 BPM the tempo map is identity (1 beat == 1 second). let tempo_map = crate::tempo_map::TempoMap::constant(60.0); - assert_eq!(instance.effective_duration(10.0, &tempo_map), 8.0); + assert_eq!(instance.effective_duration(Seconds(10.0), &tempo_map), Beats(8.0)); } #[test] diff --git a/lightningbeam-ui/lightningbeam-core/src/document.rs b/lightningbeam-ui/lightningbeam-core/src/document.rs index a27b405..f60fe33 100644 --- a/lightningbeam-ui/lightningbeam-core/src/document.rs +++ b/lightningbeam-ui/lightningbeam-core/src/document.rs @@ -5,6 +5,7 @@ use crate::asset_folder::AssetFolderTree; use crate::clip::{AudioClip, ClipInstance, ImageAsset, VideoClip, VectorClip}; +use daw_backend::{Beats, Seconds}; use crate::effect::EffectDefinition; use crate::layer::{AnyLayer, GroupLayer}; use crate::script::ScriptDefinition; @@ -438,20 +439,24 @@ impl Document { /// Returns the end time of the last clip instance across all layers, /// or the document's duration if no clips are found. pub fn calculate_timeline_endpoint(&self) -> f64 { + let tempo_map = self.tempo_map(); + // Accumulated in **beats** (as f64, to keep the recursive helper's `Fn(_, f64) -> f64` + // signature); converted to seconds once at the return. let mut max_end_time: f64 = 0.0; - // Helper function to calculate the end time of a clip instance + // End position of a clip instance, in **beats**. Its trimmed content window is in seconds + // (scaled by playback speed), so convert that seconds span to beats via the tempo map — the + // old code added a seconds duration straight onto the beats start. let calculate_instance_end = |instance: &ClipInstance, clip_duration: f64| -> f64 { - let effective_duration = if let Some(timeline_duration) = instance.timeline_duration { - // Explicit timeline duration set (may include looping) - timeline_duration + let end_beats: Beats = if let Some(timeline_duration) = instance.timeline_duration { + instance.timeline_start + timeline_duration } else { - // Calculate from trim points let trim_end = instance.trim_end.unwrap_or(clip_duration); - let trimmed_duration = trim_end - instance.trim_start; - trimmed_duration / instance.playback_speed // Adjust for playback speed + let trimmed_secs = ((trim_end - instance.trim_start) / instance.playback_speed).max(0.0); + let start_secs = tempo_map.beats_to_seconds(instance.timeline_start); + tempo_map.seconds_to_beats(start_secs + Seconds(trimmed_secs)) }; - instance.timeline_start + effective_duration + end_beats.beats_to_f64() }; // Iterate through all layers to find the maximum end time @@ -484,7 +489,7 @@ impl Document { crate::layer::AnyLayer::Effect(effect_layer) => { for instance in &effect_layer.clip_instances { if let Some(clip_duration) = self.get_clip_duration(&instance.clip_id) { - let end_time = calculate_instance_end(instance, clip_duration); + let end_time = calculate_instance_end(instance, clip_duration.seconds_to_f64()); max_end_time = max_end_time.max(end_time); } } @@ -526,7 +531,7 @@ impl Document { crate::layer::AnyLayer::Effect(el) => { for inst in &el.clip_instances { if let Some(dur) = doc.get_clip_duration(&inst.clip_id) { - *max_end = max_end.max(calc_end(inst, dur)); + *max_end = max_end.max(calc_end(inst, dur.seconds_to_f64())); } } } @@ -544,9 +549,10 @@ impl Document { } } - // Return the maximum end time, or document duration if no clips found + // Return the max end (converting the beats accumulator to seconds), or the document + // duration (already seconds) if no clips were found. if max_end_time > 0.0 { - max_end_time + tempo_map.beats_to_seconds(Beats(max_end_time)).seconds_to_f64() } else { self.duration } @@ -890,13 +896,13 @@ impl Document { /// Searches through all clip libraries to find the clip and return its duration. /// For effect definitions, returns `EFFECT_DURATION` (f64::MAX) since effects /// have infinite internal duration. - pub fn get_clip_duration(&self, clip_id: &Uuid) -> Option { + pub fn get_clip_duration(&self, clip_id: &Uuid) -> Option { if let Some(clip) = self.vector_clips.get(clip_id) { if clip.is_group { - Some(clip.duration) + Some(Seconds(clip.duration)) } else { let tempo_map = self.tempo_map(); - Some(clip.content_duration_with(self.framerate, tempo_map, |id| { + Some(Seconds(clip.content_duration_with(self.framerate, tempo_map, |id| { // Resolve nested clip durations (audio, video, other vector clips) if let Some(vc) = self.vector_clips.get(id) { // Avoid deep recursion — use stored duration for nested vector clips @@ -910,23 +916,23 @@ impl Document { } else { None } - })) + }))) } } else if let Some(clip) = self.video_clips.get(clip_id) { - Some(clip.duration) + Some(Seconds(clip.duration)) } else if let Some(clip) = self.audio_clips.get(clip_id) { - Some(clip.duration) + Some(Seconds(clip.duration)) } else if self.effect_definitions.contains_key(clip_id) { // Effects have infinite internal duration - their timeline length // is controlled by ClipInstance.trim_end - Some(crate::effect::EFFECT_DURATION) + Some(Seconds(crate::effect::EFFECT_DURATION)) } else { None } } - /// Calculate the end time of a clip instance on the timeline - pub fn get_clip_instance_end_time(&self, layer_id: &Uuid, instance_id: &Uuid) -> Option { + /// Calculate the end position of a clip instance on the timeline, in **beats**. + pub fn get_clip_instance_end_time(&self, layer_id: &Uuid, instance_id: &Uuid) -> Option { let layer = self.get_layer(layer_id)?; // Find the clip instance @@ -942,12 +948,8 @@ impl Document { let instance = instances.iter().find(|inst| &inst.id == instance_id)?; let clip_duration = self.get_clip_duration(&instance.clip_id)?; - - let trim_start = instance.trim_start; - let trim_end = instance.trim_end.unwrap_or(clip_duration); - let effective_duration = trim_end - trim_start; - - Some(instance.timeline_start + effective_duration) + // End position on the timeline, in beats (convert the seconds content window via tempo map). + Some(instance.timeline_start + instance.effective_duration_beats(clip_duration, self.tempo_map())) } /// Check if a time range overlaps with any existing clip on the layer @@ -958,8 +960,8 @@ impl Document { pub fn check_overlap_on_layer( &self, layer_id: &Uuid, - start_time: f64, - end_time: f64, + start_time: Beats, + end_time: Beats, exclude_instance_ids: &[Uuid], ) -> (bool, Option) { let Some(layer) = self.get_layer(layer_id) else { @@ -1012,14 +1014,14 @@ impl Document { pub fn find_nearest_valid_position( &self, layer_id: &Uuid, - desired_start: f64, - clip_duration: f64, + desired_start: Beats, + clip_duration: Beats, exclude_instance_ids: &[Uuid], - ) -> Option { + ) -> Option { let layer = self.get_layer(layer_id)?; // Clamp to timeline start (can't go before 0) - let desired_start = desired_start.max(0.0); + let desired_start = desired_start.max(Beats::ZERO); // Vector layers don't need overlap adjustment, but still respect timeline start if matches!(layer, AnyLayer::Vector(_) | AnyLayer::Group(_)) { @@ -1044,7 +1046,7 @@ impl Document { AnyLayer::Text(_) => return Some(desired_start), // Text layers don't have own clips }; - let mut occupied_ranges: Vec<(f64, f64, Uuid)> = Vec::new(); + let mut occupied_ranges: Vec<(Beats, Beats, Uuid)> = Vec::new(); for instance in instances { if exclude_instance_ids.contains(&instance.id) { continue; @@ -1063,7 +1065,7 @@ impl Document { // Find the clip we're overlapping with and try both sides, pick nearest for (occupied_start, occupied_end, _) in &occupied_ranges { if desired_start < *occupied_end && *occupied_start < desired_end { - let mut candidates: Vec = Vec::new(); + let mut candidates: Vec = Vec::new(); // Try snapping to the right (after this clip) let snap_right = *occupied_end; @@ -1079,8 +1081,8 @@ impl Document { } // Try snapping to the left (before this clip) - let snap_left = occupied_start - clip_duration; - if snap_left >= 0.0 { + let snap_left = *occupied_start - clip_duration; + if snap_left >= Beats::ZERO { let (overlaps_left, _) = self.check_overlap_on_layer( layer_id, snap_left, @@ -1095,8 +1097,8 @@ impl Document { // Pick the candidate closest to desired_start if !candidates.is_empty() { candidates.sort_by(|a, b| { - let dist_a = (a - desired_start).abs(); - let dist_b = (b - desired_start).abs(); + let dist_a = (*a - desired_start).abs(); + let dist_b = (*b - desired_start).abs(); dist_a.partial_cmp(&dist_b).unwrap_or(std::cmp::Ordering::Equal) }); return Some(candidates[0]); @@ -1106,7 +1108,7 @@ impl Document { // If no gap found, try placing at timeline start if occupied_ranges.is_empty() || occupied_ranges[0].0 >= clip_duration { - return Some(0.0); + return Some(Beats::ZERO); } // No valid position found @@ -1118,9 +1120,9 @@ impl Document { pub fn clamp_group_move_offset( &self, layer_id: &Uuid, - group: &[(Uuid, f64, f64)], // (instance_id, timeline_start, effective_duration) - desired_offset: f64, - ) -> f64 { + group: &[(Uuid, Beats, Beats)], // (instance_id, timeline_start, effective_duration) in beats + desired_offset: Beats, + ) -> Beats { let Some(layer) = self.get_layer(layer_id) else { return desired_offset; }; @@ -1140,8 +1142,8 @@ impl Document { AnyLayer::Text(_) => &[], }; - // Collect non-group clip ranges - let mut non_group: Vec<(f64, f64)> = Vec::new(); + // Collect non-group clip ranges (beats) + let mut non_group: Vec<(Beats, Beats)> = Vec::new(); for inst in instances { if group_ids.contains(&inst.id) { continue; @@ -1163,12 +1165,12 @@ impl Document { // Check against non-group clips for &(ns, ne) in &non_group { - if clamped < 0.0 { + if clamped < Beats::ZERO { // Moving left: if non-group clip end is between our destination and current start if ne <= start && ne > start + clamped { clamped = clamped.max(ne - start); } - } else if clamped > 0.0 { + } else if clamped > Beats::ZERO { // Moving right: if non-group clip start is between our current end and destination if ns >= end && ns < end + clamped { clamped = clamped.min(ns - end); @@ -1188,8 +1190,8 @@ impl Document { &self, layer_id: &Uuid, instance_id: &Uuid, - current_timeline_start: f64, - ) -> f64 { + current_timeline_start: Beats, + ) -> Beats { let Some(layer) = self.get_layer(layer_id) else { return current_timeline_start; // No limit if layer not found }; @@ -1200,7 +1202,7 @@ impl Document { }; // Find the nearest clip to the left - let mut nearest_end = 0.0; // Can extend to timeline start by default + let mut nearest_end = Beats::ZERO; // Can extend to timeline start by default let instances: &[ClipInstance] = match layer { AnyLayer::Audio(audio) => &audio.clip_instances, @@ -1220,6 +1222,7 @@ impl Document { // Calculate other clip's extent (accounting for loop_before) if let Some(clip_duration) = self.get_clip_duration(&other.clip_id) { let other_end = other.timeline_start + other.effective_duration(clip_duration, self.tempo_map()); + // (clip_duration is Seconds via get_clip_duration; effective_duration converts.) // If this clip is to the left and closer than current nearest if other_end <= current_timeline_start && other_end > nearest_end { @@ -1239,16 +1242,16 @@ impl Document { &self, layer_id: &Uuid, instance_id: &Uuid, - current_timeline_start: f64, - current_effective_duration: f64, - ) -> f64 { + current_timeline_start: Beats, + current_effective_duration: Beats, + ) -> Beats { let Some(layer) = self.get_layer(layer_id) else { - return f64::MAX; // No limit if layer not found + return Beats(f64::MAX); // No limit if layer not found }; // Only check audio, video, and effect layers if matches!(layer, AnyLayer::Vector(_) | AnyLayer::Group(_)) { - return f64::MAX; // No limit for vector/group layers + return Beats(f64::MAX); // No limit for vector/group layers } let instances: &[ClipInstance] = match layer { @@ -1261,7 +1264,7 @@ impl Document { AnyLayer::Text(_) => &[], }; - let mut nearest_start = f64::MAX; + let mut nearest_start = Beats(f64::MAX); let current_end = current_timeline_start + current_effective_duration; for other in instances { @@ -1276,10 +1279,10 @@ impl Document { } } - if nearest_start == f64::MAX { - f64::MAX // No clip to the right, can extend freely + if nearest_start == Beats(f64::MAX) { + Beats(f64::MAX) // No clip to the right, can extend freely } else { - (nearest_start - current_end).max(0.0) // Gap between our end and next clip's start + (nearest_start - current_end).max(Beats::ZERO) // Gap between our end and next clip's start } } /// Find the maximum amount we can extend loop_before to the left without overlapping. @@ -1289,8 +1292,8 @@ impl Document { &self, layer_id: &Uuid, instance_id: &Uuid, - current_effective_start: f64, - ) -> f64 { + current_effective_start: Beats, + ) -> Beats { let Some(layer) = self.get_layer(layer_id) else { return current_effective_start; }; @@ -1309,7 +1312,7 @@ impl Document { AnyLayer::Text(_) => &[], }; - let mut nearest_end = 0.0; + let mut nearest_end = Beats::ZERO; for other in instances { if &other.id == instance_id { diff --git a/lightningbeam-ui/lightningbeam-core/src/effect.rs b/lightningbeam-ui/lightningbeam-core/src/effect.rs index ab5186c..ccabbcd 100644 --- a/lightningbeam-ui/lightningbeam-core/src/effect.rs +++ b/lightningbeam-ui/lightningbeam-core/src/effect.rs @@ -351,7 +351,7 @@ impl EffectDefinition { /// /// * `timeline_start` - When the effect starts on the timeline (seconds) /// * `duration` - How long the effect appears on the timeline (seconds) - pub fn create_instance(&self, timeline_start: f64, duration: f64) -> ClipInstance { + pub fn create_instance(&self, timeline_start: daw_backend::Beats, duration: daw_backend::Beats) -> ClipInstance { ClipInstance::new(self.id) .with_timeline_start(timeline_start) .with_timeline_duration(duration) diff --git a/lightningbeam-ui/lightningbeam-core/src/effect_layer.rs b/lightningbeam-ui/lightningbeam-core/src/effect_layer.rs index 3a830ba..3d52675 100644 --- a/lightningbeam-ui/lightningbeam-core/src/effect_layer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/effect_layer.rs @@ -148,11 +148,11 @@ impl EffectLayer { /// `timeline_start` (beats) to seconds for comparison. pub fn active_clip_instances_at(&self, time_secs: f64, tempo_map: &crate::tempo_map::TempoMap) -> Vec<&ClipInstance> { use crate::effect::EFFECT_DURATION; - let time_beats = tempo_map.inverse_transform(time_secs); + let time_beats = tempo_map.seconds_to_beats(daw_backend::Seconds(time_secs)); self.clip_instances .iter() .filter(|e| { - let end = e.timeline_start + e.effective_duration(EFFECT_DURATION, tempo_map); + let end = e.timeline_start + e.effective_duration(daw_backend::Seconds(EFFECT_DURATION), tempo_map); time_beats >= e.timeline_start && time_beats < end }) .collect() @@ -218,7 +218,7 @@ mod tests { fn test_add_effect() { let mut layer = EffectLayer::new("Effects"); let def = create_test_effect_def(); - let effect = def.create_instance(0.0, 10.0); + let effect = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let effect_id = effect.id; let id = layer.add_clip_instance(effect); @@ -233,11 +233,11 @@ mod tests { let def = create_test_effect_def(); // Effect 1: active from 0 to 5 - let effect1 = def.create_instance(0.0, 5.0); + let effect1 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(5.0)); layer.add_clip_instance(effect1); // Effect 2: active from 3 to 10 - let effect2 = def.create_instance(3.0, 7.0); // 3.0 + 7.0 = 10.0 end + let effect2 = def.create_instance(daw_backend::Beats(3.0), daw_backend::Beats(7.0)); // 3.0 + 7.0 = 10.0 end layer.add_clip_instance(effect2); // At 60 BPM the tempo map is identity (1 beat == 1 second), so the @@ -259,11 +259,11 @@ mod tests { let mut layer = EffectLayer::new("Effects"); let def = create_test_effect_def(); - let effect1 = def.create_instance(0.0, 10.0); + let effect1 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id1 = effect1.id; layer.add_clip_instance(effect1); - let effect2 = def.create_instance(0.0, 10.0); + let effect2 = def.create_instance(daw_backend::Beats(0.0), daw_backend::Beats(10.0)); let id2 = effect2.id; layer.add_clip_instance(effect2); diff --git a/lightningbeam-ui/lightningbeam-core/src/hit_test.rs b/lightningbeam-ui/lightningbeam-core/src/hit_test.rs index 8e647c6..9cc7a92 100644 --- a/lightningbeam-ui/lightningbeam-core/src/hit_test.rs +++ b/lightningbeam-ui/lightningbeam-core/src/hit_test.rs @@ -260,15 +260,15 @@ pub fn hit_test_clip_instances( for clip_instance in clip_instances.iter().rev() { // Check time bounds: skip clip instances not active at this time // timeline_start/instance_end are in beats; convert timeline_time (seconds) to beats. - let clip_duration = document.get_clip_duration(&clip_instance.clip_id).unwrap_or(0.0); + let clip_duration = document.get_clip_duration(&clip_instance.clip_id).unwrap_or(daw_backend::Seconds::ZERO); let instance_end = clip_instance.timeline_start + clip_instance.effective_duration(clip_duration, tempo_map); - let timeline_beats = tempo_map.inverse_transform(timeline_time); + let timeline_beats = tempo_map.seconds_to_beats(daw_backend::Seconds(timeline_time)); if timeline_beats < clip_instance.timeline_start || timeline_beats >= instance_end { continue; } // clip_time is in seconds; offset from clip start (in seconds) + trim_start (seconds) - let start_secs = tempo_map.transform(clip_instance.timeline_start); + let start_secs = tempo_map.beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); let clip_time = ((timeline_time - start_secs) * clip_instance.playback_speed) + clip_instance.trim_start; let content_bounds = if let Some(vector_clip) = document.get_vector_clip(&clip_instance.clip_id) { @@ -304,14 +304,14 @@ pub fn hit_test_clip_instances_in_rect( for clip_instance in clip_instances { // Check time bounds: skip clip instances not active at this time // timeline_start/instance_end are in beats; convert timeline_time (seconds) to beats. - let clip_duration = document.get_clip_duration(&clip_instance.clip_id).unwrap_or(0.0); + let clip_duration = document.get_clip_duration(&clip_instance.clip_id).unwrap_or(daw_backend::Seconds::ZERO); let instance_end = clip_instance.timeline_start + clip_instance.effective_duration(clip_duration, tempo_map); - let timeline_beats = tempo_map.inverse_transform(timeline_time); + let timeline_beats = tempo_map.seconds_to_beats(daw_backend::Seconds(timeline_time)); if timeline_beats < clip_instance.timeline_start || timeline_beats >= instance_end { continue; } - let start_secs = tempo_map.transform(clip_instance.timeline_start); + let start_secs = tempo_map.beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); let clip_time = ((timeline_time - start_secs) * clip_instance.playback_speed) + clip_instance.trim_start; let content_bounds = if let Some(vector_clip) = document.get_vector_clip(&clip_instance.clip_id) { diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index 01b098d..9c2cbd8 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -11,6 +11,7 @@ use crate::animation::TransformProperty; use crate::clip::{ClipInstance, ImageAsset}; use crate::document::Document; +use daw_backend::Seconds; use crate::gpu::BlendMode; use crate::layer::{AnyLayer, LayerTrait, VectorLayer}; use kurbo::Affine; @@ -567,7 +568,8 @@ pub fn render_layer_isolated( let tempo_map = document.tempo_map(); for clip_instance in &video_layer.clip_instances { let Some(video_clip) = document.video_clips.get(&clip_instance.clip_id) else { continue }; - let Some(clip_time) = clip_instance.remap_time(time, video_clip.duration, tempo_map) else { continue }; + let Some(clip_time) = clip_instance.remap_time(Seconds(time), Seconds(video_clip.duration), tempo_map) else { continue }; + let clip_time = clip_time.seconds_to_f64(); let Some(frame) = video_mgr.get_frame(&clip_instance.clip_id, clip_time, target_w, target_h) else { continue }; // Evaluate animated transform properties. @@ -975,7 +977,7 @@ pub fn render_single_clip_instance( .filter(|vc| vc.is_group) .map(|_| { let frame_duration = 1.0 / document.framerate; - vector_layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration) + vector_layer.group_visibility_end(&clip_instance.id, document.tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(), frame_duration) }); render_clip_instance( @@ -1010,18 +1012,18 @@ fn render_clip_instance( let clip_time = if vector_clip.is_group { // Groups are static — visible from timeline_start to the next keyframe boundary. // timeline_start is in beats; group_end_time is in seconds (render time). - let start_secs = tempo_map.transform(clip_instance.timeline_start); + let start_secs = tempo_map.beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); let end = group_end_time.unwrap_or(start_secs); if time < start_secs || time >= end { return; } 0.0 } else { - let clip_dur = document.get_clip_duration(&vector_clip.id).unwrap_or(vector_clip.duration); - let Some(t) = clip_instance.remap_time(time, clip_dur, tempo_map) else { + let clip_dur = document.get_clip_duration(&vector_clip.id).unwrap_or(Seconds(vector_clip.duration)); + let Some(t) = clip_instance.remap_time(Seconds(time), clip_dur, tempo_map) else { return; // Clip instance not active at this time }; - t + t.seconds_to_f64() }; // Evaluate animated transform properties @@ -1172,9 +1174,10 @@ fn render_video_layer( // Remap timeline time to clip's internal time let tempo_map = document.tempo_map(); - let Some(clip_time) = clip_instance.remap_time(time, video_clip.duration, tempo_map) else { + let Some(clip_time) = clip_instance.remap_time(Seconds(time), Seconds(video_clip.duration), tempo_map) else { continue; // Clip instance not active at this time }; + let clip_time = clip_time.seconds_to_f64(); // Get video frame from VideoManager at the output (export/preview) resolution. let (target_w, target_h) = video_decode_target(document, base_transform); @@ -1568,7 +1571,7 @@ fn render_vector_layer( .filter(|vc| vc.is_group) .map(|_| { let frame_duration = 1.0 / document.framerate; - layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration) + layer.group_visibility_end(&clip_instance.id, document.tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(), frame_duration) }); render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time, extract.as_deref_mut()); } @@ -1877,7 +1880,7 @@ fn render_vector_layer_cpu( .filter(|vc| vc.is_group) .map(|_| { let frame_duration = 1.0 / document.framerate; - layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration) + layer.group_visibility_end(&clip_instance.id, document.tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(), frame_duration) }); render_clip_instance_cpu( document, time, clip_instance, layer_opacity, pixmap, base_transform, @@ -1902,14 +1905,14 @@ fn render_clip_instance_cpu( let tempo_map = document.tempo_map(); let clip_time = if vector_clip.is_group { - let start_secs = tempo_map.transform(clip_instance.timeline_start); + let start_secs = tempo_map.beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); let end = group_end_time.unwrap_or(start_secs); if time < start_secs || time >= end { return; } 0.0 } else { - let clip_dur = document.get_clip_duration(&vector_clip.id).unwrap_or(vector_clip.duration); - let Some(t) = clip_instance.remap_time(time, clip_dur, tempo_map) else { return }; - t + let clip_dur = document.get_clip_duration(&vector_clip.id).unwrap_or(Seconds(vector_clip.duration)); + let Some(t) = clip_instance.remap_time(Seconds(time), clip_dur, tempo_map) else { return }; + t.seconds_to_f64() }; let transform = &clip_instance.transform; From 0050c18623470a7928e293555d941e189fad2c70 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Sat, 11 Jul 2026 12:11:05 -0400 Subject: [PATCH 2/8] =?UTF-8?q?Timeline=20types=20stage=202:=20fix=20edito?= =?UTF-8?q?r=20seconds=E2=86=94beats=20at=20every=20clip=20boundary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread Beats/Seconds through the editor now that lightningbeam-core is typed. The timeline UI is seconds-domain (pixels_per_second, viewport_start_time, playback_time, drag_offset), while ClipInstance.timeline_start/duration/ loop_before are beats — the compiler flagged every place the two were mixed. Fixes the reported bugs and the whole class behind them: - Recording placement (audio + MIDI + webcam): the new clip's timeline_start was written from playback_time (seconds) into a beats field, so a second recording landed at the wrong time. Now converted via the tempo map. - Drag/move: introduce snapped_move_offset -> Beats (snap in the seconds/pixel domain, express the anchor's movement in beats) and moved_start(); the group clamp, live preview, and commit all use one uniform beats offset instead of adding a seconds delta to a beats position. - Trim/loop drag preview + commit: overlap limits come from the timeline (beats) but trims are content seconds — converted at the boundary. - Drop (asset drag, stage + timeline), paste, duplicate, split-at-playhead: all convert the seconds drop/playhead position to beats before placing. - Stage playback gates + clip-local time remap: compared seconds playback_time against beats timeline_start; now both in seconds. - Piano roll / infopanel / effect export: clip_dur is Seconds; effect and waveform data converted at use. Core API refinement (motivated by the above): find_max_trim_extend_left/right now return Seconds (the content-seconds gap) instead of raw Beats, since every trim caller wants seconds; loop-extend callers convert to beats. Added a beats_to_x() timeline helper. Whole workspace compiles; 299 core tests pass. --- .../src/actions/trim_clip_instances.rs | 29 +- .../lightningbeam-core/src/document.rs | 27 +- .../src/export/video_exporter.rs | 7 +- .../lightningbeam-editor/src/main.rs | 43 +- .../src/panes/infopanel.rs | 8 +- .../src/panes/piano_roll.rs | 14 +- .../lightningbeam-editor/src/panes/stage.rs | 63 ++- .../src/panes/timeline.rs | 484 +++++++++++------- 8 files changed, 398 insertions(+), 277 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs index 6d46a66..b3da86c 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs @@ -204,20 +204,12 @@ impl Action for TrimClipInstancesAction { { // If extending to the left (new_trim < old_trim) if should_validate && new_trim < old_trim { - // Max we can extend left is the timeline gap to the - // previous clip (beats). - let max_extend_beats = document.find_max_trim_extend_left( + // Max leftward extension as content seconds (the gap's wall-clock span). + let max_extend_secs = document.find_max_trim_extend_left( layer_id, instance_id, instance.timeline_start, - ); - // Audio plays 1 content-second per timeline-second, so the - // revealable content equals that gap's wall-clock span. - let tempo_map = document.tempo_map(); - let old_timeline_secs = tempo_map.beats_to_seconds(old_timeline); - let max_extend_secs = (old_timeline_secs - - tempo_map.beats_to_seconds(old_timeline - max_extend_beats)) - .seconds_to_f64(); + ).seconds_to_f64(); // Calculate how much we want to extend (content seconds) let desired_extend = old_trim - new_trim; @@ -226,8 +218,9 @@ impl Action for TrimClipInstancesAction { let actual_extend = desired_extend.min(max_extend_secs); let clamped_trim_start = old_trim - actual_extend; // Move the timeline left by the same wall-clock seconds. + let tempo_map = document.tempo_map(); let clamped_timeline_start = tempo_map - .seconds_to_beats(old_timeline_secs - Seconds(actual_extend)) + .seconds_to_beats(tempo_map.beats_to_seconds(old_timeline) - Seconds(actual_extend)) .max(Beats::ZERO); clamped_new = TrimData::left(clamped_trim_start, clamped_timeline_start); @@ -248,19 +241,13 @@ impl Action for TrimClipInstancesAction { tempo_map.beats_to_seconds(instance.timeline_start) + content_secs, ) - instance.timeline_start; - // Max we can extend right is the timeline gap to the next - // clip (beats). - let max_extend_beats = document.find_max_trim_extend_right( + // Max rightward extension as content seconds (the gap's wall-clock span). + let max_extend_secs = document.find_max_trim_extend_right( layer_id, instance_id, instance.timeline_start, current_effective_duration, - ); - // Convert that gap to content seconds at the clip's right edge. - let right_edge = instance.timeline_start + current_effective_duration; - let max_extend_secs = (tempo_map.beats_to_seconds(right_edge + max_extend_beats) - - tempo_map.beats_to_seconds(right_edge)) - .seconds_to_f64(); + ).seconds_to_f64(); // Calculate how much we want to extend (content seconds) let desired_extend = new_trim_end - old_trim_end; diff --git a/lightningbeam-ui/lightningbeam-core/src/document.rs b/lightningbeam-ui/lightningbeam-core/src/document.rs index f60fe33..7bca2ea 100644 --- a/lightningbeam-ui/lightningbeam-core/src/document.rs +++ b/lightningbeam-ui/lightningbeam-core/src/document.rs @@ -1186,19 +1186,21 @@ impl Document { /// /// Returns the distance to the nearest clip to the left, or the distance to /// timeline start (0.0) if no clips exist to the left. + /// Returns the max leftward trim extension as a content-seconds span (the wall-clock + /// length of the timeline gap to the previous clip); the trim domain is seconds. pub fn find_max_trim_extend_left( &self, layer_id: &Uuid, instance_id: &Uuid, current_timeline_start: Beats, - ) -> Beats { + ) -> Seconds { let Some(layer) = self.get_layer(layer_id) else { - return current_timeline_start; // No limit if layer not found + return self.tempo_map().beats_to_seconds(current_timeline_start); // No limit if layer not found }; // Only check audio, video, and effect layers if matches!(layer, AnyLayer::Vector(_) | AnyLayer::Group(_)) { - return current_timeline_start; // No limit for vector/group layers + return self.tempo_map().beats_to_seconds(current_timeline_start); // No limit for vector/group layers }; // Find the nearest clip to the left @@ -1231,27 +1233,27 @@ impl Document { } } - current_timeline_start - nearest_end + self.tempo_map().beats_to_seconds(current_timeline_start) - self.tempo_map().beats_to_seconds(nearest_end) } - /// Find the maximum amount we can extend a clip to the right without overlapping + /// Find the maximum amount we can extend a clip to the right without overlapping. /// - /// Returns the distance to the nearest clip to the right, or f64::MAX if no - /// clips exist to the right. + /// Returns the content-seconds span of the timeline gap to the nearest clip on the + /// right, or Seconds(f64::MAX) if none. `current_effective_duration` is beats (timeline). pub fn find_max_trim_extend_right( &self, layer_id: &Uuid, instance_id: &Uuid, current_timeline_start: Beats, current_effective_duration: Beats, - ) -> Beats { + ) -> Seconds { let Some(layer) = self.get_layer(layer_id) else { - return Beats(f64::MAX); // No limit if layer not found + return Seconds(f64::MAX); // No limit if layer not found }; // Only check audio, video, and effect layers if matches!(layer, AnyLayer::Vector(_) | AnyLayer::Group(_)) { - return Beats(f64::MAX); // No limit for vector/group layers + return Seconds(f64::MAX); // No limit for vector/group layers } let instances: &[ClipInstance] = match layer { @@ -1280,9 +1282,10 @@ impl Document { } if nearest_start == Beats(f64::MAX) { - Beats(f64::MAX) // No clip to the right, can extend freely + Seconds(f64::MAX) // No clip to the right, can extend freely } else { - (nearest_start - current_end).max(Beats::ZERO) // Gap between our end and next clip's start + // Gap between our end and next clip's start, as content seconds. + (self.tempo_map().beats_to_seconds(nearest_start) - self.tempo_map().beats_to_seconds(current_end)).max(Seconds::ZERO) } } /// Find the maximum amount we can extend loop_before to the left without overlapping. diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 8a99d2a..bea0759 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -1062,10 +1062,13 @@ fn composite_document_to_hdr( let success = gpu_resources.effect_processor.compile_effect(device, effect_def); if !success { eprintln!("Failed to compile effect: {}", effect_def.name); continue; } } + let tempo_map = document.tempo_map(); + let effect_end_beats = effect_instance.timeline_start + + effect_instance.effective_duration(daw_backend::Seconds(lightningbeam_core::effect::EFFECT_DURATION), tempo_map); let effect_inst = lightningbeam_core::effect::EffectInstance::new( effect_def, - effect_instance.timeline_start, - effect_instance.timeline_start + effect_instance.effective_duration(lightningbeam_core::effect::EFFECT_DURATION, document.tempo_map()), + tempo_map.beats_to_seconds(effect_instance.timeline_start).seconds_to_f64(), + tempo_map.beats_to_seconds(effect_end_beats).seconds_to_f64(), ); let effect_output_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); if let Some(effect_output_view) = gpu_resources.buffer_pool.get_view(effect_output_handle) { diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 6a9caaa..7f98849 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -1,6 +1,7 @@ #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] use eframe::egui; +use daw_backend::{Beats, Seconds}; use lightningbeam_core::layer::{AnyLayer, AudioLayer}; use lightningbeam_core::layout::{LayoutDefinition, LayoutNode}; use lightningbeam_core::pane::PaneType; @@ -2352,7 +2353,8 @@ impl EditorApp { use lightningbeam_core::instance_group::InstanceGroup; use std::collections::HashSet; - let split_time = self.playback_time; + // Split position as a beats timeline position (playback_time is seconds). + let split_time = self.action_executor.document().tempo_map().seconds_to_beats(Seconds(self.playback_time)); let active_layer_id = match self.active_layer_id { Some(id) => id, None => return, // No active layer, nothing to split @@ -2363,7 +2365,7 @@ impl EditorApp { // Helper to find clips that span the playhead in a specific layer fn find_splittable_clips( clip_instances: &[lightningbeam_core::clip::ClipInstance], - split_time: f64, + split_time: Beats, document: &lightningbeam_core::document::Document, ) -> Vec { let mut result = Vec::new(); @@ -2372,9 +2374,9 @@ impl EditorApp { let effective_duration = instance.effective_duration(clip_duration, document.tempo_map()); let timeline_end = instance.timeline_start + effective_duration; - const EPSILON: f64 = 0.001; - if split_time > instance.timeline_start + EPSILON - && split_time < timeline_end - EPSILON + let epsilon = Beats(0.001); + if split_time > instance.timeline_start + epsilon + && split_time < timeline_end - epsilon { result.push(instance.id); } @@ -3051,10 +3053,11 @@ impl EditorApp { let min_start = instances .iter() .map(|i| i.timeline_start) - .fold(f64::INFINITY, f64::min); - let offset = self.playback_time - min_start; + .fold(Beats(f64::INFINITY), |a, b| a.min(b)); + let playhead_beats = self.action_executor.document().tempo_map().seconds_to_beats(Seconds(self.playback_time)); + let offset = playhead_beats - min_start; for inst in &mut instances { - inst.timeline_start = (inst.timeline_start + offset).max(0.0); + inst.timeline_start = (inst.timeline_start + offset).max(Beats::ZERO); } } @@ -3256,7 +3259,7 @@ impl EditorApp { let duplicates: Vec = clips_to_duplicate.iter().map(|original| { let mut duplicate = original.clone(); duplicate.id = uuid::Uuid::new_v4(); - let clip_duration = document.get_clip_duration(&original.clip_id).unwrap_or(1.0); + let clip_duration = document.get_clip_duration(&original.clip_id).unwrap_or(Seconds(1.0)); let effective_duration = original.effective_duration(clip_duration, document.tempo_map()); duplicate.timeline_start = original.timeline_start + effective_duration; if let Some((new_clip_def_id, _)) = midi_clip_replacements.get(&original.clip_id) { @@ -5553,6 +5556,7 @@ impl EditorApp { use lightningbeam_core::layer::*; let drop_time = self.playback_time; + let drop_beats = self.action_executor.document().tempo_map().seconds_to_beats(Seconds(drop_time)); // Find or create a compatible layer let document = self.action_executor.document(); @@ -5645,7 +5649,7 @@ impl EditorApp { } else { // For clips, create a clip instance let mut clip_instance = ClipInstance::new(asset_info.clip_id) - .with_timeline_start(drop_time); + .with_timeline_start(drop_beats); // For video clips, scale to fit and center in document if asset_info.clip_type == panes::DragClipType::Video { @@ -5706,7 +5710,7 @@ impl EditorApp { // Create audio clip instance at same timeline position let audio_instance = ClipInstance::new(linked_audio_clip_id) - .with_timeline_start(drop_time); + .with_timeline_start(drop_beats); let audio_instance_id = audio_instance.id; // Execute audio action with backend sync @@ -5748,7 +5752,7 @@ impl EditorApp { // Find the video clip instance in the document let document = self.action_executor.document(); - let mut video_instance_info: Option<(uuid::Uuid, f64, bool)> = None; // (layer_id, timeline_start, already_in_group) + let mut video_instance_info: Option<(uuid::Uuid, Beats, bool)> = None; // (layer_id, timeline_start [beats], already_in_group) // Search root layers for a video clip instance with matching clip_id for layer in &document.root.children { @@ -6418,9 +6422,10 @@ impl eframe::App for EditorApp { let clip = AudioClip::new_recording("Recording..."); let doc_clip_id = self.action_executor.document_mut().add_audio_clip(clip); - // Create clip instance on the layer + // Create clip instance on the layer (recording_start_time is seconds) + let rec_start_beats = self.action_executor.document().tempo_map().seconds_to_beats(Seconds(self.recording_start_time)); let clip_instance = ClipInstance::new(doc_clip_id) - .with_timeline_start(self.recording_start_time); + .with_timeline_start(rec_start_beats); let clip_instance_id = clip_instance.id; @@ -6537,7 +6542,7 @@ impl eframe::App for EditorApp { None } }) - .unwrap_or((uuid::Uuid::nil(), uuid::Uuid::nil(), 0.0, 0.0)) + .unwrap_or((uuid::Uuid::nil(), uuid::Uuid::nil(), Beats::ZERO, 0.0)) }; if !clip_id.is_nil() { @@ -7494,9 +7499,13 @@ impl eframe::App for EditorApp { let duration = clip.duration; self.action_executor.document_mut().video_clips.insert(clip_id, clip); + // recording_start_time and duration are seconds; convert to beats. + let tempo_map = self.action_executor.document().tempo_map(); + let rec_start_beats = tempo_map.seconds_to_beats(Seconds(self.recording_start_time)); + let dur_beats = tempo_map.seconds_to_beats(tempo_map.beats_to_seconds(rec_start_beats) + Seconds(duration)) - rec_start_beats; let mut clip_instance = ClipInstance::new(clip_id) - .with_timeline_start(self.recording_start_time) - .with_timeline_duration(duration); + .with_timeline_start(rec_start_beats) + .with_timeline_duration(dur_beats); // Scale to fit document and center (like drag-dropped videos) { diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs index f776dcf..d29c26b 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs @@ -1609,15 +1609,17 @@ impl InfopanelPane { ui.horizontal(|ui| { ui.label("Start:"); - ui.label(format!("{:.2}s", ci.effective_start())); + ui.label(format!("{:.2}s", document.tempo_map().beats_to_seconds(ci.effective_start()).seconds_to_f64())); }); let clip_dur = document.get_clip_duration(&ci.clip_id) - .unwrap_or_else(|| ci.trim_end.unwrap_or(1.0) - ci.trim_start); + .unwrap_or_else(|| daw_backend::Seconds(ci.trim_end.unwrap_or(1.0) - ci.trim_start)); let total_dur = ci.total_duration(clip_dur, document.tempo_map()); + let total_dur_secs = (document.tempo_map().beats_to_seconds(ci.effective_start() + total_dur) + - document.tempo_map().beats_to_seconds(ci.effective_start())).seconds_to_f64(); ui.horizontal(|ui| { ui.label("Duration:"); - ui.label(format!("{:.2}s", total_dur)); + ui.label(format!("{:.2}s", total_dur_secs)); }); if ci.trim_start > 0.0 { diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs index ad47424..c1e24ad 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs @@ -5,6 +5,7 @@ /// When a sampled audio layer is selected, shows a GPU-rendered spectrogram. use eframe::egui; +use daw_backend::Seconds; use egui::{pos2, vec2, Align2, Color32, FontId, Rect, Stroke, StrokeKind}; use std::collections::{HashMap, HashSet}; use uuid::Uuid; @@ -465,8 +466,8 @@ impl PianoRollPane { for instance in &audio_layer.clip_instances { if let Some(clip) = document.audio_clips.get(&instance.clip_id) { if let AudioClipType::Midi { midi_clip_id } = clip.clip_type { - let duration = instance.effective_duration(clip.duration, document.tempo_map()); - clip_data.push((midi_clip_id, instance.timeline_start, instance.trim_start, duration, instance.id)); + let duration = instance.effective_duration(Seconds(clip.duration), document.tempo_map()); + clip_data.push((midi_clip_id, instance.timeline_start.beats_to_f64(), instance.trim_start, duration.beats_to_f64(), instance.id)); } } } @@ -2454,10 +2455,15 @@ impl PianoRollPane { for instance in &audio_layer.clip_instances { if let Some(clip) = document.audio_clips.get(&instance.clip_id) { if let AudioClipType::Sampled { audio_pool_index } = clip.clip_type { - let duration = instance.timeline_duration.unwrap_or(clip.duration); + // Duration in beats: explicit timeline_duration, else the clip's content + // length converted to beats at the clip's start. + let duration = instance.timeline_duration.unwrap_or_else(|| { + let tmap = document.tempo_map(); + tmap.seconds_to_beats(tmap.beats_to_seconds(instance.timeline_start) + Seconds(clip.duration)) - instance.timeline_start + }); // Get sample rate from raw_audio_cache if let Some((_samples, sr, _ch)) = shared.raw_audio_cache.get(&audio_pool_index) { - clip_infos.push((audio_pool_index, instance.timeline_start, instance.trim_start, duration, *sr)); + clip_infos.push((audio_pool_index, instance.timeline_start.beats_to_f64(), instance.trim_start, duration.beats_to_f64(), *sr)); } } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 5c2f15b..73dd94d 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -4,6 +4,7 @@ /// Supports HDR compositing pipeline with per-layer buffers and effects. use eframe::egui; +use daw_backend::Seconds; use lightningbeam_core::action::Action; use lightningbeam_core::clip::ClipInstance; use lightningbeam_core::gpu::{BufferPool, BufferFormat, BufferSpec, Compositor, EffectProcessor, SrgbToLinearConverter}; @@ -1851,10 +1852,13 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // Create EffectInstance from ClipInstance for the processor // For now, create a simple effect instance with default parameters + let tempo_map = self.ctx.document.tempo_map(); + let effect_end_beats = effect_instance.timeline_start + + effect_instance.effective_duration(Seconds(lightningbeam_core::effect::EFFECT_DURATION), tempo_map); let effect_inst = lightningbeam_core::effect::EffectInstance::new( effect_def, - effect_instance.timeline_start, - effect_instance.timeline_start + effect_instance.effective_duration(lightningbeam_core::effect::EFFECT_DURATION, self.ctx.document.tempo_map()), + tempo_map.beats_to_seconds(effect_instance.timeline_start).seconds_to_f64(), + tempo_map.beats_to_seconds(effect_end_beats).seconds_to_f64(), ); // Acquire temp buffer for effect output (HDR format) @@ -2204,7 +2208,8 @@ impl egui_wgpu::CallbackTrait for VelloCallback { let combined_transform = overlay_transform * clip_transform; // Calculate clip bounds for preview - let clip_time = ((self.ctx.playback_time - clip_inst.timeline_start) * clip_inst.playback_speed) + clip_inst.trim_start; + let start_secs = self.ctx.document.tempo_map().beats_to_seconds(clip_inst.timeline_start).seconds_to_f64(); + let clip_time = ((self.ctx.playback_time - start_secs) * clip_inst.playback_speed) + clip_inst.trim_start; let content_bounds = if let Some(vector_clip) = self.ctx.document.get_vector_clip(&clip_inst.clip_id) { vector_clip.calculate_content_bounds(&self.ctx.document, clip_time) } else if let Some(video_clip) = self.ctx.document.get_video_clip(&clip_inst.clip_id) { @@ -2293,15 +2298,19 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // Also draw selection outlines for clip instances for &clip_id in self.ctx.selection.clip_instances() { if let Some(clip_instance) = vector_layer.clip_instances.iter().find(|ci| ci.id == clip_id) { - // Skip clip instances not active at current time - let clip_dur = self.ctx.document.get_clip_duration(&clip_instance.clip_id).unwrap_or(0.0); - let instance_end = clip_instance.timeline_start + clip_instance.effective_duration(clip_dur, self.ctx.document.tempo_map()); - if self.ctx.playback_time < clip_instance.timeline_start || self.ctx.playback_time >= instance_end { + // Skip clip instances not active at current time (compare in seconds). + let clip_dur = self.ctx.document.get_clip_duration(&clip_instance.clip_id).unwrap_or(Seconds::ZERO); + let tempo_map = self.ctx.document.tempo_map(); + let start_secs = tempo_map.beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); + let instance_end = tempo_map.beats_to_seconds( + clip_instance.timeline_start + clip_instance.effective_duration(clip_dur, tempo_map) + ).seconds_to_f64(); + if self.ctx.playback_time < start_secs || self.ctx.playback_time >= instance_end { continue; } // Calculate clip-local time - let clip_time = ((self.ctx.playback_time - clip_instance.timeline_start) * clip_instance.playback_speed) + clip_instance.trim_start; + let clip_time = ((self.ctx.playback_time - start_secs) * clip_instance.playback_speed) + clip_instance.trim_start; // Get dynamic clip bounds from content at current time let bbox = if let Some(vector_clip) = self.ctx.document.get_vector_clip(&clip_instance.clip_id) { @@ -2671,9 +2680,11 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // Find clip instance visible at playback time let visible_clip = video_layer.clip_instances.iter().find(|inst| { - let clip_duration = self.ctx.document.get_clip_duration(&inst.clip_id).unwrap_or(0.0); - let effective_duration = inst.effective_duration(clip_duration, self.ctx.document.tempo_map()); - playback_time >= inst.timeline_start && playback_time < inst.timeline_start + effective_duration + let clip_duration = self.ctx.document.get_clip_duration(&inst.clip_id).unwrap_or(Seconds::ZERO); + let tempo_map = self.ctx.document.tempo_map(); + let start_secs = tempo_map.beats_to_seconds(inst.timeline_start).seconds_to_f64(); + let end_secs = tempo_map.beats_to_seconds(inst.timeline_start + inst.effective_duration(clip_duration, tempo_map)).seconds_to_f64(); + playback_time >= start_secs && playback_time < end_secs }); if let Some(clip_inst) = visible_clip { @@ -10130,7 +10141,8 @@ impl StagePane { for &clip_id in shared.selection.clip_instances() { if let Some(clip_instance) = vector_layer.clip_instances.iter().find(|ci| ci.id == clip_id) { // Calculate clip-local time - let clip_time = ((*shared.playback_time - clip_instance.timeline_start) * clip_instance.playback_speed) + clip_instance.trim_start; + let start_secs = shared.action_executor.document().tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); + let clip_time = ((*shared.playback_time - start_secs) * clip_instance.playback_speed) + clip_instance.trim_start; // Get dynamic clip bounds from content at current time use vello::kurbo::Rect as KurboRect; @@ -10330,7 +10342,8 @@ impl StagePane { // Try clip instance if let Some(clip_instance) = vector_layer.clip_instances.iter().find(|ci| ci.id == object_id) { // Calculate clip-local time - let clip_time = ((*shared.playback_time - clip_instance.timeline_start) * clip_instance.playback_speed) + clip_instance.trim_start; + let start_secs = shared.action_executor.document().tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); + let clip_time = ((*shared.playback_time - start_secs) * clip_instance.playback_speed) + clip_instance.trim_start; // Get dynamic clip bounds from content at current time let local_bbox = if let Some(vector_clip) = shared.action_executor.document().get_vector_clip(&clip_instance.clip_id) { @@ -11046,9 +11059,11 @@ impl StagePane { let document = shared.action_executor.document(); if let Some(AnyLayer::Video(video_layer)) = document.get_layer(layer_id) { video_layer.clip_instances.iter().find(|inst| { - let clip_duration = document.get_clip_duration(&inst.clip_id).unwrap_or(0.0); - let effective_duration = inst.effective_duration(clip_duration, document.tempo_map()); - playback_time >= inst.timeline_start && playback_time < inst.timeline_start + effective_duration + let clip_duration = document.get_clip_duration(&inst.clip_id).unwrap_or(Seconds::ZERO); + let tempo_map = document.tempo_map(); + let start_secs = tempo_map.beats_to_seconds(inst.timeline_start).seconds_to_f64(); + let end_secs = tempo_map.beats_to_seconds(inst.timeline_start + inst.effective_duration(clip_duration, tempo_map)).seconds_to_f64(); + playback_time >= start_secs && playback_time < end_secs }).map(|inst| inst.id) } else { None @@ -12487,8 +12502,14 @@ impl PaneRenderer for StagePane { let canvas_pos = pointer_pos - rect.min; let world_pos = (canvas_pos - self.pan_offset) / self.zoom; - // Use playhead time + // Use playhead time (seconds); the beats placement position for clips. let drop_time = *shared.playback_time; + let drop_beats = shared.action_executor.document().tempo_map().seconds_to_beats(Seconds(drop_time)); + // 5-second default effect duration as a beats span at the drop point. + let effect_dur_beats = { + let tmap = shared.action_executor.document().tempo_map(); + tmap.seconds_to_beats(tmap.beats_to_seconds(drop_beats) + Seconds(5.0)) - drop_beats + }; // Find or create a compatible layer let document = shared.action_executor.document(); @@ -12559,8 +12580,8 @@ impl PaneRenderer for StagePane { // Create clip instance for effect with 5 second default duration let clip_instance = ClipInstance::new(def.id) - .with_timeline_start(drop_time) - .with_timeline_duration(5.0); + .with_timeline_start(drop_beats) + .with_timeline_duration(effect_dur_beats); // Use AddEffectAction for effect layers let action = lightningbeam_core::actions::AddEffectAction::new( @@ -12572,7 +12593,7 @@ impl PaneRenderer for StagePane { } else { // For clips, create a clip instance let mut clip_instance = ClipInstance::new(dragging.clip_id) - .with_timeline_start(drop_time); + .with_timeline_start(drop_beats); // For video clips, scale to fit and center in document if dragging.clip_type == DragClipType::Video { @@ -12642,7 +12663,7 @@ impl PaneRenderer for StagePane { // Create audio clip instance at same timeline position let audio_instance = ClipInstance::new(linked_audio_clip_id) - .with_timeline_start(drop_time); + .with_timeline_start(drop_beats); let audio_instance_id = audio_instance.id; eprintln!("DEBUG STAGE: Created audio instance: {} for clip: {}", audio_instance_id, linked_audio_clip_id); diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index 793835b..b67ffe2 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -7,6 +7,7 @@ /// - Basic layer visualization use eframe::egui; +use daw_backend::{Beats, Seconds}; use lightningbeam_core::clip::{ ClipInstance, audio_backend_uuid, midi_backend_uuid, }; @@ -77,11 +78,12 @@ fn compute_clip_stacking( } let tempo_map = document.tempo_map(); + // Stacking only needs relative overlap, so compare in the beats domain. let ranges: Vec<(f64, f64)> = clip_instances.iter().map(|ci| { - let clip_dur = effective_clip_duration(document, layer, ci).unwrap_or(0.0); + let clip_dur = effective_clip_duration(document, layer, ci).unwrap_or(Seconds::ZERO); let start = ci.effective_start(); let end = start + ci.total_duration(clip_dur, tempo_map); - (start, end) + (start.beats_to_f64(), end.beats_to_f64()) }).collect(); compute_clip_stacking_from_ranges(&ranges) @@ -196,22 +198,23 @@ fn effective_clip_duration( document: &lightningbeam_core::document::Document, layer: &AnyLayer, clip_instance: &ClipInstance, -) -> Option { +) -> Option { match layer { AnyLayer::Vector(vl) => { let vc = document.get_vector_clip(&clip_instance.clip_id)?; if vc.is_group { let frame_duration = 1.0 / document.framerate; - let end = vl.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration); - Some((end - clip_instance.timeline_start).max(0.0)) + let start_secs = document.tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); + let end = vl.group_visibility_end(&clip_instance.id, start_secs, frame_duration); + Some(Seconds((end - start_secs).max(0.0))) } else { // Movie clips: duration based on all internal content (keyframes + clip instances) document.get_clip_duration(&clip_instance.clip_id) } } - AnyLayer::Audio(_) => document.get_audio_clip(&clip_instance.clip_id).map(|c| c.duration), - AnyLayer::Video(_) => document.get_video_clip(&clip_instance.clip_id).map(|c| c.duration), - AnyLayer::Effect(_) => Some(lightningbeam_core::effect::EFFECT_DURATION), + AnyLayer::Audio(_) => document.get_audio_clip(&clip_instance.clip_id).map(|c| Seconds(c.duration)), + AnyLayer::Video(_) => document.get_video_clip(&clip_instance.clip_id).map(|c| Seconds(c.duration)), + AnyLayer::Effect(_) => Some(Seconds(lightningbeam_core::effect::EFFECT_DURATION)), AnyLayer::Group(_) => None, AnyLayer::Raster(_) => None, AnyLayer::Text(_) => None, @@ -626,14 +629,14 @@ fn build_audio_clip_cache( .unwrap_or_else(|| audio_backend_uuid(ac.id)); let mut ci = ClipInstance::new(clip_id); ci.id = instance_id; - ci.timeline_start = ac.external_start.beats_to_f64(); + ci.timeline_start = ac.external_start; ci.trim_start = ac.internal_start.seconds_to_f64(); ci.trim_end = Some(ac.internal_end.seconds_to_f64()); let internal_dur_secs = (ac.internal_end - ac.internal_start).seconds_to_f64(); let tempo_map = document.tempo_map(); let external_dur_secs = tempo_map.transform(ac.external_duration.beats_to_f64()); if (external_dur_secs - internal_dur_secs).abs() > 1e-9 { - ci.timeline_duration = Some(ac.external_duration.beats_to_f64()); + ci.timeline_duration = Some(ac.external_duration); } ci.gain = ac.gain; instances.push(ci); @@ -650,12 +653,12 @@ fn build_audio_clip_cache( .unwrap_or_else(|| midi_backend_uuid(mc.id)); let mut ci = ClipInstance::new(clip_id); ci.id = instance_id; - ci.timeline_start = mc.external_start.beats_to_f64(); + ci.timeline_start = mc.external_start; ci.trim_start = mc.internal_start.beats_to_f64(); ci.trim_end = Some(mc.internal_end.beats_to_f64()); // Always set timeline_duration for MIDI clips: duration is in beats, so we // must bypass the content_window_secs * bpm/60 formula (which expects seconds). - ci.timeline_duration = Some(mc.external_duration.beats_to_f64()); + ci.timeline_duration = Some(mc.external_duration); instances.push(ci); } } @@ -734,12 +737,13 @@ fn collect_clip_instances<'a>( fn find_sampled_audio_track_for_clip( document: &lightningbeam_core::document::Document, clip_id: uuid::Uuid, - timeline_start: f64, + timeline_start: Beats, editing_clip_id: Option<&uuid::Uuid>, ) -> Option { - // Get the clip duration + // Get the clip duration (content seconds) and convert its span to beats at the drop point. let clip_duration = document.get_clip_duration(&clip_id)?; - let clip_end = timeline_start + clip_duration; + let tempo_map = document.tempo_map(); + let clip_end = tempo_map.seconds_to_beats(tempo_map.beats_to_seconds(timeline_start) + clip_duration); // Check each sampled audio layer let context_layers = document.context_layers(editing_clip_id); @@ -1170,8 +1174,10 @@ impl TimelinePane { *shared.recording_clips.get(&layer_id).unwrap_or(&0), daw_backend::Beats::ZERO); let doc_clip_id = shared.action_executor.document_mut().add_audio_clip(doc_clip); + let start_beats = shared.action_executor.document().tempo_map() + .seconds_to_beats(Seconds(start_time)); let clip_instance = ClipInstance::new(doc_clip_id) - .with_timeline_start(start_time); + .with_timeline_start(start_beats); if let Some(layer) = shared.action_executor.document_mut().get_layer_mut(&layer_id) { if let lightningbeam_core::layer::AnyLayer::Audio(audio_layer) = layer { @@ -1347,8 +1353,8 @@ impl TimelinePane { let instance_duration = clip_instance.total_duration(clip_duration, tempo_map); let instance_end = instance_start + instance_duration; - let start_x = self.time_to_x(instance_start); - let end_x = self.time_to_x(instance_end).max(start_x + MIN_CLIP_WIDTH_PX); + let start_x = self.beats_to_x(instance_start, tempo_map); + let end_x = self.beats_to_x(instance_end, tempo_map).max(start_x + MIN_CLIP_WIDTH_PX); let mouse_x = pointer_pos.x - content_rect.min.x; if mouse_x >= start_x && mouse_x <= end_x { @@ -1422,12 +1428,12 @@ impl TimelinePane { // Compute merged spans with the child clip IDs that contribute to each let child_clips = group.all_child_clip_instances(); - let mut spans: Vec<(f64, f64, Vec)> = Vec::new(); // (start, end, clip_ids) + let mut spans: Vec<(Beats, Beats, Vec)> = Vec::new(); // (start, end, clip_ids) in beats let tempo_map = document.tempo_map(); for (_child_layer_id, ci) in &child_clips { let clip_dur = document.get_clip_duration(&ci.clip_id).unwrap_or_else(|| { - ci.trim_end.unwrap_or(1.0) - ci.trim_start + Seconds(ci.trim_end.unwrap_or(1.0) - ci.trim_start) }); let start = ci.effective_start(); let end = start + ci.total_duration(clip_dur, tempo_map); @@ -1437,7 +1443,7 @@ impl TimelinePane { spans.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); // Merge overlapping spans - let mut merged: Vec<(f64, f64, Vec)> = Vec::new(); + let mut merged: Vec<(Beats, Beats, Vec)> = Vec::new(); for (s, e, ids) in spans { if let Some(last) = merged.last_mut() { if s <= last.1 { @@ -1454,8 +1460,8 @@ impl TimelinePane { // Check which merged span the pointer is over let mouse_x = pointer_pos.x - content_rect.min.x; for (s, e, ids) in merged { - let sx = self.time_to_x(s); - let ex = self.time_to_x(e).max(sx + MIN_CLIP_WIDTH_PX); + let sx = self.beats_to_x(s, tempo_map); + let ex = self.beats_to_x(e, tempo_map).max(sx + MIN_CLIP_WIDTH_PX); if mouse_x >= sx && mouse_x <= ex { return Some(ids); } @@ -1517,24 +1523,30 @@ impl TimelinePane { ((time - self.viewport_start_time) * self.pixels_per_second as f64) as f32 } + /// Convert a beats timeline position to pixel x-coordinate (via seconds). + fn beats_to_x(&self, beats: Beats, tempo_map: &daw_backend::TempoMap) -> f32 { + self.time_to_x(tempo_map.beats_to_seconds(beats).seconds_to_f64()) + } + /// Effective display start for a clip instance, in seconds. /// /// `timeline_start` is in beats; converts to seconds using the current (preview) BPM so /// clips stay anchored to their beat position during live BPM drag. fn instance_display_start(&self, ci: &lightningbeam_core::clip::ClipInstance, tempo_map: &daw_backend::TempoMap) -> f64 { - tempo_map.transform(ci.effective_start()) + tempo_map.beats_to_seconds(ci.effective_start()).seconds_to_f64() } /// Effective on-timeline duration for a clip instance, in seconds. /// /// `total_duration` is in beats; converts to seconds using the current (preview) BPM. - fn instance_display_duration(&self, ci: &lightningbeam_core::clip::ClipInstance, clip_dur_secs: f64, tempo_map: &daw_backend::TempoMap) -> f64 { - tempo_map.transform(ci.timeline_start + ci.total_duration(clip_dur_secs, tempo_map)) - tempo_map.transform(ci.effective_start()) + fn instance_display_duration(&self, ci: &lightningbeam_core::clip::ClipInstance, clip_dur_secs: Seconds, tempo_map: &daw_backend::TempoMap) -> f64 { + (tempo_map.beats_to_seconds(ci.timeline_start + ci.total_duration(clip_dur_secs, tempo_map)) + - tempo_map.beats_to_seconds(ci.effective_start())).seconds_to_f64() } /// Returns the clip content start (trim_start) and duration in display seconds. - fn content_display_range(&self, ci: &lightningbeam_core::clip::ClipInstance, clip_dur_secs: f64, _bpm: f64) -> (f64, f64) { - let trim_end = ci.trim_end.unwrap_or(clip_dur_secs); + fn content_display_range(&self, ci: &lightningbeam_core::clip::ClipInstance, clip_dur_secs: Seconds, _bpm: f64) -> (f64, f64) { + let trim_end = ci.trim_end.unwrap_or(clip_dur_secs.seconds_to_f64()); (ci.trim_start, (trim_end - ci.trim_start).max(0.0)) } @@ -1593,21 +1605,33 @@ impl TimelinePane { } } - /// Effective drag offset for Move operations. - /// Snaps the anchor clip's resulting position to the grid; all selected clips use the same offset. + /// Effective drag offset for Move operations, as a uniform beats delta. + /// Snapping happens in the seconds/pixel domain (where the grid and `drag_anchor_start`/ + /// `drag_offset` live); the result is the anchor's net snapped movement expressed in beats, + /// so every selected clip shifts by the same beats amount (preserving beat-relative spacing). fn snapped_move_offset( &self, tempo_map: &daw_backend::TempoMap, time_sig: &lightningbeam_core::document::TimeSignature, framerate: f64, - ) -> f64 { - match self.quantize_grid_size(tempo_map, time_sig, framerate) { - Some(grid) => { - let snapped = ((self.drag_anchor_start + self.drag_offset) / grid).round() * grid; - snapped - self.drag_anchor_start - } - None => self.drag_offset, - } + ) -> Beats { + let anchor = self.drag_anchor_start; // seconds + let target = match self.quantize_grid_size(tempo_map, time_sig, framerate) { + Some(grid) => ((anchor + self.drag_offset) / grid).round() * grid, + None => anchor + self.drag_offset, + }; + tempo_map.seconds_to_beats(Seconds(target)) - tempo_map.seconds_to_beats(Seconds(anchor)) + } + + /// Apply the current snapped move-drag offset to a clip's beats start, clamped to ≥ 0. + fn moved_start( + &self, + start: Beats, + tempo_map: &daw_backend::TempoMap, + time_sig: &lightningbeam_core::document::TimeSignature, + framerate: f64, + ) -> Beats { + (start + self.snapped_move_offset(tempo_map, time_sig, framerate)).max(Beats::ZERO) } /// Calculate appropriate interval for time ruler based on zoom level @@ -2995,23 +3019,23 @@ impl TimelinePane { // Collect all child clip time ranges (with drag preview offset) let child_clips = g.all_child_clip_instances(); let is_move_drag = self.clip_drag_state == Some(ClipDragType::Move); - let mut ranges: Vec<(f64, f64)> = Vec::new(); + let mut ranges: Vec<(Beats, Beats)> = Vec::new(); for (_child_layer_id, ci) in &child_clips { let clip_dur = document.get_clip_duration(&ci.clip_id).unwrap_or_else(|| { - ci.trim_end.unwrap_or(1.0) - ci.trim_start + Seconds(ci.trim_end.unwrap_or(1.0) - ci.trim_start) }); let mut start = ci.effective_start(); let dur = ci.total_duration(clip_dur, document.tempo_map()); // Apply drag offset for selected clips during move if is_move_drag && selection.contains_clip_instance(&ci.id) { - start = (start + self.snapped_move_offset(document.tempo_map(), &document.time_signature, document.framerate)).max(0.0); + start = self.moved_start(start, document.tempo_map(), &document.time_signature, document.framerate); } ranges.push((start, start + dur)); } // Sort and merge overlapping ranges ranges.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); - let mut merged: Vec<(f64, f64)> = Vec::new(); + let mut merged: Vec<(Beats, Beats)> = Vec::new(); for (s, e) in ranges { if let Some(last) = merged.last_mut() { if s <= last.1 { @@ -3038,9 +3062,9 @@ impl TimelinePane { theme.text_color(&["#timeline", ".group-bar"], ui.ctx(), egui::Color32::from_rgb(100, 220, 220)) }; for (s, e) in &merged { - // `merged` ranges are in beats; convert to seconds for time_to_x. - let sx = self.time_to_x(document.tempo_map().transform(*s)); - let ex = self.time_to_x(document.tempo_map().transform(*e)).max(sx + MIN_CLIP_WIDTH_PX); + // `merged` ranges are in beats. + let sx = self.beats_to_x(*s, document.tempo_map()); + let ex = self.beats_to_x(*e, document.tempo_map()).max(sx + MIN_CLIP_WIDTH_PX); if ex >= 0.0 && sx <= rect.width() { let vsx = sx.max(0.0); let vex = ex.min(rect.width()); @@ -3072,17 +3096,17 @@ impl TimelinePane { if let AnyLayer::Video(vl) = video_child { for ci in &vl.clip_instances { let clip_dur = document.get_clip_duration(&ci.clip_id) - .unwrap_or_else(|| ci.trim_end.unwrap_or(1.0) - ci.trim_start); + .unwrap_or_else(|| Seconds(ci.trim_end.unwrap_or(1.0) - ci.trim_start)); let mut ci_start = ci.effective_start(); if is_move_drag && selection.contains_clip_instance(&ci.id) { - ci_start = (ci_start + self.snapped_move_offset(document.tempo_map(), &document.time_signature, document.framerate)).max(0.0); + ci_start = self.moved_start(ci_start, document.tempo_map(), &document.time_signature, document.framerate); } let ci_duration = ci.total_duration(clip_dur, document.tempo_map()); let ci_end = ci_start + ci_duration; - // ci_start/ci_end are in beats; convert to seconds for time_to_x. - let sx = self.time_to_x(document.tempo_map().transform(ci_start)); - let ex = self.time_to_x(document.tempo_map().transform(ci_end)); + // ci_start/ci_end are in beats. + let sx = self.beats_to_x(ci_start, document.tempo_map()); + let ex = self.beats_to_x(ci_end, document.tempo_map()); if ex < 0.0 || sx > rect.width() { continue; } let ci_rect = egui::Rect::from_min_max( @@ -3161,16 +3185,16 @@ impl TimelinePane { }; let audio_file_duration = total_frames as f64 / eff_sr as f64; - let clip_dur = audio_clip.duration; + let clip_dur = Seconds(audio_clip.duration); let mut ci_start = ci.effective_start(); if is_move_drag && selection.contains_clip_instance(&ci.id) { - ci_start = (ci_start + self.snapped_move_offset(document.tempo_map(), &document.time_signature, document.framerate)).max(0.0); + ci_start = self.moved_start(ci_start, document.tempo_map(), &document.time_signature, document.framerate); } let ci_duration = ci.total_duration(clip_dur, document.tempo_map()); - // ci_start/ci_duration are in beats; convert to seconds for time_to_x. - let ci_screen_start = rect.min.x + self.time_to_x(document.tempo_map().transform(ci_start)); - let ci_screen_end = rect.min.x + self.time_to_x(document.tempo_map().transform(ci_start + ci_duration)); + // ci_start/ci_duration are in beats. + let ci_screen_start = rect.min.x + self.beats_to_x(ci_start, document.tempo_map()); + let ci_screen_end = rect.min.x + self.beats_to_x(ci_start + ci_duration, document.tempo_map()); let waveform_rect = egui::Rect::from_min_max( egui::pos2(ci_screen_start.max(rect.min.x), wave_y_min), @@ -3292,7 +3316,7 @@ impl TimelinePane { // For moves, precompute the clamped offset so all selected clips move uniformly let group_move_offset = if self.clip_drag_state == Some(ClipDragType::Move) { - let group: Vec<(uuid::Uuid, f64, f64)> = clip_instances.iter() + let group: Vec<(uuid::Uuid, Beats, Beats)> = clip_instances.iter() .filter(|ci| selection.contains_clip_instance(&ci.id)) .filter_map(|ci| { let dur = document.get_clip_duration(&ci.clip_id)?; @@ -3311,9 +3335,18 @@ impl TimelinePane { // Compute stacking using preview positions (with drag offsets) for vector layers let clip_stacking = if matches!(layer, AnyLayer::Vector(_)) && clip_instances.len() > 1 { let preview_ranges: Vec<(f64, f64)> = clip_instances.iter().map(|ci| { - let clip_dur = effective_clip_duration(document, layer, ci).unwrap_or(0.0); + let tmap = document.tempo_map(); + // Beats span occupied by `secs` of content starting at beats position `anchor`. + let secs_to_beats_at = |anchor: Beats, secs: f64| + tmap.seconds_to_beats(tmap.beats_to_seconds(anchor) + Seconds(secs)) - anchor; + // Beats position `secs` seconds (wall-clock) after beats position `anchor`. + let shift_beats = |anchor: Beats, secs: f64| + tmap.seconds_to_beats(tmap.beats_to_seconds(anchor) + Seconds(secs)); + + let clip_dur = effective_clip_duration(document, layer, ci).unwrap_or(Seconds::ZERO); + let clip_dur_secs = clip_dur.seconds_to_f64(); let mut start = ci.effective_start(); - let mut duration = ci.total_duration(clip_dur, document.tempo_map()); + let mut duration = ci.total_duration(clip_dur, tmap); let is_selected = selection.contains_clip_instance(&ci.id); let is_linked = if self.clip_drag_state.is_some() { @@ -3329,47 +3362,57 @@ impl TimelinePane { match drag_type { ClipDragType::Move => { if let Some(offset) = group_move_offset { - start = (ci.effective_start() + offset).max(0.0); + start = (ci.effective_start() + offset).max(Beats::ZERO); } } ClipDragType::TrimLeft => { - let new_trim = self.snap_to_grid(ci.trim_start + self.drag_offset, document.tempo_map(), &document.time_signature, document.framerate).max(0.0).min(clip_dur); - let offset = new_trim - ci.trim_start; - start = (ci.timeline_start + offset).max(0.0); - duration = (clip_dur - new_trim).max(0.0); - if let Some(trim_end) = ci.trim_end { - duration = (trim_end - new_trim).max(0.0); - } + let new_trim = self.snap_to_grid(ci.trim_start + self.drag_offset, tmap, &document.time_signature, document.framerate).max(0.0).min(clip_dur_secs); + let trim_offset_secs = new_trim - ci.trim_start; + start = shift_beats(ci.timeline_start, trim_offset_secs).max(Beats::ZERO); + let dur_secs = if let Some(trim_end) = ci.trim_end { + (trim_end - new_trim).max(0.0) + } else { + (clip_dur_secs - new_trim).max(0.0) + }; + duration = secs_to_beats_at(start, dur_secs); } ClipDragType::TrimRight => { - let old_trim_end = ci.trim_end.unwrap_or(clip_dur); - let new_trim_end = self.snap_to_grid(old_trim_end + self.drag_offset, document.tempo_map(), &document.time_signature, document.framerate).max(ci.trim_start).min(clip_dur); - duration = (new_trim_end - ci.trim_start).max(0.0); + let old_trim_end = ci.trim_end.unwrap_or(clip_dur_secs); + let new_trim_end = self.snap_to_grid(old_trim_end + self.drag_offset, tmap, &document.time_signature, document.framerate).max(ci.trim_start).min(clip_dur_secs); + let dur_secs = (new_trim_end - ci.trim_start).max(0.0); + duration = secs_to_beats_at(start, dur_secs); } ClipDragType::LoopExtendRight => { - let trim_end = ci.trim_end.unwrap_or(clip_dur); - let content_window = (trim_end - ci.trim_start).max(0.0); + let trim_end = ci.trim_end.unwrap_or(clip_dur_secs); + let content_window_secs = (trim_end - ci.trim_start).max(0.0); + let content_window = secs_to_beats_at(ci.timeline_start, content_window_secs); let current_right = ci.timeline_duration.unwrap_or(content_window); - let right_edge = ci.timeline_start + current_right + self.drag_offset; - let snapped_edge = self.snap_to_grid(right_edge, document.tempo_map(), &document.time_signature, document.framerate); + let right_edge_secs = tmap.beats_to_seconds(ci.timeline_start + current_right).seconds_to_f64() + self.drag_offset; + let snapped_edge_secs = self.snap_to_grid(right_edge_secs, tmap, &document.time_signature, document.framerate); + let snapped_edge = tmap.seconds_to_beats(Seconds(snapped_edge_secs)); let new_right = (snapped_edge - ci.timeline_start).max(content_window); - let loop_before = ci.loop_before.unwrap_or(0.0); + let loop_before = ci.loop_before.unwrap_or(Beats::ZERO); duration = loop_before + new_right; } ClipDragType::LoopExtendLeft => { - let trim_end = ci.trim_end.unwrap_or(clip_dur); - let content_window = (trim_end - ci.trim_start).max(0.001); - let current_loop_before = ci.loop_before.unwrap_or(0.0); - let desired = (current_loop_before - self.drag_offset).max(0.0); - let snapped = (desired / content_window).round() * content_window; + let trim_end = ci.trim_end.unwrap_or(clip_dur_secs); + let content_window_secs = (trim_end - ci.trim_start).max(0.001); + let content_window = secs_to_beats_at(ci.timeline_start, content_window_secs); + let current_loop_before = ci.loop_before.unwrap_or(Beats::ZERO); + // drag_offset (seconds) as a beats delta at this clip's start. + let drag_beats = shift_beats(ci.timeline_start, self.drag_offset) - ci.timeline_start; + let desired = (current_loop_before - drag_beats).max(Beats::ZERO); + // Snap loop-before to whole content-window multiples. + let cw = content_window.beats_to_f64().max(1e-9); + let snapped = Beats((desired.beats_to_f64() / cw).round() * cw); start = ci.timeline_start - snapped; - duration = snapped + ci.effective_duration(clip_dur, document.tempo_map()); + duration = snapped + ci.effective_duration(clip_dur, tmap); } } } } - (start, start + duration) + (start.beats_to_f64(), (start + duration).beats_to_f64()) }).collect(); compute_clip_stacking_from_ranges(&preview_ranges) } else { @@ -3409,7 +3452,7 @@ impl TimelinePane { // Content origin: where the first "real" content iteration starts // Loop iterations tile outward from this point - let mut content_origin = instance_start + clip_instance.loop_before.unwrap_or(0.0); + let mut content_origin = document.tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64(); // Track preview trim values for note/waveform rendering. // In Measures mode, derive from beats so they track BPM during live drag. @@ -3422,26 +3465,27 @@ impl TimelinePane { match drag_type { ClipDragType::Move => { if let Some(offset) = group_move_offset { - instance_start = (clip_instance.effective_start() + offset).max(0.0); - content_origin = instance_start + clip_instance.loop_before.unwrap_or(0.0); + let tmap = document.tempo_map(); + instance_start = tmap.beats_to_seconds((clip_instance.effective_start() + offset).max(Beats::ZERO)).seconds_to_f64(); + content_origin = tmap.beats_to_seconds(clip_instance.timeline_start + offset).seconds_to_f64(); } } ClipDragType::TrimLeft => { // Trim left: calculate new trim_start with snap to adjacent clips let desired_trim_start = self.snap_to_grid(clip_instance.trim_start + self.drag_offset, document.tempo_map(), &document.time_signature, document.framerate) .max(0.0) - .min(clip_duration); + .min(clip_duration.seconds_to_f64()); let new_trim_start = if desired_trim_start < clip_instance.trim_start { - // Extending left - check for adjacent clips - let max_extend = document.find_max_trim_extend_left( + // Extending left - limit is the content-seconds gap to the previous clip. + let max_extend_secs = document.find_max_trim_extend_left( &layer.id(), &clip_instance.id, clip_instance.effective_start(), - ); + ).seconds_to_f64(); let desired_extend = clip_instance.trim_start - desired_trim_start; - let actual_extend = desired_extend.min(max_extend); + let actual_extend = desired_extend.min(max_extend_secs); clip_instance.trim_start - actual_extend } else { // Shrinking - no snap needed @@ -3450,11 +3494,11 @@ impl TimelinePane { let actual_offset = new_trim_start - clip_instance.trim_start; - // Move start and reduce duration by actual clamped offset - instance_start = (clip_instance.timeline_start + actual_offset) + // Move start (display seconds) and reduce duration by the clamped offset. + instance_start = (document.tempo_map().beats_to_seconds(clip_instance.timeline_start).seconds_to_f64() + actual_offset) .max(0.0); - instance_duration = (clip_duration - new_trim_start).max(0.0); + instance_duration = (clip_duration.seconds_to_f64() - new_trim_start).max(0.0); // Adjust for existing trim_end if let Some(trim_end) = clip_instance.trim_end { @@ -3467,23 +3511,27 @@ impl TimelinePane { } ClipDragType::TrimRight => { // Trim right: extend or reduce duration with snap to adjacent clips - let old_trim_end = clip_instance.trim_end.unwrap_or(clip_duration); + let old_trim_end = clip_instance.trim_end.unwrap_or(clip_duration.seconds_to_f64()); let desired_trim_end = self.snap_to_grid(old_trim_end + self.drag_offset, document.tempo_map(), &document.time_signature, document.framerate) .max(clip_instance.trim_start) - .min(clip_duration); + .min(clip_duration.seconds_to_f64()); let new_trim_end = if desired_trim_end > old_trim_end { - // Extending right - check for adjacent clips - let current_duration = old_trim_end - clip_instance.trim_start; - let max_extend = document.find_max_trim_extend_right( + // Extending right - limit is the content-seconds gap to the next clip. + let current_duration_secs = old_trim_end - clip_instance.trim_start; + let tmap = document.tempo_map(); + let current_duration = tmap.seconds_to_beats( + tmap.beats_to_seconds(clip_instance.timeline_start) + Seconds(current_duration_secs) + ) - clip_instance.timeline_start; + let max_extend_secs = document.find_max_trim_extend_right( &layer.id(), &clip_instance.id, clip_instance.timeline_start, current_duration, - ); + ).seconds_to_f64(); let desired_extend = desired_trim_end - old_trim_end; - let actual_extend = desired_extend.min(max_extend); + let actual_extend = desired_extend.min(max_extend_secs); old_trim_end + actual_extend } else { // Shrinking - no snap needed @@ -3498,41 +3546,56 @@ impl TimelinePane { } ClipDragType::LoopExtendRight => { // Loop extend right: extend clip beyond content window - let trim_end = clip_instance.trim_end.unwrap_or(clip_duration); - let content_window = (trim_end - clip_instance.trim_start).max(0.0); + let trim_end = clip_instance.trim_end.unwrap_or(clip_duration.seconds_to_f64()); + let content_window_secs = (trim_end - clip_instance.trim_start).max(0.0); + let tmap = document.tempo_map(); + let ts = clip_instance.timeline_start; + // content window and right-duration are beats-domain timeline spans. + let content_window = tmap.seconds_to_beats(tmap.beats_to_seconds(ts) + Seconds(content_window_secs)) - ts; let current_right = clip_instance.timeline_duration.unwrap_or(content_window); - let right_edge = clip_instance.timeline_start + current_right + self.drag_offset; - let snapped_edge = self.snap_to_grid(right_edge, document.tempo_map(), &document.time_signature, document.framerate); - let desired_right = (snapped_edge - clip_instance.timeline_start).max(content_window); + // Snap the right edge in the seconds/pixel domain (drag_offset is seconds). + let right_edge_secs = tmap.beats_to_seconds(ts + current_right).seconds_to_f64() + self.drag_offset; + let snapped_edge_secs = self.snap_to_grid(right_edge_secs, tmap, &document.time_signature, document.framerate); + let snapped_edge = tmap.seconds_to_beats(Seconds(snapped_edge_secs)); + let desired_right = (snapped_edge - ts).max(content_window); let new_right = if desired_right > current_right { - let max_extend = document.find_max_trim_extend_right( + // Gap to the next clip comes back as content seconds; convert to a + // beats span at the clip's right edge for the loop-length limit. + let max_extend_secs = document.find_max_trim_extend_right( &layer.id(), &clip_instance.id, - clip_instance.timeline_start, + ts, current_right, ); + let right_edge = ts + current_right; + let max_extend = tmap.seconds_to_beats(tmap.beats_to_seconds(right_edge) + max_extend_secs) - right_edge; let extend_amount = (desired_right - current_right).min(max_extend); current_right + extend_amount } else { desired_right }; - // Total duration = loop_before + right duration - let loop_before = clip_instance.loop_before.unwrap_or(0.0); - instance_duration = loop_before + new_right; + // Right edge lands at ts + new_right (beats); duration in display seconds. + instance_duration = tmap.beats_to_seconds(ts + new_right).seconds_to_f64() - instance_start; } ClipDragType::LoopExtendLeft => { // Loop extend left: extend loop_before (pre-loop region) // Snap to multiples of content_window so iterations align with backend - let trim_end = clip_instance.trim_end.unwrap_or(clip_duration); - let content_window = (trim_end - clip_instance.trim_start).max(0.001); - let current_loop_before = clip_instance.loop_before.unwrap_or(0.0); - // Invert: dragging left (negative offset) = extend - let desired_loop_before = (current_loop_before - self.drag_offset).max(0.0); - // Snap to whole iterations - let desired_iters = (desired_loop_before / content_window).round(); - let snapped_loop_before = desired_iters * content_window; + let trim_end = clip_instance.trim_end.unwrap_or(clip_duration.seconds_to_f64()); + let content_window_secs = (trim_end - clip_instance.trim_start).max(0.001); + let tmap = document.tempo_map(); + let ts = clip_instance.timeline_start; + // content window is a beats-domain span; guard against zero for division. + let content_window = tmap.seconds_to_beats(tmap.beats_to_seconds(ts) + Seconds(content_window_secs)) - ts; + let cw = content_window.beats_to_f64().max(1e-9); + let current_loop_before = clip_instance.loop_before.unwrap_or(Beats::ZERO); + // Invert: dragging left (negative seconds offset) = extend. Convert to a beats delta. + let drag_beats = tmap.seconds_to_beats(tmap.beats_to_seconds(ts) + Seconds(self.drag_offset)) - ts; + let desired_loop_before = (current_loop_before - drag_beats).max(Beats::ZERO); + // Snap to whole content-window iterations + let desired_iters = (desired_loop_before.beats_to_f64() / cw).round(); + let snapped_loop_before = Beats(desired_iters * cw); let new_loop_before = if snapped_loop_before > current_loop_before { // Extending left - check for adjacent clips @@ -3544,16 +3607,17 @@ impl TimelinePane { let extend_amount = (snapped_loop_before - current_loop_before).min(max_extend); // Re-snap after clamping let clamped = current_loop_before + extend_amount; - (clamped / content_window).floor() * content_window + Beats((clamped.beats_to_f64() / cw).floor() * cw) } else { snapped_loop_before }; - // Recompute instance_start and instance_duration - let right_duration = clip_instance.effective_duration(clip_duration, document.tempo_map()); - instance_start = clip_instance.timeline_start - new_loop_before; - instance_duration = new_loop_before + right_duration; - content_origin = clip_instance.timeline_start; + // Recompute instance_start and instance_duration (display seconds). + // Real content ends at ts + right_duration (beats), independent of loop_before. + let right_duration = clip_instance.effective_duration(clip_duration, tmap); + instance_start = tmap.beats_to_seconds(ts - new_loop_before).seconds_to_f64(); + instance_duration = tmap.beats_to_seconds(ts + right_duration).seconds_to_f64() - instance_start; + content_origin = tmap.beats_to_seconds(ts).seconds_to_f64(); } } } @@ -3757,7 +3821,7 @@ impl TimelinePane { // Calculate content window for loop detection // Use trimmed content window (preview_trim_start accounts for TrimLeft drag) - let preview_trim_end = clip_instance.trim_end.unwrap_or(clip_duration); + let preview_trim_end = clip_instance.trim_end.unwrap_or(clip_duration.seconds_to_f64()); let content_window = (preview_trim_end - preview_trim_start).max(0.0); let is_looping = instance_duration > content_window + 0.001; @@ -4211,8 +4275,8 @@ impl TimelinePane { let instance_end = instance_start + instance_duration; // Check if click is within this clip instance's pixel range and vertical bounds - let ci_start_x = self.time_to_x(instance_start); - let ci_end_x = self.time_to_x(instance_end).max(ci_start_x + MIN_CLIP_WIDTH_PX); + let ci_start_x = self.beats_to_x(instance_start, document.tempo_map()); + let ci_end_x = self.beats_to_x(instance_end, document.tempo_map()).max(ci_start_x + MIN_CLIP_WIDTH_PX); let click_x = pos.x - content_rect.min.x; let (row, total_rows) = click_stacking[ci_idx]; let (cy_min, cy_max) = clip_instance_y_bounds(row, total_rows); @@ -4503,8 +4567,9 @@ impl TimelinePane { let mut earliest = f64::MAX; for (_, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { for ci in clip_instances { - if selection.contains_clip_instance(&ci.id) && ci.timeline_start < earliest { - earliest = ci.timeline_start; + if selection.contains_clip_instance(&ci.id) { + let start_secs = document.tempo_map().beats_to_seconds(ci.timeline_start).seconds_to_f64(); + if start_secs < earliest { earliest = start_secs; } } } } @@ -4532,8 +4597,9 @@ impl TimelinePane { let mut earliest = f64::MAX; for (_, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { for ci in clip_instances { - if selection.contains_clip_instance(&ci.id) && ci.timeline_start < earliest { - earliest = ci.timeline_start; + if selection.contains_clip_instance(&ci.id) { + let start_secs = document.tempo_map().beats_to_seconds(ci.timeline_start).seconds_to_f64(); + if start_secs < earliest { earliest = start_secs; } } } } @@ -4555,10 +4621,11 @@ impl TimelinePane { if response.drag_stopped() { // Build layer_moves map for the action use std::collections::HashMap; - let mut layer_moves: HashMap> = + let mut layer_moves: HashMap> = HashMap::new(); - // Compute snapped offset once for all selected clips (preserves relative spacing) + // Compute snapped offset once for all selected clips (preserves relative spacing). + // `snapped_move_offset` is a uniform beats delta. let move_offset = self.snapped_move_offset(document.tempo_map(), &document.time_signature, document.framerate); // Iterate through all layers (including group children) to find selected clip instances @@ -4622,26 +4689,28 @@ impl TimelinePane { // New trim_start is snapped then clamped to valid range let desired_trim_start = self.snap_to_grid( old_trim_start + self.drag_offset, document.tempo_map(), &document.time_signature, document.framerate, - ).max(0.0).min(clip_duration); + ).max(0.0).min(clip_duration.seconds_to_f64()); - // Apply overlap prevention when extending left + // Apply overlap prevention when extending left (content-seconds gap). let new_trim_start = if desired_trim_start < old_trim_start { - let max_extend = document.find_max_trim_extend_left( + let max_extend_secs = document.find_max_trim_extend_left( &layer_id, &clip_instance.id, old_timeline_start, - ); + ).seconds_to_f64(); let desired_extend = old_trim_start - desired_trim_start; - let actual_extend = desired_extend.min(max_extend); + let actual_extend = desired_extend.min(max_extend_secs); old_trim_start - actual_extend } else { desired_trim_start }; - // Calculate actual offset after clamping + // Calculate actual offset after clamping (seconds), then the + // new timeline start in beats. let actual_offset = new_trim_start - old_trim_start; - let new_timeline_start = - old_timeline_start + actual_offset; + let new_timeline_start = document.tempo_map().seconds_to_beats( + document.tempo_map().beats_to_seconds(old_timeline_start) + Seconds(actual_offset) + ); layer_trims .entry(layer_id) @@ -4665,21 +4734,21 @@ impl TimelinePane { // Calculate new trim_end based on current duration let current_duration = clip_instance.effective_duration(clip_duration, document.tempo_map()); - let old_trim_end_val = clip_instance.trim_end.unwrap_or(clip_duration); + let old_trim_end_val = clip_instance.trim_end.unwrap_or(clip_duration.seconds_to_f64()); let desired_trim_end = self.snap_to_grid( old_trim_end_val + self.drag_offset, document.tempo_map(), &document.time_signature, document.framerate, - ).max(clip_instance.trim_start).min(clip_duration); + ).max(clip_instance.trim_start).min(clip_duration.seconds_to_f64()); - // Apply overlap prevention when extending right + // Apply overlap prevention when extending right (content-seconds gap). let new_trim_end_val = if desired_trim_end > old_trim_end_val { - let max_extend = document.find_max_trim_extend_right( + let max_extend_secs = document.find_max_trim_extend_right( &layer_id, &clip_instance.id, clip_instance.timeline_start, current_duration, - ); + ).seconds_to_f64(); let desired_extend = desired_trim_end - old_trim_end_val; - let actual_extend = desired_extend.min(max_extend); + let actual_extend = desired_extend.min(max_extend_secs); old_trim_end_val + actual_extend } else { desired_trim_end @@ -4688,10 +4757,10 @@ impl TimelinePane { let new_duration = (new_trim_end_val - clip_instance.trim_start).max(0.0); // Convert new duration back to trim_end value - let new_trim_end = if new_duration >= clip_duration { + let new_trim_end = if new_duration >= clip_duration.seconds_to_f64() { None // Use full clip duration } else { - Some((clip_instance.trim_start + new_duration).min(clip_duration)) + Some((clip_instance.trim_start + new_duration).min(clip_duration.seconds_to_f64())) }; layer_trims @@ -4741,20 +4810,26 @@ impl TimelinePane { }; if let Some(clip_duration) = clip_duration { + let tmap = document.tempo_map(); + let ts = clip_instance.timeline_start; let trim_end = clip_instance.trim_end.unwrap_or(clip_duration); - let content_window = (trim_end - clip_instance.trim_start).max(0.0); + let content_window_secs = (trim_end - clip_instance.trim_start).max(0.0); + let content_window = tmap.seconds_to_beats(tmap.beats_to_seconds(ts) + Seconds(content_window_secs)) - ts; let current_right = clip_instance.timeline_duration.unwrap_or(content_window); - let right_edge = clip_instance.timeline_start + current_right + self.drag_offset; - let snapped_edge = self.snap_to_grid(right_edge, document.tempo_map(), &document.time_signature, document.framerate); - let desired_right = snapped_edge - clip_instance.timeline_start; + // Snap the right edge in the seconds/pixel domain. + let right_edge_secs = tmap.beats_to_seconds(ts + current_right).seconds_to_f64() + self.drag_offset; + let snapped_edge_secs = self.snap_to_grid(right_edge_secs, tmap, &document.time_signature, document.framerate); + let desired_right = tmap.seconds_to_beats(Seconds(snapped_edge_secs)) - ts; let new_right = if desired_right > current_right { - let max_extend = document.find_max_trim_extend_right( + let max_extend_secs = document.find_max_trim_extend_right( &layer_id, &clip_instance.id, - clip_instance.timeline_start, + ts, current_right, ); + let right_edge = ts + current_right; + let max_extend = tmap.seconds_to_beats(tmap.beats_to_seconds(right_edge) + max_extend_secs) - right_edge; let extend_amount = (desired_right - current_right).min(max_extend); current_right + extend_amount } else { @@ -4762,7 +4837,7 @@ impl TimelinePane { }; let old_timeline_duration = clip_instance.timeline_duration; - let new_timeline_duration = if new_right > content_window + 0.001 { + let new_timeline_duration = if new_right > content_window + Beats(0.001) { Some(new_right) } else { None @@ -4809,14 +4884,19 @@ impl TimelinePane { }; if let Some(clip_duration) = clip_duration { + let tmap = document.tempo_map(); + let ts = clip_instance.timeline_start; let trim_end = clip_instance.trim_end.unwrap_or(clip_duration); - let content_window = (trim_end - clip_instance.trim_start).max(0.001); - let current_loop_before = clip_instance.loop_before.unwrap_or(0.0); - // Invert: dragging left (negative offset) = extend - let desired_loop_before = (current_loop_before - self.drag_offset).max(0.0); + let content_window_secs = (trim_end - clip_instance.trim_start).max(0.001); + let content_window = tmap.seconds_to_beats(tmap.beats_to_seconds(ts) + Seconds(content_window_secs)) - ts; + let cw = content_window.beats_to_f64().max(1e-9); + let current_loop_before = clip_instance.loop_before.unwrap_or(Beats::ZERO); + // Invert: dragging left (negative seconds offset) = extend. + let drag_beats = tmap.seconds_to_beats(tmap.beats_to_seconds(ts) + Seconds(self.drag_offset)) - ts; + let desired_loop_before = (current_loop_before - drag_beats).max(Beats::ZERO); // Snap to whole iterations so backend modulo aligns - let desired_iters = (desired_loop_before / content_window).round(); - let snapped = desired_iters * content_window; + let desired_iters = (desired_loop_before.beats_to_f64() / cw).round(); + let snapped = Beats(desired_iters * cw); let new_loop_before = if snapped > current_loop_before { let max_extend = document.find_max_loop_extend_left( @@ -4826,13 +4906,13 @@ impl TimelinePane { ); let extend_amount = (snapped - current_loop_before).min(max_extend); let clamped = current_loop_before + extend_amount; - (clamped / content_window).floor() * content_window + Beats((clamped.beats_to_f64() / cw).floor() * cw) } else { snapped }; let old_loop_before = clip_instance.loop_before; - let new_lb = if new_loop_before > 0.001 { + let new_lb = if new_loop_before > Beats(0.001) { Some(new_loop_before) } else { None @@ -5512,7 +5592,7 @@ impl PaneRenderer for TimelinePane { if let Some(clip_duration) = clip_duration { let instance_duration = clip_instance.effective_duration(clip_duration, document.tempo_map()); let instance_end = clip_instance.timeline_start + instance_duration; - max_endpoint = max_endpoint.max(instance_end); + max_endpoint = max_endpoint.max(document.tempo_map().beats_to_seconds(instance_end).seconds_to_f64()); } } } @@ -5943,8 +6023,8 @@ impl PaneRenderer for TimelinePane { if !shared.selection.contains_clip_instance(&inst.id) { continue; } if let Some(dur) = document.get_clip_duration(&inst.clip_id) { let eff = inst.effective_duration(dur, document.tempo_map()); - let start = inst.timeline_start; - let end = start + eff; + let start = document.tempo_map().beats_to_seconds(inst.timeline_start).seconds_to_f64(); + let end = document.tempo_map().beats_to_seconds(inst.timeline_start + eff).seconds_to_f64(); let min_dist = min_split_px as f64 / self.pixels_per_second as f64; if playback_time > start + min_dist && playback_time < end - min_dist { enabled = true; @@ -5969,10 +6049,14 @@ impl PaneRenderer for TimelinePane { .all(|ci| { if let Some(dur) = document.get_clip_duration(&ci.clip_id) { let eff = ci.effective_duration(dur, document.tempo_map()); - let max_extend = document.find_max_trim_extend_right( + // Room to duplicate = seconds gap to the right ≥ this clip's own length. + let max_extend_secs = document.find_max_trim_extend_right( &layer_id, &ci.id, ci.timeline_start, eff, - ); - max_extend >= eff + ).seconds_to_f64(); + let tmap = document.tempo_map(); + let eff_secs = (tmap.beats_to_seconds(ci.timeline_start + eff) + - tmap.beats_to_seconds(ci.timeline_start)).seconds_to_f64(); + max_extend_secs >= eff_secs } else { false } @@ -6001,11 +6085,13 @@ impl PaneRenderer for TimelinePane { let min_start = instances .iter() .map(|i| i.timeline_start) - .fold(f64::INFINITY, f64::min); - let offset = *shared.playback_time - min_start; + .fold(Beats(f64::INFINITY), |a, b| a.min(b)); + // Place so the earliest clip lands at the playhead (beats). + let playhead_beats = document.tempo_map().seconds_to_beats(Seconds(*shared.playback_time)); + let offset = playhead_beats - min_start; enabled = instances.iter().all(|ci| { - let paste_start = (ci.timeline_start + offset).max(0.0); + let paste_start = (ci.timeline_start + offset).max(Beats::ZERO); if let Some(dur) = document.get_clip_duration(&ci.clip_id) { let eff = ci.effective_duration(dur, document.tempo_map()); document @@ -6210,24 +6296,22 @@ impl PaneRenderer for TimelinePane { // Show drop time indicator with snap preview let raw_drop_time = self.x_to_time(pointer_pos.x - content_rect.min.x).max(0.0); - // Calculate snapped drop time for preview + // Calculate snapped drop time for preview (seconds, for time_to_x) let drop_time = if is_compatible { - // Get clip duration to calculate snapped position - let clip_duration = { - let doc = shared.action_executor.document(); - doc.get_clip_duration(&dragging.clip_id).unwrap_or(1.0) - }; - - // Find nearest valid position (auto-snap for preview) - let snapped = shared.action_executor.document() - .find_nearest_valid_position( - &layer.id(), - raw_drop_time, - clip_duration, - &[], - ); - - snapped.unwrap_or(raw_drop_time) + let doc = shared.action_executor.document(); + let tmap = doc.tempo_map(); + let clip_duration = doc.get_clip_duration(&dragging.clip_id).unwrap_or(Seconds(1.0)); + // Overlap testing is beats-domain: convert the drop point and the clip's + // content span to beats. + let raw_drop_beats = tmap.seconds_to_beats(Seconds(raw_drop_time)); + let clip_dur_beats = tmap.seconds_to_beats(tmap.beats_to_seconds(raw_drop_beats) + clip_duration) - raw_drop_beats; + let snapped = doc.find_nearest_valid_position( + &layer.id(), + raw_drop_beats, + clip_dur_beats, + &[], + ); + snapped.map(|b| tmap.beats_to_seconds(b).seconds_to_f64()).unwrap_or(raw_drop_time) } else { raw_drop_time }; @@ -6261,9 +6345,12 @@ impl PaneRenderer for TimelinePane { } // Create clip instance for effect with 5 second default duration + let tmap = shared.action_executor.document().tempo_map(); + let drop_beats = tmap.seconds_to_beats(Seconds(drop_time)); + let dur_beats = tmap.seconds_to_beats(tmap.beats_to_seconds(drop_beats) + Seconds(5.0)) - drop_beats; let clip_instance = ClipInstance::new(def.id) - .with_timeline_start(drop_time) - .with_timeline_duration(5.0); + .with_timeline_start(drop_beats) + .with_timeline_duration(dur_beats); // Use AddEffectAction for effect layers let action = lightningbeam_core::actions::AddEffectAction::new( @@ -6283,7 +6370,7 @@ impl PaneRenderer for TimelinePane { let center_y = doc.height / 2.0; let mut clip_instance = ClipInstance::new(dragging.clip_id) - .with_timeline_start(drop_time); + .with_timeline_start(doc.tempo_map().seconds_to_beats(Seconds(drop_time))); // For video clips, fit uniformly + centered (preserve aspect). // Shared with the direct-import path via Transform::fit_centered. @@ -6321,7 +6408,7 @@ impl PaneRenderer for TimelinePane { // Find or create sampled audio track where the audio won't overlap let audio_layer_id = { let doc = shared.action_executor.document(); - let result = find_sampled_audio_track_for_clip(doc, linked_audio_clip_id, drop_time, editing_clip_id.as_ref()); + let result = find_sampled_audio_track_for_clip(doc, linked_audio_clip_id, doc.tempo_map().seconds_to_beats(Seconds(drop_time)), editing_clip_id.as_ref()); if let Some(id) = result { eprintln!("DEBUG: Found existing audio track without overlap: {}", id); } else { @@ -6343,7 +6430,7 @@ impl PaneRenderer for TimelinePane { // Create audio clip instance at same timeline position let audio_instance = ClipInstance::new(linked_audio_clip_id) - .with_timeline_start(drop_time); + .with_timeline_start(shared.action_executor.document().tempo_map().seconds_to_beats(Seconds(drop_time))); let audio_instance_id = audio_instance.id; eprintln!("DEBUG: Created audio instance: {} for clip: {}", audio_instance_id, linked_audio_clip_id); @@ -6436,9 +6523,12 @@ impl PaneRenderer for TimelinePane { shared.action_executor.document_mut().add_effect_definition(def.clone()); } + let tmap = shared.action_executor.document().tempo_map(); + let drop_beats = tmap.seconds_to_beats(Seconds(drop_time)); + let dur_beats = tmap.seconds_to_beats(tmap.beats_to_seconds(drop_beats) + Seconds(5.0)) - drop_beats; let clip_instance = ClipInstance::new(def.id) - .with_timeline_start(drop_time) - .with_timeline_duration(5.0); + .with_timeline_start(drop_beats) + .with_timeline_duration(dur_beats); let action = lightningbeam_core::actions::AddEffectAction::new( new_layer_id, @@ -6449,7 +6539,7 @@ impl PaneRenderer for TimelinePane { } else { // Handle other clip types let clip_instance = ClipInstance::new(dragging.clip_id) - .with_timeline_start(drop_time); + .with_timeline_start(shared.action_executor.document().tempo_map().seconds_to_beats(Seconds(drop_time))); let action = lightningbeam_core::actions::AddClipInstanceAction::new( new_layer_id, From b5766672ee353980a3de257e4b2b2f4a37fed9fc Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Sat, 11 Jul 2026 14:10:44 -0400 Subject: [PATCH 3/8] Fix recording start position: pass beats, not seconds, to the backend The reported bug (a second recording lands early and overlaps the first) survived the timeline type refactor: start_recording/create_midi_clip/ start_midi_recording took f64 and wrapped Beats(x) internally, so the type boundary stopped at the method and the timeline handed them *shared.playback_time (seconds). At 120 BPM a 5s playhead (=10 beats) was recorded at beat 5 = 2.5s. Type all three backend methods to take Beats so the caller must convert; the timeline now converts the seconds playhead once (start_beats) and passes it to every recording command and the placeholder clip. TUI debug caller updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- daw-backend/src/audio/engine.rs | 12 ++++++------ daw-backend/src/tui/mod.rs | 2 +- .../lightningbeam-editor/src/panes/timeline.rs | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index a7dcab5..7581025 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -3607,10 +3607,10 @@ impl EngineController { } /// Create a new MIDI clip on a track - pub fn create_midi_clip(&mut self, track_id: TrackId, start_time: f64, duration: f64) -> MidiClipId { + pub fn create_midi_clip(&mut self, track_id: TrackId, start_time: Beats, duration: Beats) -> MidiClipId { // Peek at the next clip ID that will be used let clip_id = self.next_midi_clip_id.load(Ordering::Relaxed); - let _ = self.command_tx.push(Command::CreateMidiClip(track_id, start_time, duration)); + let _ = self.command_tx.push(Command::CreateMidiClip(track_id, start_time.beats_to_f64(), duration.beats_to_f64())); clip_id } @@ -3739,8 +3739,8 @@ impl EngineController { } /// Start recording on a track - pub fn start_recording(&mut self, track_id: TrackId, start_time: f64) { - let _ = self.command_tx.push(Command::StartRecording(track_id, Beats(start_time))); + pub fn start_recording(&mut self, track_id: TrackId, start_time: Beats) { + let _ = self.command_tx.push(Command::StartRecording(track_id, start_time)); } /// Stop the current recording @@ -3759,8 +3759,8 @@ impl EngineController { } /// Start MIDI recording on a track - pub fn start_midi_recording(&mut self, track_id: TrackId, clip_id: MidiClipId, start_time: f64) { - let _ = self.command_tx.push(Command::StartMidiRecording(track_id, clip_id, Beats(start_time))); + pub fn start_midi_recording(&mut self, track_id: TrackId, clip_id: MidiClipId, start_time: Beats) { + let _ = self.command_tx.push(Command::StartMidiRecording(track_id, clip_id, start_time)); } /// Stop the current MIDI recording diff --git a/daw-backend/src/tui/mod.rs b/daw-backend/src/tui/mod.rs index 3dc57bc..85d19bc 100644 --- a/daw-backend/src/tui/mod.rs +++ b/daw-backend/src/tui/mod.rs @@ -830,7 +830,7 @@ fn execute_command( app.next_clip_id += 1; app.add_clip(track_id, clip_id, start_time, duration, format!("Clip {}", clip_id), Vec::new()); - controller.create_midi_clip(track_id, start_time, duration); + controller.create_midi_clip(track_id, crate::Beats(start_time), crate::Beats(duration)); app.set_status(format!("Created MIDI clip on track {} at {:.2}s for {:.2}s", track_id, start_time, duration)); } "loadmidi" => { diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index b67ffe2..8274f1d 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -1127,6 +1127,9 @@ impl TimelinePane { shared.recording_layer_ids.clear(); + // The backend records in the beats domain; start_time is the seconds playhead. + let start_beats = shared.action_executor.document().tempo_map().seconds_to_beats(Seconds(start_time)); + // Step 4: Dispatch recording for each candidate for &(layer_id, ref cat, _) in &candidates { match cat { @@ -1150,7 +1153,7 @@ impl TimelinePane { } if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.start_recording(track_id, start_time); + controller.start_recording(track_id, start_beats); println!("🎤 Started audio recording on track {:?} at {:.2}s", track_id, start_time); } shared.recording_layer_ids.push(layer_id); @@ -1162,8 +1165,8 @@ impl TimelinePane { if let Some(&track_id) = shared.layer_to_track_map.get(&layer_id) { if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - let clip_id = controller.create_midi_clip(track_id, start_time, 0.0); - controller.start_midi_recording(track_id, clip_id, start_time); + let clip_id = controller.create_midi_clip(track_id, start_beats, Beats::ZERO); + controller.start_midi_recording(track_id, clip_id, start_beats); shared.recording_clips.insert(layer_id, clip_id); println!("🎹 Started MIDI recording on track {:?} at {:.2}s, clip_id={}", track_id, start_time, clip_id); @@ -1174,8 +1177,6 @@ impl TimelinePane { *shared.recording_clips.get(&layer_id).unwrap_or(&0), daw_backend::Beats::ZERO); let doc_clip_id = shared.action_executor.document_mut().add_audio_clip(doc_clip); - let start_beats = shared.action_executor.document().tempo_map() - .seconds_to_beats(Seconds(start_time)); let clip_instance = ClipInstance::new(doc_clip_id) .with_timeline_start(start_beats); From 64bf9bb431e66d528049c86494cded1ad62650b7 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Sat, 11 Jul 2026 14:33:14 -0400 Subject: [PATCH 4/8] Type the rest of the audio-controller boundary (no more bare-f64 wrapping) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Push Beats/Seconds through the remaining controller methods that took a bare f64 and let the audio thread wrap it in a newtype, so the caller's domain is now compiler-checked (the seam that hid the recording bug): - seek -> Seconds - set_trim_start/set_trim_end -> Seconds / Option (metatrack, always seconds) - add_midi_note, add_loaded_midi_clip, update_midi_clip_notes -> Beats - add_automation_point, remove_automation_point, automation_add_keyframe, automation_remove_keyframe -> Beats Command enums stay raw f64 transport; only the public signatures + call sites change. No behavior change — every caller already passed the right domain, this just makes it enforced. Two deliberate exceptions, documented in place: - trim_clip stays f64: the TrimClip handler interprets it as Seconds for a sampled-audio clip but Beats for a MIDI clip, so no single newtype fits; callers pass the clip's own trim value, which matches its content domain. - The piano-roll MIDI note model stays f64 internally (a beats-only subsystem with no seconds anywhere); it's converted to Beats at the update_midi_clip_notes boundary in UpdateMidiNotesAction, same as trim_start f64 -> Seconds at add_audio_clip. --- daw-backend/src/audio/engine.rs | 51 ++++++++++--------- daw-backend/src/tui/mod.rs | 4 +- .../src/actions/move_clip_instances.rs | 8 +-- .../src/actions/trim_clip_instances.rs | 8 +-- .../src/actions/update_midi_notes.rs | 11 +++- .../src/mobile/transport.rs | 5 +- .../src/panes/piano_roll.rs | 4 +- .../src/panes/timeline.rs | 24 ++++----- 8 files changed, 64 insertions(+), 51 deletions(-) diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index 7581025..bc3c224 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -3348,8 +3348,8 @@ impl EngineController { } /// Seek to a specific position in seconds - pub fn seek(&mut self, seconds: f64) { - let _ = self.command_tx.push(Command::Seek(seconds)); + pub fn seek(&mut self, seconds: Seconds) { + let _ = self.command_tx.push(Command::Seek(seconds.seconds_to_f64())); } /// Set track volume (0.0 = silence, 1.0 = unity gain) @@ -3386,6 +3386,10 @@ impl EngineController { /// Trim a clip's internal boundaries (changes which portion of source content is used) /// This also resets external_duration to match internal duration (disables looping) + /// Trim a clip's internal content bounds. The units are content-domain and depend on the + /// track type: SECONDS for a sampled-audio clip, BEATS for a MIDI clip (see the TrimClip + /// handler). Left as raw f64 because a single newtype can't express both; callers pass the + /// clip's own `trim_start`/`trim_end`, which already match its content domain. pub fn trim_clip(&mut self, track_id: TrackId, clip_id: ClipId, new_internal_start: f64, new_internal_end: f64) { let _ = self.command_tx.push(Command::TrimClip(track_id, clip_id, new_internal_start, new_internal_end)); } @@ -3450,13 +3454,13 @@ impl EngineController { } /// Set metatrack trim start in seconds - pub fn set_trim_start(&mut self, track_id: TrackId, trim_start: f64) { - let _ = self.command_tx.push(Command::SetTrimStart(track_id, trim_start)); + pub fn set_trim_start(&mut self, track_id: TrackId, trim_start: Seconds) { + let _ = self.command_tx.push(Command::SetTrimStart(track_id, trim_start.seconds_to_f64())); } /// Set metatrack trim end in seconds (None = no end trim) - pub fn set_trim_end(&mut self, track_id: TrackId, trim_end: Option) { - let _ = self.command_tx.push(Command::SetTrimEnd(track_id, trim_end)); + pub fn set_trim_end(&mut self, track_id: TrackId, trim_end: Option) { + let _ = self.command_tx.push(Command::SetTrimEnd(track_id, trim_end.map(|s| s.seconds_to_f64()))); } /// Create a new audio track @@ -3615,17 +3619,18 @@ impl EngineController { } /// Add a MIDI note to a clip - pub fn add_midi_note(&mut self, track_id: TrackId, clip_id: MidiClipId, time_offset: f64, note: u8, velocity: u8, duration: f64) { - let _ = self.command_tx.push(Command::AddMidiNote(track_id, clip_id, time_offset, note, velocity, duration)); + pub fn add_midi_note(&mut self, track_id: TrackId, clip_id: MidiClipId, time_offset: Beats, note: u8, velocity: u8, duration: Beats) { + let _ = self.command_tx.push(Command::AddMidiNote(track_id, clip_id, time_offset.beats_to_f64(), note, velocity, duration.beats_to_f64())); } - /// Add a pre-loaded MIDI clip to a track at the given timeline position - pub fn add_loaded_midi_clip(&mut self, track_id: TrackId, clip: MidiClip, start_time: f64) { - let _ = self.command_tx.push(Command::AddLoadedMidiClip(track_id, clip, start_time)); + /// Add a pre-loaded MIDI clip to a track at the given timeline position (beats) + pub fn add_loaded_midi_clip(&mut self, track_id: TrackId, clip: MidiClip, start_time: Beats) { + let _ = self.command_tx.push(Command::AddLoadedMidiClip(track_id, clip, start_time.beats_to_f64())); } - /// Update all notes in a MIDI clip - pub fn update_midi_clip_notes(&mut self, track_id: TrackId, clip_id: MidiClipId, notes: Vec<(f64, u8, u8, f64)>) { + /// Update all notes in a MIDI clip. Note tuples are (start [beats], note, velocity, duration [beats]). + pub fn update_midi_clip_notes(&mut self, track_id: TrackId, clip_id: MidiClipId, notes: Vec<(Beats, u8, u8, Beats)>) { + let notes = notes.into_iter().map(|(t, n, v, d)| (t.beats_to_f64(), n, v, d.beats_to_f64())).collect(); let _ = self.command_tx.push(Command::UpdateMidiClipNotes(track_id, clip_id, notes)); } @@ -3661,25 +3666,25 @@ impl EngineController { &mut self, track_id: TrackId, lane_id: crate::audio::AutomationLaneId, - time: f64, + time: Beats, value: f32, curve: crate::audio::CurveType, ) { let _ = self.command_tx.push(Command::AddAutomationPoint( - track_id, lane_id, time, value, curve, + track_id, lane_id, time.beats_to_f64(), value, curve, )); } - /// Remove an automation point at a specific time + /// Remove an automation point at a specific time (beats); tolerance is a beats delta pub fn remove_automation_point( &mut self, track_id: TrackId, lane_id: crate::audio::AutomationLaneId, - time: f64, - tolerance: f64, + time: Beats, + tolerance: Beats, ) { let _ = self.command_tx.push(Command::RemoveAutomationPoint( - track_id, lane_id, time, tolerance, + track_id, lane_id, time.beats_to_f64(), tolerance.beats_to_f64(), )); } @@ -3715,16 +3720,16 @@ impl EngineController { /// Add a keyframe to an AutomationInput node pub fn automation_add_keyframe(&mut self, track_id: TrackId, node_id: u32, - time: f64, value: f32, interpolation: String, + time: Beats, value: f32, interpolation: String, ease_out: (f32, f32), ease_in: (f32, f32)) { let _ = self.command_tx.push(Command::AutomationAddKeyframe( - track_id, node_id, time, value, interpolation, ease_out, ease_in)); + track_id, node_id, time.beats_to_f64(), value, interpolation, ease_out, ease_in)); } /// Remove a keyframe from an AutomationInput node - pub fn automation_remove_keyframe(&mut self, track_id: TrackId, node_id: u32, time: f64) { + pub fn automation_remove_keyframe(&mut self, track_id: TrackId, node_id: u32, time: Beats) { let _ = self.command_tx.push(Command::AutomationRemoveKeyframe( - track_id, node_id, time)); + track_id, node_id, time.beats_to_f64())); } /// Set the display name of an AutomationInput node diff --git a/daw-backend/src/tui/mod.rs b/daw-backend/src/tui/mod.rs index 85d19bc..0d44a6b 100644 --- a/daw-backend/src/tui/mod.rs +++ b/daw-backend/src/tui/mod.rs @@ -790,7 +790,7 @@ fn execute_command( return Err("Usage: seek ".to_string()); } let pos: f64 = parts[1].parse().map_err(|_| "Invalid position")?; - controller.seek(pos); + controller.seek(crate::Seconds(pos)); app.set_status(format!("Seeked to {:.2}s", pos)); } "track" => { @@ -882,7 +882,7 @@ fn execute_command( app.next_clip_id += 1; // Send to audio engine with the start_time (clip content is separate from timeline position) - controller.add_loaded_midi_clip(track_id, midi_clip, start_time); + controller.add_loaded_midi_clip(track_id, midi_clip, crate::Beats(start_time)); app.set_status(format!("Loaded {} ({} events, {:.2}s) to track {} at {:.2}s", file_path, event_count, duration, track_id, start_time)); diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs index 6664d3d..73f49b2 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/move_clip_instances.rs @@ -211,8 +211,8 @@ impl Action for MoveClipInstancesAction { // Check if this clip has a metatrack if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(*new_start)); - controller.set_trim_start(metatrack_id, instance.trim_start); - controller.set_trim_end(metatrack_id, instance.trim_end); + controller.set_trim_start(metatrack_id, daw_backend::Seconds(instance.trim_start)); + controller.set_trim_end(metatrack_id, instance.trim_end.map(daw_backend::Seconds)); } } } @@ -295,8 +295,8 @@ impl Action for MoveClipInstancesAction { if let Some(instance) = vl.clip_instances.iter().find(|ci| ci.id == *instance_id) { if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(*old_start)); - controller.set_trim_start(metatrack_id, instance.trim_start); - controller.set_trim_end(metatrack_id, instance.trim_end); + controller.set_trim_start(metatrack_id, daw_backend::Seconds(instance.trim_start)); + controller.set_trim_end(metatrack_id, instance.trim_end.map(daw_backend::Seconds)); } } } diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs index b3da86c..55fa95d 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs @@ -387,8 +387,8 @@ impl Action for TrimClipInstancesAction { if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { // Instance already has new values after execute() controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(instance.timeline_start)); - controller.set_trim_start(metatrack_id, instance.trim_start); - controller.set_trim_end(metatrack_id, instance.trim_end); + controller.set_trim_start(metatrack_id, daw_backend::Seconds(instance.trim_start)); + controller.set_trim_end(metatrack_id, instance.trim_end.map(daw_backend::Seconds)); } } } @@ -477,8 +477,8 @@ impl Action for TrimClipInstancesAction { if let Some(&metatrack_id) = backend.layer_to_track_map.get(&instance.clip_id) { // Instance already has old values after rollback() controller.set_offset(metatrack_id, document.tempo_map().beats_to_seconds(instance.timeline_start)); - controller.set_trim_start(metatrack_id, instance.trim_start); - controller.set_trim_end(metatrack_id, instance.trim_end); + controller.set_trim_start(metatrack_id, daw_backend::Seconds(instance.trim_start)); + controller.set_trim_end(metatrack_id, instance.trim_end.map(daw_backend::Seconds)); } } } diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/update_midi_notes.rs b/lightningbeam-ui/lightningbeam-core/src/actions/update_midi_notes.rs index 06ca4f2..a74074b 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/update_midi_notes.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/update_midi_notes.rs @@ -1,7 +1,13 @@ use crate::action::Action; use crate::document::Document; +use daw_backend::Beats; use uuid::Uuid; +/// Convert editor-side beats-domain note tuples to the typed backend form. +fn notes_to_beats(notes: &[(f64, u8, u8, f64)]) -> Vec<(Beats, u8, u8, Beats)> { + notes.iter().map(|&(t, n, v, d)| (Beats(t), n, v, Beats(d))).collect() +} + /// Action to update MIDI notes in a clip (supports undo/redo) /// /// Stores the before and after note states. MIDI note data lives in the backend, @@ -49,7 +55,8 @@ impl Action for UpdateMidiNotesAction { .get(&self.layer_id) .ok_or_else(|| format!("Layer {} not mapped to backend track", self.layer_id))?; - controller.update_midi_clip_notes(*track_id, self.midi_clip_id, self.new_notes.clone()); + // Note times/durations are beats (MIDI content domain); assert that at the typed boundary. + controller.update_midi_clip_notes(*track_id, self.midi_clip_id, notes_to_beats(&self.new_notes)); Ok(()) } @@ -68,7 +75,7 @@ impl Action for UpdateMidiNotesAction { .get(&self.layer_id) .ok_or_else(|| format!("Layer {} not mapped to backend track", self.layer_id))?; - controller.update_midi_clip_notes(*track_id, self.midi_clip_id, self.old_notes.clone()); + controller.update_midi_clip_notes(*track_id, self.midi_clip_id, notes_to_beats(&self.old_notes)); Ok(()) } diff --git a/lightningbeam-ui/lightningbeam-editor/src/mobile/transport.rs b/lightningbeam-ui/lightningbeam-editor/src/mobile/transport.rs index 8b4bc1b..101fe5c 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/mobile/transport.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/mobile/transport.rs @@ -2,6 +2,7 @@ //! project scrub. Wired to the audio controller exactly like `TimelinePane`'s header. use eframe::egui; +use daw_backend::Seconds; use super::{icons, Palette}; use crate::panes::SharedPaneState; @@ -32,7 +33,7 @@ pub fn render(ui: &mut egui::Ui, rect: egui::Rect, shared: &mut SharedPaneState, if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); if *shared.is_playing { - controller.seek(*shared.playback_time); + controller.seek(Seconds(*shared.playback_time)); controller.play(); } else { controller.pause(); @@ -101,7 +102,7 @@ pub fn render(ui: &mut egui::Ui, rect: egui::Rect, shared: &mut SharedPaneState, *shared.playback_time = new_time; if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(new_time); + controller.seek(Seconds(new_time)); } } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs index c1e24ad..34b2145 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs @@ -651,7 +651,7 @@ impl PianoRollPane { *shared.playback_time = nt; if let Some(ctrl) = shared.audio_controller.as_ref() { if let Ok(mut c) = ctrl.lock() { - c.seek(nt); + c.seek(Seconds(nt)); } } } @@ -1764,7 +1764,7 @@ impl PianoRollPane { let seek_time = snap_to_value(time.max(0.0), self.snap_value, tempo_map); *shared.playback_time = seek_time; if let Some(ctrl) = shared.audio_controller.as_ref() { - if let Ok(mut c) = ctrl.lock() { c.seek(seek_time); } + if let Ok(mut c) = ctrl.lock() { c.seek(Seconds(seek_time)); } } } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index 8274f1d..a694da2 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -1110,7 +1110,7 @@ impl TimelinePane { if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(seek_to); + controller.seek(Seconds(seek_to)); controller.set_metronome_enabled(true); if !*shared.is_playing { controller.play(); @@ -5006,7 +5006,7 @@ impl TimelinePane { *playback_time = kf_time; if let Some(controller_arc) = audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(kf_time); + controller.seek(Seconds(kf_time)); } } } @@ -5034,7 +5034,7 @@ impl TimelinePane { // Seek immediately so it works while playing if let Some(controller_arc) = audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(new_time); + controller.seek(Seconds(new_time)); } } } @@ -5046,7 +5046,7 @@ impl TimelinePane { *playback_time = new_time; if let Some(controller_arc) = audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(new_time); + controller.seek(Seconds(new_time)); } } } @@ -5206,7 +5206,7 @@ impl PaneRenderer for TimelinePane { *shared.playback_time = 0.0; if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(0.0); + controller.seek(Seconds(0.0)); } } @@ -5215,7 +5215,7 @@ impl PaneRenderer for TimelinePane { *shared.playback_time = (*shared.playback_time - 0.1).max(0.0); if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(*shared.playback_time); + controller.seek(Seconds(*shared.playback_time)); } } @@ -5251,7 +5251,7 @@ impl PaneRenderer for TimelinePane { *shared.playback_time = (*shared.playback_time + 0.1).min(self.duration); if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(*shared.playback_time); + controller.seek(Seconds(*shared.playback_time)); } } @@ -5260,7 +5260,7 @@ impl PaneRenderer for TimelinePane { *shared.playback_time = self.duration; if let Some(controller_arc) = shared.audio_controller { let mut controller = controller_arc.lock().unwrap(); - controller.seek(self.duration); + controller.seek(Seconds(self.duration)); } } @@ -5692,7 +5692,7 @@ impl PaneRenderer for TimelinePane { self.automation_cache.remove(&layer_id); } else if let Some(&track_id) = shared.layer_to_track_map.get(&layer_id) { // time is already in beats (all automation x-axes use beats) - controller.automation_add_keyframe(track_id, node_id, time, value, "linear".to_string(), (0.0, 0.0), (0.0, 0.0)); + controller.automation_add_keyframe(track_id, node_id, Beats(time), value, "linear".to_string(), (0.0, 0.0), (0.0, 0.0)); // Optimistic cache update (beats) if let Some(lanes) = self.automation_cache.get_mut(&layer_id) { if let Some(lane) = lanes.iter_mut().find(|l| l.node_id == node_id) { @@ -5740,8 +5740,8 @@ impl PaneRenderer for TimelinePane { self.automation_cache.remove(&layer_id); } else if let Some(&track_id) = shared.layer_to_track_map.get(&layer_id) { // old_time / new_time are already in beats - controller.automation_remove_keyframe(track_id, node_id, old_time); - controller.automation_add_keyframe(track_id, node_id, new_time, new_value, interpolation.clone(), ease_out, ease_in); + controller.automation_remove_keyframe(track_id, node_id, Beats(old_time)); + controller.automation_add_keyframe(track_id, node_id, Beats(new_time), new_value, interpolation.clone(), ease_out, ease_in); // Optimistic cache update (beats) if let Some(lanes) = self.automation_cache.get_mut(&layer_id) { if let Some(lane) = lanes.iter_mut().find(|l| l.node_id == node_id) { @@ -5769,7 +5769,7 @@ impl PaneRenderer for TimelinePane { } } else if let Some(&track_id) = shared.layer_to_track_map.get(&layer_id) { // time is already in beats - controller.automation_remove_keyframe(track_id, node_id, time); + controller.automation_remove_keyframe(track_id, node_id, Beats(time)); // Optimistic cache update (beats) if let Some(lanes) = self.automation_cache.get_mut(&layer_id) { if let Some(lane) = lanes.iter_mut().find(|l| l.node_id == node_id) { From c74712675919fb4f2abc2a376cd94e71d0fe44dc Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Sat, 11 Jul 2026 16:42:34 -0400 Subject: [PATCH 5/8] Timeline: display audio/MIDI clips from the doc, not a backend snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The timeline was the only place that reconstructed audio-layer clip instances from a backend snapshot (build_audio_clip_cache) instead of reading the document like every other layer type. But the document is the actual source of truth: clip actions mutate *Layer::clip_instances, it's what persists to .beam, and it drives the backend on load. The snapshot is downstream of it. That dual source caused the second-recording preview bug: a recording clip lives only in the document (its backend clip has a temporary pool index of 0, so the snapshot can't represent it). layer_clips() only fell back to the doc when the snapshot cache was *empty*, so the first recording previewed but the second — once the layer already had a finalized clip — was dropped and showed as zero-length until finalized. Fix: make layer_clips() read .clip_instances for audio too, and delete build_audio_clip_cache + the audio_cache plumbing threaded through ~7 methods and ~25 call sites. Recording clips grow via the existing RecordingProgress / MidiRecordingProgress mirror to the doc; finalized clips come from the actions; both states now live in one place. Root fix this exposed: get_clip_duration() wrapped every audio clip's duration as Seconds, but MIDI clips store their duration in BEATS (they share AudioClip with sampled clips). The snapshot had masked this by always forcing a MIDI clip's timeline_duration; reading from the doc requires the value to be right, so get_clip_duration now converts a MIDI clip's beats duration to seconds. Net -120 lines. Whole workspace compiles; 299 core tests pass. The piano roll still uses the snapshot for its own MIDI editor (separate pane, out of scope). --- .../lightningbeam-core/src/document.rs | 10 +- .../src/panes/timeline.rs | 187 +++--------------- 2 files changed, 38 insertions(+), 159 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/document.rs b/lightningbeam-ui/lightningbeam-core/src/document.rs index 7bca2ea..9596f74 100644 --- a/lightningbeam-ui/lightningbeam-core/src/document.rs +++ b/lightningbeam-ui/lightningbeam-core/src/document.rs @@ -921,7 +921,15 @@ impl Document { } else if let Some(clip) = self.video_clips.get(clip_id) { Some(Seconds(clip.duration)) } else if let Some(clip) = self.audio_clips.get(clip_id) { - Some(Seconds(clip.duration)) + // MIDI clips store `duration` in BEATS (they share the AudioClip struct with + // sampled clips, whose duration is seconds). Convert to wall-clock seconds so + // the content-window sizing works uniformly. + match clip.clip_type { + crate::clip::AudioClipType::Midi { .. } => { + Some(self.tempo_map().beats_to_seconds(Beats(clip.duration))) + } + _ => Some(Seconds(clip.duration)), + } } else if self.effect_definitions.contains_key(clip_id) { // Effects have infinite internal duration - their timeline length // is controlled by ClipInstance.trim_end diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index a694da2..2ed636d 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -8,12 +8,9 @@ use eframe::egui; use daw_backend::{Beats, Seconds}; -use lightningbeam_core::clip::{ - ClipInstance, audio_backend_uuid, midi_backend_uuid, -}; +use lightningbeam_core::clip::ClipInstance; use lightningbeam_core::layer::{AnyLayer, AudioLayerType, GroupLayer, LayerTrait}; use super::{DragClipType, NodePath, PaneRenderer, SharedPaneState}; -use std::collections::HashMap; const RULER_HEIGHT: f32 = 30.0; const LAYER_HEIGHT: f32 = 60.0; @@ -582,113 +579,16 @@ fn shift_toggle_layer( *focus = lightningbeam_core::selection::FocusSelection::Layers(vec![layer_id]); } -/// Build a per-audio-layer clip instance cache from the backend snapshot. +/// Get a layer's clip instances from the document. /// -/// Audio layers read clip instances from the backend snapshot (source of truth) rather -/// than from `AudioLayer::clip_instances`. The cache maps layer_id → Vec. -/// -/// Clip instance UUIDs in the cache are doc UUIDs when available (via reverse lookup of -/// `clip_instance_to_backend_map`), falling back to synthetic `audio_backend_uuid` / -/// `midi_backend_uuid` values for clips not yet in the map. -fn build_audio_clip_cache( - snap: &daw_backend::AudioClipSnapshot, - layer_to_track_map: &HashMap, - document: &lightningbeam_core::document::Document, - clip_map: &HashMap, -) -> HashMap> { - use lightningbeam_core::action::BackendClipInstanceId; - - // Build reverse maps: backend_id → doc_instance_uuid - let mut audio_id_to_doc: HashMap = HashMap::new(); - let mut midi_id_to_doc: HashMap = HashMap::new(); - for (&doc_uuid, backend_id) in clip_map { - match backend_id { - BackendClipInstanceId::Audio(id) => { audio_id_to_doc.insert(*id, doc_uuid); } - BackendClipInstanceId::Midi(id) => { midi_id_to_doc.insert(*id, doc_uuid); } - } - } - - let mut cache: HashMap> = HashMap::new(); - - for (&layer_id, &track_id) in layer_to_track_map { - // Only process audio layers - match document.get_layer(&layer_id) { - Some(AnyLayer::Audio(_)) => {} - _ => continue, - } - - let mut instances = Vec::new(); - - // Sampled audio clips - if let Some(audio_clips) = snap.audio.get(&track_id) { - for ac in audio_clips { - if let Some((clip_id, _)) = document.audio_clip_by_pool_index(ac.audio_pool_index) { - // Use doc UUID if we have it; otherwise fall back to synthetic UUID - let instance_id = audio_id_to_doc.get(&ac.id) - .copied() - .unwrap_or_else(|| audio_backend_uuid(ac.id)); - let mut ci = ClipInstance::new(clip_id); - ci.id = instance_id; - ci.timeline_start = ac.external_start; - ci.trim_start = ac.internal_start.seconds_to_f64(); - ci.trim_end = Some(ac.internal_end.seconds_to_f64()); - let internal_dur_secs = (ac.internal_end - ac.internal_start).seconds_to_f64(); - let tempo_map = document.tempo_map(); - let external_dur_secs = tempo_map.transform(ac.external_duration.beats_to_f64()); - if (external_dur_secs - internal_dur_secs).abs() > 1e-9 { - ci.timeline_duration = Some(ac.external_duration); - } - ci.gain = ac.gain; - instances.push(ci); - } - } - } - - // MIDI clips - if let Some(midi_clips) = snap.midi.get(&track_id) { - for mc in midi_clips { - if let Some((clip_id, _)) = document.audio_clip_by_midi_clip_id(mc.clip_id) { - let instance_id = midi_id_to_doc.get(&mc.id) - .copied() - .unwrap_or_else(|| midi_backend_uuid(mc.id)); - let mut ci = ClipInstance::new(clip_id); - ci.id = instance_id; - ci.timeline_start = mc.external_start; - ci.trim_start = mc.internal_start.beats_to_f64(); - ci.trim_end = Some(mc.internal_end.beats_to_f64()); - // Always set timeline_duration for MIDI clips: duration is in beats, so we - // must bypass the content_window_secs * bpm/60 formula (which expects seconds). - ci.timeline_duration = Some(mc.external_duration); - instances.push(ci); - } - } - } - - // Only insert if we found clips (so layer_clips() can fall back to al.clip_instances - // for layers where the snapshot has no clips yet, e.g. during recording setup) - if !instances.is_empty() { - cache.insert(layer_id, instances); - } - } - - cache -} - -/// Get clip instances for a layer, using the snapshot-based cache for audio layers -/// and falling back to the doc's `clip_instances` if the cache has no entry OR is empty -/// while the doc has clips (e.g., a recording clip not yet reflected in the snapshot). -fn layer_clips<'a>( - layer: &'a AnyLayer, - audio_cache: &'a HashMap>, -) -> &'a [ClipInstance] { +/// The document is the single source of truth for every layer type: clip actions mutate +/// `*Layer::clip_instances`, it's what persists to the `.beam` file, and it drives the audio +/// backend on load. Recording clips grow here via the RecordingProgress/MidiRecordingProgress +/// mirror. (Audio used to be reconstructed from a backend snapshot, which dropped the doc-only +/// in-progress recording clip once a layer had any finalized clip.) +fn layer_clips(layer: &AnyLayer) -> &[ClipInstance] { match layer { - AnyLayer::Audio(al) => { - match audio_cache.get(&al.layer.id) { - Some(cached) if !cached.is_empty() => cached.as_slice(), - // Cache empty or missing: fall back to doc (covers recording-in-progress) - _ => &al.clip_instances, - } - } + AnyLayer::Audio(l) => &l.clip_instances, AnyLayer::Vector(l) => &l.clip_instances, AnyLayer::Video(l) => &l.clip_instances, AnyLayer::Effect(l) => &l.clip_instances, @@ -703,28 +603,26 @@ fn layer_clips<'a>( /// Returns (&AnyLayer, &[ClipInstance]) so callers have access to both layer info and clips. fn all_layer_clip_instances<'a>( context_layers: &[&'a AnyLayer], - audio_cache: &'a HashMap>, ) -> Vec<(&'a AnyLayer, &'a [ClipInstance])> { let mut result = Vec::new(); for &layer in context_layers { - collect_clip_instances(layer, audio_cache, &mut result); + collect_clip_instances(layer, &mut result); } result } fn collect_clip_instances<'a>( layer: &'a AnyLayer, - audio_cache: &'a HashMap>, result: &mut Vec<(&'a AnyLayer, &'a [ClipInstance])>, ) { match layer { - AnyLayer::Audio(_) => result.push((layer, layer_clips(layer, audio_cache))), + AnyLayer::Audio(l) => result.push((layer, &l.clip_instances)), AnyLayer::Vector(l) => result.push((layer, &l.clip_instances)), AnyLayer::Video(l) => result.push((layer, &l.clip_instances)), AnyLayer::Effect(l) => result.push((layer, &l.clip_instances)), AnyLayer::Group(g) => { for child in &g.children { - collect_clip_instances(child, audio_cache, result); + collect_clip_instances(child, result); } } AnyLayer::Raster(_) => {} @@ -1312,7 +1210,6 @@ impl TimelinePane { content_rect: egui::Rect, header_rect: egui::Rect, editing_clip_id: Option<&uuid::Uuid>, - audio_cache: &HashMap>, ) -> Option<(ClipDragType, uuid::Uuid)> { let context_layers = document.context_layers(editing_clip_id); let rows = build_timeline_rows(&context_layers); @@ -1342,7 +1239,7 @@ impl TimelinePane { }; let _layer_data = layer.layer(); - let clip_instances = layer_clips(layer, audio_cache); + let clip_instances = layer_clips(layer); // Check each clip instance let stacking = compute_clip_stacking(document, layer, clip_instances); @@ -2798,7 +2695,6 @@ impl TimelinePane { waveform_stereo: bool, context_layers: &[&lightningbeam_core::layer::AnyLayer], video_manager: &std::sync::Arc>, - audio_cache: &HashMap>, playback_time: f64, ) -> (Vec<(egui::Rect, uuid::Uuid, f64, f32)>, Vec) { let painter = ui.painter().clone(); @@ -3160,7 +3056,7 @@ impl TimelinePane { ]; for child in &g.children { if let AnyLayer::Audio(_) = child { - for ci in layer_clips(child, &audio_cache) { + for ci in layer_clips(child) { let audio_clip = match document.get_audio_clip(&ci.clip_id) { Some(c) => c, None => continue, @@ -3313,7 +3209,7 @@ impl TimelinePane { }; // Draw clip instances for this layer - let clip_instances = layer_clips(layer, &audio_cache); + let clip_instances = layer_clips(layer); // For moves, precompute the clamped offset so all selected clips move uniformly let group_move_offset = if self.clip_drag_state == Some(ClipDragType::Move) { @@ -4192,7 +4088,6 @@ impl TimelinePane { audio_controller: Option<&std::sync::Arc>>, context_layers: &[&lightningbeam_core::layer::AnyLayer], editing_clip_id: Option<&uuid::Uuid>, - audio_cache: &HashMap>, ) { // Only allocate content area (ruler + layers) with click and drag let content_response = ui.allocate_rect( @@ -4262,7 +4157,7 @@ impl TimelinePane { let _layer_data = layer.layer(); // Get clip instances for this layer - let clip_instances = layer_clips(layer, &audio_cache); + let clip_instances = layer_clips(layer); // Check if click is within any clip instance let click_stacking = compute_clip_stacking(document, layer, clip_instances); @@ -4548,7 +4443,6 @@ impl TimelinePane { content_rect, header_rect, editing_clip_id, - &audio_cache, ) { // If this clip is not selected, select it (respecting shift key) if !selection.contains_clip_instance(&clip_id) { @@ -4566,7 +4460,7 @@ impl TimelinePane { if drag_type == ClipDragType::Move { // Find earliest selected clip as snap anchor for quantized moves let mut earliest = f64::MAX; - for (_, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { + for (_, clip_instances) in all_layer_clip_instances(context_layers) { for ci in clip_instances { if selection.contains_clip_instance(&ci.id) { let start_secs = document.tempo_map().beats_to_seconds(ci.timeline_start).seconds_to_f64(); @@ -4596,7 +4490,7 @@ impl TimelinePane { self.drag_offset = 0.0; // Find earliest selected clip as snap anchor let mut earliest = f64::MAX; - for (_, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { + for (_, clip_instances) in all_layer_clip_instances(context_layers) { for ci in clip_instances { if selection.contains_clip_instance(&ci.id) { let start_secs = document.tempo_map().beats_to_seconds(ci.timeline_start).seconds_to_f64(); @@ -4630,7 +4524,7 @@ impl TimelinePane { let move_offset = self.snapped_move_offset(document.tempo_map(), &document.time_signature, document.framerate); // Iterate through all layers (including group children) to find selected clip instances - for (layer, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { + for (layer, clip_instances) in all_layer_clip_instances(context_layers) { let layer_id = layer.id(); // Find selected clip instances in this layer for clip_instance in clip_instances { @@ -4672,7 +4566,7 @@ impl TimelinePane { > = HashMap::new(); // Iterate through all layers (including group children) to find selected clip instances - for (layer, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { + for (layer, clip_instances) in all_layer_clip_instances(context_layers) { let layer_id = layer.id(); // Find selected clip instances in this layer @@ -4798,7 +4692,7 @@ impl TimelinePane { ClipDragType::LoopExtendRight => { let mut layer_loops: HashMap> = HashMap::new(); - for (layer, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { + for (layer, clip_instances) in all_layer_clip_instances(context_layers) { let layer_id = layer.id(); for clip_instance in clip_instances { @@ -4872,7 +4766,7 @@ impl TimelinePane { // Extend loop_before (pre-loop region) let mut layer_loops: HashMap> = HashMap::new(); - for (layer, clip_instances) in all_layer_clip_instances(context_layers, &audio_cache) { + for (layer, clip_instances) in all_layer_clip_instances(context_layers) { let layer_id = layer.id(); for clip_instance in clip_instances { @@ -5139,7 +5033,7 @@ impl TimelinePane { if response.double_clicked() { if let Some(pos) = response.interact_pointer_pos() { let over_clip = self - .detect_clip_at_pointer(pos, document, content_rect, header_rect, editing_clip_id, &audio_cache) + .detect_clip_at_pointer(pos, document, content_rect, header_rect, editing_clip_id) .is_some(); if !over_clip { self.fit_to_project(content_rect.width()); @@ -5168,7 +5062,6 @@ impl TimelinePane { content_rect, header_rect, editing_clip_id, - &audio_cache, ) { match drag_type { ClipDragType::TrimLeft | ClipDragType::TrimRight => { @@ -5561,31 +5454,10 @@ impl PaneRenderer for TimelinePane { // Use virtual row count (includes expanded group children) for height calculations let layer_count = build_timeline_rows(&context_layers).len(); - // Build audio clip cache from backend snapshot (backend-as-source-of-truth for audio). - // Uses doc UUIDs via reverse lookup of clip_instance_to_backend_map so that selection - // and action dispatch continue to work with doc UUIDs. - // Falls back to AudioLayer::clip_instances for layers with no snapshot data yet - // (e.g., layers where recording is in progress but not yet finalized). - let audio_cache: HashMap> = - if let Some(snap_arc) = shared.clip_snapshot.as_ref() { - if let Ok(snap) = snap_arc.read() { - build_audio_clip_cache( - &snap, - shared.layer_to_track_map, - document, - shared.clip_instance_to_backend_map, - ) - } else { - HashMap::new() - } - } else { - HashMap::new() - }; - // Calculate project duration from last clip endpoint across all layers let mut max_endpoint: f64 = 10.0; // Default minimum duration for &layer in &context_layers { - let clip_instances = layer_clips(layer, &audio_cache); + let clip_instances = layer_clips(layer); for clip_instance in clip_instances { let clip_duration = effective_clip_duration(document, layer, clip_instance); @@ -5660,7 +5532,7 @@ impl PaneRenderer for TimelinePane { // Render layer rows with clipping ui.set_clip_rect(content_rect.intersect(original_clip_rect)); - let (video_clip_hovers, pending_lane_renders) = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.focus, shared.selection, shared.midi_event_cache, shared.raw_audio_cache, shared.waveform_gpu_dirty, shared.waveform_minmax_pools, shared.target_format, shared.waveform_stereo, &context_layers, shared.video_manager, &audio_cache, *shared.playback_time); + let (video_clip_hovers, pending_lane_renders) = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.focus, shared.selection, shared.midi_event_cache, shared.raw_audio_cache, shared.waveform_gpu_dirty, shared.waveform_minmax_pools, shared.target_format, shared.waveform_stereo, &context_layers, shared.video_manager, *shared.playback_time); // Render playhead on top (clip to timeline area) ui.set_clip_rect(timeline_rect.intersect(original_clip_rect)); @@ -5852,7 +5724,6 @@ impl PaneRenderer for TimelinePane { shared.audio_controller, &context_layers, editing_clip_id.as_ref(), - &audio_cache, ); // Render automation lanes AFTER handle_input so our ui.interact registers last and wins @@ -5930,7 +5801,7 @@ impl PaneRenderer for TimelinePane { if secondary_clicked { if let Some(pos) = ui.input(|i| i.pointer.interact_pos()) { if content_rect.contains(pos) { - if let Some((_drag_type, clip_id)) = self.detect_clip_at_pointer(pos, document, content_rect, layer_headers_rect, editing_clip_id.as_ref(), &audio_cache) { + if let Some((_drag_type, clip_id)) = self.detect_clip_at_pointer(pos, document, content_rect, layer_headers_rect, editing_clip_id.as_ref()) { // Right-clicked on a clip if !shared.selection.contains_clip_instance(&clip_id) { shared.selection.select_only_clip_instance(clip_id); @@ -5983,7 +5854,7 @@ impl PaneRenderer for TimelinePane { if let Some(pos) = long_press { use crate::menu::MenuAction; let mut items: Vec<(String, MenuAction)> = Vec::new(); - if let Some((_drag_type, clip_id)) = self.detect_clip_at_pointer(pos, document, content_rect, layer_headers_rect, editing_clip_id.as_ref(), &audio_cache) { + if let Some((_drag_type, clip_id)) = self.detect_clip_at_pointer(pos, document, content_rect, layer_headers_rect, editing_clip_id.as_ref()) { if !shared.selection.contains_clip_instance(&clip_id) { shared.selection.select_only_clip_instance(clip_id); } @@ -6019,7 +5890,7 @@ impl PaneRenderer for TimelinePane { let mut enabled = false; if let Some(layer_id) = *shared.active_layer_id { if let Some(layer) = document.get_layer(&layer_id) { - let instances = layer_clips(layer, &audio_cache); + let instances = layer_clips(layer); for inst in instances { if !shared.selection.contains_clip_instance(&inst.id) { continue; } if let Some(dur) = document.get_clip_duration(&inst.clip_id) { @@ -6043,7 +5914,7 @@ impl PaneRenderer for TimelinePane { let mut enabled = false; if let Some(layer_id) = *shared.active_layer_id { if let Some(layer) = document.get_layer(&layer_id) { - let instances = layer_clips(layer, &audio_cache); + let instances = layer_clips(layer); // Check each selected clip enabled = instances.iter() .filter(|ci| shared.selection.contains_clip_instance(&ci.id)) From 8ed2320dca3d4f55ab3428c8e56bcc0c2a2e1cc6 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Sat, 11 Jul 2026 17:08:09 -0400 Subject: [PATCH 6/8] Type AudioClip duration by clip kind; fix MIDI clips growing too fast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MIDI-clip-end-grows-too-fast bug was a units confusion: AudioClip.duration is documented "seconds" but MIDI clips store their length in BEATS (they share the AudioClip struct with sampled clips). The timeline's effective_clip_duration read it as raw seconds, so at 120 BPM a MIDI clip rendered ~2x too long. The old backend-snapshot display had hidden this by forcing timeline_duration in beats. Root fix — make the domain explicit and unforgeable: - `duration` is now a private field. Reading it in the wrong unit is impossible because access goes through typed accessors: AudioClip::content_duration() -> ClipDuration (a Seconds|Beats enum tagged by clip_type) and set_content_duration(). serde still serializes the private field, so the .beam format is unchanged (bare number). - ClipDuration::to_seconds(tempo_map) for display/sizing; ::native() for code that already works in the clip's native domain (trim math shares it). - get_clip_duration + the timeline-endpoint calc go through to_seconds, so MIDI is converted correctly; effective_clip_duration delegates to get_clip_duration. - Recording mirrors (audio/MIDI progress + finalize) write via set_content_duration (debug-asserts the value's domain matches the clip type). Every former raw read/write of the field (core actions, timeline, piano roll, infopanel, asset library, recording handlers) now goes through the accessors. Whole workspace compiles; 299 core tests pass. --- STREAMING_TO_DISK_PLAN.md | 859 ------------------ TODO.md | 310 +------ daw-backend/Cargo.lock | 10 + lightningbeam-ui/Cargo.lock | 2 +- lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md | 68 +- .../src/actions/add_clip_instance.rs | 4 +- .../src/actions/loop_clip_instances.rs | 2 +- .../src/actions/remove_clip_instances.rs | 4 +- .../src/actions/split_clip_instance.rs | 10 +- .../src/actions/trim_clip_instances.rs | 6 +- .../lightningbeam-core/src/clip.rs | 66 +- .../lightningbeam-core/src/document.rs | 24 +- .../lightningbeam-editor/src/main.rs | 9 +- .../src/panes/asset_library.rs | 12 +- .../src/panes/infopanel.rs | 2 +- .../src/panes/piano_roll.rs | 4 +- .../src/panes/timeline.rs | 10 +- 17 files changed, 186 insertions(+), 1216 deletions(-) delete mode 100644 STREAMING_TO_DISK_PLAN.md diff --git a/STREAMING_TO_DISK_PLAN.md b/STREAMING_TO_DISK_PLAN.md deleted file mode 100644 index abd9921..0000000 --- a/STREAMING_TO_DISK_PLAN.md +++ /dev/null @@ -1,859 +0,0 @@ -# Streaming Media To/From Disk — Plan - -**Goal:** Lightningbeam must handle audio and video files (and raster animation, and -image assets) of *arbitrary length/size*. Anywhere we touch media we should stream from -and to disk when the data is too large to fit comfortably in memory, rather than loading -the entire file regardless of size. - -**Scope of this document:** audio, video, raster frames, image-asset paging, **and the -`.beam` container format** — these turned out to be one problem, not two. Streaming on load -is impossible while the container forces a full decode, so the container decision (below) -is now part of this plan. - -## Deferred bugs (do at the end) -- [x] **Timeline thumbnail scroll (FIXED):** the strip tiled from the *clamped* visible-left of the - clip, so when a clip was scrolled partly off the left it showed the clip's start content at the - viewport edge. Now tiled from the clip's **true (unclamped) origin** over its full width, drawing - only the tiles intersecting the visible rect (`draw_video_thumbnail_strip` in timeline.rs). Both - render sites (collapsed-group + expanded-track) share the helper. *(Compiles; needs in-app check.)* -- [x] **Clip thumbnails stop updating (FIXED):** the GPU texture cache was keyed by the *requested* - content time, so once a tile cached the first (often far-off) thumbnail it never refreshed as - closer ones loaded. `VideoManager::get_thumbnail_at` now also returns the **actual** thumbnail - timestamp, and the cache keys on that — so a tile picks up a new texture when a closer thumbnail - finishes generating. Existing `retain`-by-visible-clip cleanup keeps it bounded. *(Needs in-app check.)* - -## Raster-keyframe-UI bugs — **[DONE]** (built the raster keyframe timeline UI, 2026-06-20) -Both resolved by the raster-keyframe-timeline-UI work: timeline now draws a diamond per -`RasterKeyframe` (mirrors vector), `K`/New Keyframe inserts a blank cel via `AddRasterKeyframeAction` -(canvas refreshes), paint tools edit the active keyframe instead of lazily creating, diamonds are -click-to-seek (pointing-hand cursor), playback prefetches frames, and onion skinning (raster+vector, -tinted, Info-Panel settings) is in. (a) canvas-refresh-on-new-keyframe and (b) keyframes-on-timeline -are both fixed. - -## Noted enhancements (later, after the phases) -- [x] **Surround → stereo downmix (DONE).** Done uniformly in `render_from_file` (`pool.rs`) so it - covers every storage type (PCM/InMemory, compressed via symphonia, video-audio via ffmpeg — all - flow through this mixer with the source kept multichannel in the read-ahead buffer). New - `stereo_downmix_matrix(src_channels)` gives `[L][src]`/`[R][src]` coefficients for the conventional - interleave order (FL FR FC LFE BL BR SL SR…) for 3/4/5/5.1/6.1/7.1: full level for the matching - front, `1/√2` for centre + each surround, LFE dropped; each row normalized so |coef| sum ≤ 1 to - prevent clipping (matches ffmpeg's default). Applied in both the direct-copy and sinc-resample - paths (only when `dst==2 && src>2`; unknown layouts fall back to front L/R). Compiles clean. - *(Needs in-app check: a 5.1 file now has centre/dialog present and isn't thin; not distorted/clipping.)* - Native multichannel support remains a separate, larger project. -- **Export speed (audited 2026-06-21):** a 1:14 1080p MP4 took ~9:06 (~7.4x realtime, ~135 ms/frame). - Audit **refuted** the per-frame-seek theory — export decodes the source *sequentially* - (`video.rs` `need_seek` is false once advancing forward), and readback is already async + - triple-buffered. Real hotspots: - - **[DONE] #1 — per-frame renderer rebuild.** The export pump built a fresh `vello::Renderer` - (full wgpu pipeline init) + empty `ImageCache` *every egui repaint* (`main.rs` ~6218). Now built - once per export and reused; also fixed lazy-image export (the throwaway cache had no container - path). **Expected the dominant win.** - - **[DONE] #2a — encode swscale rebuilt per frame.** `CpuYuvConverter::convert` now caches the - RGBA→YUV420p `scaling::Context` + frames in `new()` instead of per call. - - **[TODO] #2b — decode swscale + stride-repack** per frame in `video.rs:294-320` (shared with - scrubbing; cache the YUV→RGBA scaler on the decoder). Small win, modest risk. - - **Result of #1+#2a (measured):** ~7.4x → **~1.74x realtime** (130.7 s for 4488 frames @ 60 fps; - 34 fps). Per-stage avg: Render(CPU build) 15 ms, **Readback(GPU latency) 42 ms**, Extract 1.3 ms, - Convert 5.7 ms. - - **Now GPU-bound.** Per ~87 ms poll cycle the CPU does ~66 ms (3× build 45 + convert 17 + extract 4) - but the GPU does ~87 ms (3 × ~29 ms composite) → GPU saturated at ~29 ms/frame; "Readback 42 ms" is - queue latency, not transfer (8 MB is sub-ms). - - **[SKIP] #3 GPU YUV / #5 pacing** — both only trim the CPU side, which is already *under* the GPU. - Won't move a GPU-bound throughput. - - **[TODO, big] Reduce the GPU composite (~29 ms/frame).** The per-layer HDR pipeline (Vello render → - linear → composite, ×layers) is the wall, shared with live rendering. Options: batch composite - passes; a fast-path skipping HDR compositing for simple single-layer/no-blend docs; cache unchanged - layers' scenes (CPU-side, only helps if it later becomes CPU-bound). Render-architecture project. - - Non-issues: per-frame seek, blocking readback, audio. (`video.rs:237` container-reopen-on-seek is - a latent cost but doesn't fire on forward export.) -- **AAC export NaN guard (done):** `convert_chunk_to_planar_f32` now sanitizes non-finite samples - (NaN/Inf → 0, finite clamped to [-1,1]) like the integer paths, with a one-time warning — a stray - non-finite render sample no longer fails the whole export. Upstream NaN source (effect/automation/ - decode) still worth chasing if it recurs. -- [x] **Persist video thumbnails (DONE).** Mirrors waveform persistence: each clip's thumbnails are - PNG-encoded + packed into one opaque `LBTN` blob (editor owns the format; `encode/decode_thumbnail_blob` - in main.rs), stored as a `MediaKind::Thumbnail` row keyed by `thumbnail_media_id(clip_id)` (clip id XOR - a fixed sentinel). Save: a cheap Arc-clone snapshot (`VideoManager::snapshot_all_thumbnails`) rides the - `FileCommand::Save`, PNG-encoded off the UI thread in the worker, written by `save_beam` (kept in place - on re-save). Load: `load_beam_sqlite` reads the packs into `LoadedProject.thumbnail_blobs`; the editor - decodes + `insert_thumbnail`s them on a background thread and **gates regeneration** (`register_loaded_videos` - skips clips with persisted thumbnails). Bonus: thumbnails show even if the source video file is missing. - **Partial sets are persisted and resumed** (not thrown away): the `LBTN` blob (v2) carries a `complete` - flag (`VideoManager.thumbnails_complete`, marked when the keyframe pass finishes). On load, complete - packs are restored + skip regeneration; *partial* packs are restored AND generation is resumed — - `generate_keyframe_thumbnails` takes a `should_skip` predicate (`has_thumbnail_near`) so it only decodes - the keyframes not already covered. `insert_thumbnail` is now sorted + idempotent (fixes a latent - unsorted-`binary_search` bug and makes concurrent restore + resume race-safe). So a save 50 min into a - 2 h video keeps that work and continues from there on reload. - Container tests still green; all crates compile. *(Needs in-app check: reload = instant thumbnails for - complete clips; a mid-generation save resumes from where it left off on reload.)* - **Size assessment (done):** thumbnails are 128px wide, height by aspect (72px at 16:9 → - 128×72×4 ≈ **36 KB raw** each; 4:3 ≈ 49 KB), generated **one per ~5 s** (capped `interval_secs`, - at keyframes — so ~12/min). Raw: ~0.5 MB per 1:14 clip, ~26 MB/hour, ~52 MB/2 h. Compressed for - on-disk: JPEG ~3–6 KB/thumb → **~6 MB/2 h**; PNG ~8–15 KB → ~14 MB/2 h. So persistence is cheap - (≤ the waveform's ~36 MB/2 h), especially as JPEG. Plan: encode each clip's thumbnails (JPEG) + - their timestamps into one blob, a new `MediaKind::Thumbnail` row keyed by the clip/media id (mirror - the waveform persistence: write on save, restore via `insert_thumbnail` on load, regenerate if - absent). The 5 s interval already bounds count; no extra budget needed. -- **Progressive waveform on first import:** generation streams the whole file before the - waveform appears (several seconds for large files). Since `build_waveform_pyramid` already - streams, emit partial floors as it advances (e.g. flush every N seconds of decoded audio via - the existing `waveform_result` channel + chunked GPU upload) so the overview fills in across - the clip left-to-right instead of appearing all at once. Persistence saves only the final - complete pyramid. - -## Guiding principle -Three subsystems already have the right streaming primitive; most of the work is wiring, -bounding caches, and adding a residency window. The recurring pattern: - -> Keep tiny metadata always-resident, fault the heavy payload in on demand keyed by a -> stable ID, and evict everything outside a window around the playhead. - ---- - -## Audit summary (where we stand today) - -### Correctly streaming / bounded -- Video frame decode/seek/playback (`lightningbeam-core/src/video.rs:191` `get_frame` — - keyframe-index seek + decode-until-target, one frame resident). -- WAV/AIFF import via mmap (`daw-backend/src/audio/engine.rs:2328`). -- Webcam capture encodes directly to disk (`lightningbeam-core/src/webcam.rs`). -- `WaveformCache` (100MB cap), decoder `LruCache` (20 frames), export render loop (≤3 - frames in flight). -- The compressed-audio disk reader `daw-backend/src/audio/disk_reader.rs` - (`CompressedReader` + 3s `ReadAheadBuffer`) — **correct but never activated** (Phase 1a). - -### Fully-loaded, unbounded by file length (the problems) -| Site | Issue | -|---|---| -| `daw-backend/src/io/audio_file.rs:344` `decode_progressive` | Decodes whole compressed file into a `Vec`; de-facto playback source. | -| `daw-backend/src/audio/pool.rs:1071` `load_file_into_pool` | Every audio file in a saved project fully decoded to `InMemory` on open. | -| `lightningbeam-core/src/video.rs:711` `extract_audio_from_video` | Whole video audio track into one `Vec`. | -| `lightningbeam-core/src/video.rs:412` `VideoManager.frame_cache` | Unbounded `HashMap` of full-res RGBA frames; grows while scrubbing. | -| `export/mod.rs:388-400` | Mux step buffers all compressed packets into `Vec`s; O(duration). | -| `lightningbeam-core/src/raster_layer.rs:115` `RasterKeyframe.raw_pixels` | ~8MB/frame at 1080p; all keyframes decoded from PNG at load (`file_io.rs:611-640`), never evicted. | -| `lightningbeam-editor/src/gpu_brush.rs:1051` `raster_layer_cache` | Unbounded GPU texture `HashMap`. | -| `lightningbeam-core/src/renderer.rs:25` `ImageCache` | Unbounded decoded image cache (asset textures). | -| `Document.image_assets` (`document.rs:206`) | Every image asset's compressed bytes resident for document life. | - ---- - -## Container format decision: `.beam` → SQLite *(DECIDED)* - -The `.beam` container moves from a **ZIP archive** to a **SQLite database file** (same -`.beam` extension). This is the foundation the rest of the plan builds on. - -### Why -ZIP can stream `Stored` entries in place (via `data_start()`), but it has **no in-place -mutation** — every save and every raster frame write-back rewrites the whole archive — and -embedded PCM is rarely mmap-aligned. The current load path is even worse: it reads each -ZIP audio entry fully, decodes FLAC → re-encodes WAV → base64 → base64-decodes → temp file -→ full Symphonia decode → resident `Vec` (`file_io.rs:513-604`, `pool.rs:1071`). - -SQLite dissolves the single-file-vs-performance tension: -- **Single file** — beginner-friendly, behaves like a file on every OS (no package-folder - confusion; we have no bundle magic on Linux/Windows). -- **Streaming reads** — `sqlite3_blob_open` / `blob_read(offset, len)` gives seekable, - chunked reads through the pager (mmap mode for the DB). For chunked streaming the - pager-copy is negligible vs. decode cost, so the lack of zero-copy mmap doesn't matter. -- **Cheap, crash-safe mutation** — raster frame write-back is a transactional `UPDATE`; - save is a metadata write + dirty-blob updates. **ACID** means a force-quit / power loss / - crash mid-save can't corrupt the project (ZIP and package-dirs both have to hand-roll - atomicity). -- **Inspectable / scriptable** — `sqlite3` CLI; `beam_inspector.py` can read it directly. - -**Net effect: there is no scratch directory anywhere in this plan.** Media stream via blob -reads (or external paths); raster frames live in blob rows and write back transactionally. - -### Large-media policy: packed OR referenced -Two storage modes per media item, both supported: -- **Packed** — bytes live in the DB. To stay under SQLite's ~2GB per-blob ceiling (and to - make reads naturally chunked), large media is split into **multiple blob-chunk rows** - (e.g. 64 MB/chunk); streaming reads address `(chunk_index, offset)`. -- **Referenced** — the DB stores only a path; bytes stay on disk (useful for shared media - on a network drive, or media too large/volatile to pack). - -**Default-mode preference for files over the per-blob limit (~2GB):** -- A user preference `large_media_default: Pack | Reference` controls what happens to - imports above the threshold. -- The **first time** the user imports a media file over the limit, **prompt** them - (Pack vs Reference), apply it, and **persist the choice** as the preference for future - large imports (changeable later in settings). -- Files under the limit are packed by default (chunked only if needed). - -### Schema sketch -``` -media( - id BLOB PRIMARY KEY, -- stable Uuid - kind INTEGER, -- audio | video | raster | image-asset - codec TEXT, -- "flac","mp3","png",... (original, lossless-preserving) - storage INTEGER, -- 0 = packed, 1 = referenced - ext_path TEXT, -- set when storage = referenced - total_len INTEGER, -- bytes (packed) for chunk math - channels INTEGER, sample_rate INTEGER, width INTEGER, height INTEGER -- kind-specific meta -) -media_chunk( - media_id BLOB, chunk_index INTEGER, bytes BLOB, - PRIMARY KEY (media_id, chunk_index) -) -project_json(id INTEGER PRIMARY KEY CHECK (id = 0), data TEXT) -- existing project.json, verbatim -meta(key TEXT PRIMARY KEY, value TEXT) -- version, created, modified -``` -`project.json` stays the same serialized `BeamProject` for now — only its container and the -media storage change. A migration reads a legacy ZIP `.beam` and writes the SQLite form on -first open/save. - -### Streaming reads from packed media -A `BlobReader` implementing `Read + Seek` over `media_chunk` rows feeds the existing -streaming consumers unchanged: `CompressedReader` (audio) decodes from it instead of a -`File`; the video decoder seeks within it; raster `UPDATE`s a chunk. Referenced media uses a -plain `File` exactly as `do_import_audio` already does for originals today. - ---- - -## Phase 1 — Audio: activate what already exists *(highest impact, lowest effort)* - -### 1a. Turn on the compressed-audio disk reader -The `CompressedReader` + 3-second `ReadAheadBuffer` in `disk_reader.rs` is complete but -never invoked (`DiskReaderCommand::ActivateFile` / `DiskReader::create_buffer` are never -called; `AudioClip::read_ahead` at `clip.rs:63` is hard-wired to `None`). -- On compressed import (`engine.rs:2381`) and during playback setup, activate the file and - assign `AudioClip::read_ahead`. -- Change `decode_progressive` (`io/audio_file.rs:344`) to produce only the downsampled - waveform overview (min/max peaks) the UI needs, then drop decoded PCM. Playback comes - from the ring buffer, not RAM. -- Verify `render_from_file` (`pool.rs:449`) reads from `read_ahead` when `data()` is empty. - -**Risk:** the real-time thread must never block on disk. The ring buffer prefetches ~2s -ahead; underruns degrade to silence (live) or block-wait (export), which `disk_reader.rs` -already distinguishes. - -### 1b. Stream on project load *(depends on the SQLite container)* -Three coupled changes (none works alone): -1. Replace `load_file_into_pool`'s full decode (`pool.rs:1071`) with the same branching as - `do_import_audio`: PCM → mmap (referenced) or in-memory for tiny packed PCM; compressed - (incl. FLAC) → `from_compressed` placeholder backed by a `BlobReader` (packed) or `File` - (referenced). The claxon FLAC→WAV→base64 round-trip in `file_io.rs:533-591` is deleted. -2. **Bulk read-ahead activation:** loaded clips are deserialized directly - (`audio_backend.project`), bypassing `AddAudioClip`, so the Phase 1a wiring never fires - for them. After the engine installs the project, walk all audio clips and - `create_buffer` + `ActivateFile` + set `read_ahead` for every clip referencing a - `Compressed` pool entry. (`CompressedReader::open` needs a variant that takes a - `BlobReader` instead of a path for packed media.) -3. Pool entries carry storage mode (packed-chunks vs referenced path) from the `media` - table instead of base64 `embedded_data`. - -### 1c. Video's embedded audio track — stream from the video via ffmpeg - -**Interim stopgap (shipped):** `extract_audio_from_video_to_wav` streams the decoded audio to -a temp WAV, imported via `import_audio_sync` (mmap). Fixes the RAM OOM but writes the whole -uncompressed track to `/tmp` (fills small temp partitions) and the temp path doesn't survive -save/reload. **Superseded by the design below.** - -**Proper design — stream the video's audio track on demand, never materialized.** - -*Enabler:* `daw-backend` already depends on `ffmpeg-next` (used for MP3/AAC encoding), so the -ffmpeg audio decoder lives beside `CompressedReader` in `daw-backend/src/audio/`. No -cross-crate work (`core → daw-backend` is one-way). `CompressedReader` already has the needed -interface. - -1. **`VideoAudioReader` (ffmpeg)** — mirrors `CompressedReader`: - `open(path)`, `decode_next(&mut Vec) -> frames` (resample → interleaved f32 at native - rate; reuse the old extraction resampler), `seek(target_frame) -> actual`, - `sample_rate`/`channels`/`total_frames`. -2. **Source dispatch:** `enum StreamSource { Compressed(CompressedReader), Video(VideoAudioReader) }` - (or a small `trait AudioFrameSource`) held by the reader thread; ring buffer / prefetch / - export-blocking unchanged. `DiskReaderCommand::ActivateFile` gains a `kind: SourceKind`. -3. **Pool model:** `AudioStorage::VideoAudio { video_path, decoded_for_waveform, decoded_frames, - total_frames }` (near-copy of `Compressed`); `data()` empty, playback via `read_ahead`. Pool - entry `path` = the video file. -4. **Engine API:** `EngineController::add_video_audio_sync(video_path) -> usize` — ffmpeg-probe - the audio track (rate/channels/frames/duration, no decode), build the pool entry, return index. -5. **Clip activation:** extend the Phase 1a `AddAudioClip` wiring — if entry is `VideoAudio`, - make the buffer + `ActivateFile{kind:VideoAudio, path:video_path}` + set `clip.read_ahead`. - One ffmpeg context + 3 s buffer per active clip instance. -6. **Import flow:** `import_video` calls `add_video_audio_sync(video_path)` → - `AudioClip::new_sampled`. **Remove** `extract_audio_from_video_to_wav`, the temp-WAV - handling, and the now-dead `add_audio_file_sync`. No WAV / `/tmp` / RAM. -7. **Save/load:** the `VideoAudio` entry serializes as a path reference to the video (no media - bytes — the video is already referenced by its `VideoClip`); reconstruct on load by - re-probing. Fixes the stopgap's reload fragility (nothing to persist). -8. **Waveform overview:** background ffmpeg pass emitting **downsampled peaks only** (bounded - memory) into the existing waveform path — shared with the Phase 1a `decode_progressive` - cleanup. - -**Sample accuracy (required — video audio must stay frame-synced with other clips):** -Coarse ffmpeg seeks are NOT sufficient. `VideoAudioReader::seek(target_frame)` must: -- coarse-seek to a point ≤ target, then **decode-and-discard** to land exactly on - `target_frame`, tracking the absolute sample position from decoded-frame PTS (discard whole - frames before target; for the frame straddling target, drop its leading samples). After - `seek`, `decode_next` yields samples starting at exactly `target_frame`. -- This makes frame N of the video-audio pool entry correspond to the exact timeline position, - so it mixes sample-aligned with mmap/InMemory clips. Continuous decode advances frame-exact. -- *Consistency note:* `CompressedReader` should get the same decode-discard alignment (its - current coarse-seek-then-write-at-target can misalign by up to a GOP after a seek). Fold in - while here, or at least flag. - -*Model decision (confirmed):* the video's audio stays a **separate, editable `AudioClip`** on -an audio track, backed by the `VideoAudio` pool entry — users can move/trim/mute/detach it. - -*Build order:* `VideoAudioReader` + `StreamSource` → pool `VideoAudio` variant → -`add_video_audio_sync` + activation → swap `import_video` (remove WAV path) → sample-accurate -seek (both readers) → waveform-peaks pass. - ---- - -## Phase 2 — Video: bound the caches *(small, isolated)* - -### 2a. Bound `VideoManager.frame_cache` -`video.rs:412` — convert the unbounded `HashMap<(Uuid,i64), Arc>` to an LRU -mirroring the decoder-level cache (`video.rs:34`). Frame-count or byte budget. - -### 2b. Stream the export mux -`export/mod.rs:388-400` — interleave-write packets to the output as produced (compare PTS, -write the earlier stream) instead of collecting all then writing. O(duration) → O(1). - ---- - -## Phase 3 — Raster: disk-backed keyframe paging *(the heavy one)* **[locked design]** - -Today `load_beam_sqlite` (`file_io.rs:564`) eagerly `decode_png`s **every** raster keyframe's -`Raster` media row into `RasterKeyframe.raw_pixels` (`raster_layer.rs:115`, `w·h·4` ≈ 8 MB @ -1080p, `#[serde(skip)]`), never evicts, has an unbounded GPU texture cache, and holds full-frame -undo snapshots. `raw_pixels` is the working rep (edits write it, save reads it, render reads it), -`has_pixels()` = `!raw_pixels.is_empty()`, `keyframe_at` is a `partition_point` binary search, and -the container is opened only at load/save (no live handle). - -**Design (confirmed with user):** keep `raw_pixels` as the working rep; make residency explicit -via a `RasterStore` + an editor-run fault-in/evict pass *before* the immutable render. Async -fault-in (no scrub hitch), with a **low-res image proxy** shown until the full frame lands. -Decisions: small window (±~2 keyframes); **dirty (edited-unsaved) frames stay fully resident** -(spill-to-scratch deferred); fault-in is **async**; proxy is a **per-keyframe low-res RGBA image** -(PNG/WebP, correct alpha), NOT a video (VP9-alpha was rejected as finicky for negligible disk win). - -### Drive-by (Arc pixels): DROPPED -Investigated and rejected: `raw_pixels` has ~64 access sites, and most `.clone()`s genuinely need -an owned `Vec` (undo buffers, export, GPU readback) so `Arc>` would force `(*p).clone()` -and still copy. The only beneficiary, the per-frame `renderer.rs:550` Vello clone, is on the -**legacy/dead** path — the live HDR canvas renders raster as `RenderedLayerType::Raster` → GPU -upload in `stage.rs` which passes a `&[u8]` slice and uploads only on cache-miss (no per-frame -clone). Not worth 64 edits. Start at 3a. - -### 3a. Lazy async fault-in + image proxy -- **[DONE 3a-1]** Lazy load: full-decode removed; `raw_pixels` empty on load, `needs_fault_in` - armed recursively; canvas records misses → App pages in via `RasterStore.load_pixels`. -- **[DONE 3a-2]** Async: page-in runs on a background thread (deduped via `raster_loads_inflight`); - results applied at top of `update()`. No UI block on cold scrub. -- **[DONE 3a-3]** Image proxy: `MediaKind::RasterProxy` (≤192px PNG, derived id), written - beside each resident full PNG on save + eager-decoded on load into `RasterKeyframe::proxy`. - Separate `proxy_layer_cache` (own LRU, budget 64); the raster render blits the proxy mapped to - the keyframe's FULL logical dims (upscales via sampler) when the full texture isn't resident. - *(Proxies exist only after a save+reload; eager decode → lazy/paged is a refinement for huge - paint projects.)* - -- **`RasterStore`** (core): current `.beam` path + a read-only connection; `load_pixels(kf_id,w,h)` - reads the `Raster` row and `decode_png`s it. Set/cleared by the editor on load + save-as. -- **Save:** alongside the full PNG, write a low-res RGBA proxy per resident keyframe - (`MediaKind::RasterProxy`, ≤~480px long edge, keyed by `kf.id`). -- **Load:** stop eager full-decode; decode **proxies** eagerly (cheap → instant scrub everywhere); - leave full `raw_pixels` empty. -- **Fault-in pass** (editor, `&mut document` + store, each frame before render): for each raster - layer ensure the active keyframe ±N is requested; load full PNGs on a **background thread pool**; - on arrival, set `raw_pixels` + `texture_dirty`. Render uses full `raw_pixels` if resident, else the - upscaled proxy. Reused by the exporter (already frame-by-frame). - -### 3b. Residency window + eviction **[DONE]** -- Added `#[serde(skip)] dirty: bool` (edited-since-persist; distinct from `texture_dirty`). Set on - stroke/fill/paint-bucket/floating-lift commits + undo/redo; cleared on save (which re-arms the LRU). -- Implemented as a fault-in-recency **LRU** (`RASTER_RESIDENT_MAX = 12`), not a strict ±N window: - evict the oldest **clean** frame (drop `raw_pixels`, re-arm `needs_fault_in`); the shown frame is - always most-recent so it's protected; **dirty frames never evicted**. Save preserves evicted frames' - rows via `media_exists` (no data loss) and walks all layers to match load. - *(Refinement deferred: count budget → byte budget for 4K resolution-robustness.)* - -### 3c. Bound the GPU cache **[DONE for raster_layer_cache]** -`raster_layer_cache` (`gpu_brush.rs`, `HashMap`, Rgba16Float ping-pong -≈ `w·h·16`/entry, was **unbounded**) → recency LRU (`RASTER_LAYER_CACHE_MAX = 12`) in -`ensure_layer_texture`: bump-to-most-recent + evict oldest; shown frames protected. F3 overlay -now shows tracked VRAM (raster cache MB + count). *(Refinements: count→byte budget; raise/headroom -if >12 raster layers are visible at once. Export `raster_cache` lives one export — fine. Vello -`ImageCache` is image *assets* → Phase 4.)* - -### 3d. Undo memory **[DONE]** -`RasterStrokeAction`/`RasterFillAction` stored `buffer_before`+`buffer_after` full frames. -Now store a `RasterDiff` (`actions/raster_diff.rs`) — changed bbox before/after only, computed in -`new()`, full buffers dropped. Undo/redo apply onto the keyframe's resident pixels; the editor -faults the target frame in first (`Action::raster_resident_hint` + `peek_undo/redo_raster_hint`), -correct because a clean evicted frame's container bytes == its logical state. Non-resident base ⇒ -skip (no corruption). Unit-tested round-trip. *(Refinement: compress full-canvas-fill diffs, whose -bbox is the whole frame.)* - -### 3e. Prefetch frames **[DONE for playback]** -Implemented for playback: each update during playback, page in the next `PREFETCH_AHEAD=4` -upcoming keyframes per raster layer (reusing the async worker + `raster_loads_inflight` dedup), so -full frames are resident before the playhead arrives — fixes "proxy on every frame"/flicker during -playback. *(Caveat: with many simultaneous raster layers the 12-frame resident budget may evict a -prefetched frame before it's shown — raise budget or scale prefetch if that surfaces. Scrub-direction -prefetch still TODO.)* - -Original note: *(future, after 3d — pure latency win, no correctness need)* -Fault-in is reactive (page in only on a render miss), so a never-visited frame still shows the -proxy for a beat before the full lands. **Prefetch the full pixels for frames about to be shown**: -on scrub/playback, dispatch background page-ins for the active keyframe ±N in the direction of -playhead motion (and during playback, the next K keyframes), reusing the 3a-2 async worker + -`raster_loads_inflight` dedup. Keep prefetched frames in the 3b LRU so they're still bounded; cap -concurrent prefetch loads so scrubbing fast doesn't thrash the disk. Optional: also prewarm the GPU -texture (3c cache) for the immediate next frame. Net effect: cold scrubbing/playback shows full-res -frames with no proxy flicker. Proxy stays as the instant fallback when prefetch can't keep up. - -### Build order & tests -1. Arc drive-by — COW make_mut test. 2. 3a fault-in + store + proxy — load→empty-until-faulted, -PNG round-trip, proxy-then-swap. 3. 3b window/evict/dirty — residency ≤ window while scrubbing, -dirty never evicted. 4. 3c GPU bound. 5. 3d undo diffs reproduce pre-stroke buffer exactly. - ---- - -## Phase 3.5 — Image textures in vector scenes **[DONE 2026-06-21]** *(prereq for Phase 4; fixed DCEL-broken image import)* - -**Done:** 3.5a — import/drop places an image as a borderless image-filled rectangle -(`AddShapeAction::image_rect`), centered (direct import) or at the drop point (library drag); -renderer now maps the image brush onto the fill's bounding box (was anchored at world origin → -only a corner showed); `SetImageFillAction` + an **Image** fill-type tab (None|Solid|Gradient|Image) -with an asset picker in the Info Panel. 3.5b — image bytes persist as `MediaKind::ImageAsset` rows in -the `.beam` (kept-in-place; `ImageAsset.data` is `skip_serializing` + container-backed; old base64 -projects migrate on re-save); eager-read on load. *(ImageCache still unbounded — Phase 4 adds the -usage-based LRU/lazy paging.)* - -### (original plan below) -## Phase 3.5 — Image textures in vector scenes *(prereq for testing Phase 4; fixes DCEL-broken image import)* - -**Why:** Phase 4 pages *image assets*, but there's currently no way to get an image asset into a -vector scene — so nothing to page. This also repairs image import, half-broken since the DCEL switch. - -**Current state (audited 2026-06-21):** -- *Works:* `import_image` (`main.rs`) decodes dims + creates an `ImageAsset` (raw bytes embedded in - `Document::image_assets`, serialized as **base64 in project JSON**). The renderer's image-fill paths - are **complete** — GPU/Vello (`renderer.rs:~1160`, `ImageBrush` via `ImageCache.get_or_decode`) and - CPU/tiny-skia (`renderer.rs:~1486`). `Fill::image_fill` (`vector_graph/mod.rs:110`) and - `Face::image_fill` (`dcel2/mod.rs:117`) fields exist and render when set. -- *Broken/missing (the workflow):* - 1. **Drop image → canvas is stubbed:** `stage.rs:~11782` and `main.rs:~4924` both just print - "Image drag to stage not yet supported with DCEL backend". Nothing is added to the scene. - 2. **No way to assign an image fill:** no `SetImageFillAction` (only `SetFillPaintAction` for - color/gradient); no Info-Panel picker. `Fill`/`Face.image_fill` are never populated. - 3. **DCEL faces never get `image_fill`** (`dcel2/import.rs:275` always `None`; topology copies from - parent which is also `None`). - 4. **Not in the container:** `MediaKind::ImageAsset` exists but is **dead** — image bytes live only - as base64 in project JSON. Not chunked, not pageable (so Phase 4 can't page them). - -**Tasks:** -- **3.5a — Place + assign.** Replace the two drop stubs: dropping an image onto a vector layer creates - a rectangle face sized to the image at the drop point with `image_fill = asset_id`. Add - `SetImageFillAction` (set/clear an image fill on the selected face/shape; mirrors `SetFillPaintAction`) - + an Info-Panel image-asset picker for the selected shape's fill. Populate `Face.image_fill` in DCEL - (and keep it through topology ops — already copied from parent). -- **3.5b — Persist in the container.** Write image assets as `MediaKind::ImageAsset` rows in the `.beam` - SQLite (like raster/audio: write on save kept-in-place on re-save; read on load), keyed by asset id; - drop the base64-in-JSON embedding (or keep a tiny ref). This is the storage Phase 4 pages from. -- **3.5c — Lazy decode hook.** Image bytes load from the container into `ImageCache` on first render - (decode → `ImageBrush`/`Pixmap`). Leave `ImageCache` **unbounded for now**; Phase 4 adds the - usage-based LRU/eviction (this phase just makes there *be* real, container-backed image assets to page). -- **Tests:** import→drop→render round-trip; save/reload preserves the image fill + reads bytes from the - container (not JSON); CPU and GPU render paths both show the image. - ---- - -## Phase 4 — Asset paging by usage + LRU *(vector's real cost is assets, not geometry)* - -Vector geometry is compact flat POD (tens of KB/frame, no cached tessellation/DCEL) — leave -it resident. The heavy, evictable thing is the **image assets** referenced by fills. - -**Data model.** -- `ImageAsset` (`clip.rs:250`): `path: PathBuf` + `data: Option>` (whole compressed - file bytes) + dims. Imported fully into `data` at `main.rs:3936`. -- All assets resident in `Document.image_assets: HashMap` (`document.rs:206`). -- Decoded form in `ImageCache` (`renderer.rs:25`): `HashMap>` + CPU - `Pixmap` map, keyed by asset id, **unbounded**. -- A `Fill` references an asset by `image_fill: Option` (`vector_graph/mod.rs:110`). - Same UUID may appear in many fills/keyframes/layers and recursively through clip instances. - **No asset→frame or frame→asset index exists today.** - -**Two evictable tiers:** Tier 1 = compressed bytes (`ImageAsset.data`, droppable, reload -from blob row or external `path`); Tier 2 = decoded pixels (`ImageCache` + GPU textures — -the heavy one). - -**Progress (2026-06-21):** -- **[DONE] Tier 2 — bound the decoded `ImageCache`.** 256 MB **usage-LRU**: every - `get_or_decode`/`_cpu` bumps the asset's recency; inserts past budget evict the least-recently-used - (a miss re-decodes from `asset.data`). Achieves usage-based eviction via render-access recency - (simpler than the frame→asset enumeration below; that enumeration is only needed for *prefetch*). -- **[DONE] Tier 1 — lazy compressed bytes.** `ImageCache` holds the container path (threaded - App.current_file_path → SharedPaneState → VelloRenderContext) and pages bytes on a decode miss via - `read_packed_media_readonly`; `load_beam_sqlite` no longer eager-reads → instant load, compressed - bytes don't accumulate. `asset.data` is still used when resident (fresh import / old base64 project). - *(Refinement: persistent read connection vs open-per-miss.)* -- **[DONE] Prefetch.** `assets_needed_at(document, time)` enumerates image ids in the visible vector - layers' active keyframes; during playback the stage decodes the ~0.5s-ahead set into the cache. - *(Refinements: nested clip-instance recursion; background-thread decode.)* - -**Phase 4 = DONE** (image asset paging by usage + LRU). - -### 4a. Frame→asset enumeration (incl. nested clips — see note below) -A function `assets_needed_at(time) -> HashSet`: walk each visible vector layer's active -`ShapeKeyframe`, collect `fill.image_fill` across its `VectorGraph.fills`, **recursing into -clip instances** with the outer→inner local-time mapping. This is "needed now". Scanning -upcoming keyframes (and upcoming nested-clip keyframes) gives "needed soon" for prefetch. - -### 4b. Usage bookkeeping (the multi-frame problem) -Maintain a reverse index `asset_id → usage count` (fills referencing it across the whole -document), updated incrementally as edits add/remove `image_fill`s (hook the fill-mutation -paths in `vector_graph` and the relevant actions). -- count 0 → dead, fully evictable / GC candidate. -- count > 0 → keep metadata; residency of `data`/decoded pixels driven by **proximity to - playhead**, not by count (a high-count asset far from the playhead is still evicted). - -Residency decision: `resident = needed-now ∪ needed-soon`; beyond that, an **LRU with a byte -budget** for referenced-but-distant assets (covers scrubbing back without a reload). -Eviction never touches an asset in needed-now. - -### 4c. Bound the decoded tier -Convert `ImageCache`'s two maps to LRU/byte-budgeted (`renderer.rs:25`) and bound the GPU -image-texture cache the same way, keyed to the residency window. - -### Nested-clip prefetch (important) -A clip instance placed on an outer frame has its **own internal timeline of keyframes**, -each of which can reference its own image assets. Prefetch must therefore: -- Recurse through clip instances when computing both needed-now and needed-soon. -- Map outer playhead time → each nested clip's local time, and look ahead along the - **nested** timeline (not just the outer one) so assets used by an upcoming *inner* - keyframe are loaded before the nested clip reaches it. -- Deduplicate across the whole recursion (an asset shared by outer and inner frames counts - once); the usage index handles refcounting. - ---- - -## Cross-cutting: a shared residency abstraction - -A generic **`PagedStore`** with three consumers — always-resident metadata, -disk backing, residency = window/needed-set around playhead + LRU byte budget: - -| Consumer | Metadata kept | Paged payload | Backing | "Needed now" key | -|---|---|---|---|---| -| Raster keyframes (Ph 3) | id, dims, time | `raw_pixels` + GPU texture | SQLite blob row (`UPDATE` on write-back) | active keyframe per layer | -| Image assets (Ph 4) | id, dims, storage | `data` bytes + decoded pixels/texture | SQLite blob row or external path | fills' `image_fill` set at time (recursive) | -| Video frames (Ph 2a) | — | RGBA frame | source via ffmpeg seek | requested timestamps | - -Audio stays separate (real-time ring buffer, different constraints). The frame→asset -enumeration + usage index is unique to Phase 4. - ---- - -## Sequencing -1. **Phase 1a** — done; independent of the container, works with the current ZIP loader. -2. **Phase 2** — small, isolated, independently shippable; container-independent. -3. **Phase 0 (container)** — `.beam` ZIP → SQLite + `BlobReader` + large-media policy + - legacy-ZIP migration. Prerequisite for 1b/1c/3/4. -4. **Phase 1b** — streaming pool loader + bulk read-ahead activation (on the SQLite store). -5. **Phase 1c** — depends on 1b's pool path. -6. **Phase 3** — the substantial build; implement `PagedStore` over blob rows. -7. **Phase 4** — thin layer on the same abstraction + the frame→asset/usage index. - -Phase 1a and Phase 2 can ship now; everything else waits on Phase 0 (the container). - ---- - -## Status -- [~] Phase 1a — activate compressed-audio disk reader ← **in progress** - - [x] Wire `ActivateFile` + assign `clip.read_ahead` on `AddAudioClip` for compressed - pool files (`engine.rs:909`). Per-clip reader keyed by `clip_id`; matches the - existing `DeactivateFile` convention in `RemoveAudioClip`. Compiles clean. - - [ ] Stop `decode_progressive` (`io/audio_file.rs:344`) from accumulating/streaming the - full PCM; emit only the downsampled waveform overview. (Crosses into the UI - waveform pipeline — `AudioDecodeProgress` consumer — so handled as its own step.) - - [ ] Runtime verification: confirm a compressed clip actually plays from the ring - buffer (was effectively silent before, since `read_ahead` was always `None`). -- [~] **Phase 0 — container migration `.beam` ZIP → SQLite** ← **in progress** - - [x] SQLite schema (`media`, `media_chunk`, `project_json`, `meta`) + `rusqlite` dep - (bundled) — `lightningbeam-core/src/beam_archive.rs` - - [x] `BlobReader` (`Read + Seek` over `media_chunk`, owns its own read-only connection, - opens a blob handle per read with rowids resolved once) — for `CompressedReader` / - video decoder in 1b. 5 integration tests pass (`tests/beam_archive.rs`): json - round-trip, packed full read, streaming reads + seeks across chunk boundaries, - referenced-path, overwrite-replaces-chunks. - - [x] Packed (chunked) + referenced media write/read API; `is_sqlite()` format detection; - `MediaKind`/`MediaStorage`/`MediaMeta`/`MediaInfo`. - - [x] `BeamArchive::transaction()` / `BeamTxn` — in-place transactional save (only - changed rows written; unchanged large media never rewritten); orphan cleanup via - `retain_media`. 7 archive tests pass (added txn-grouping + rollback). Per user: save - must NOT copy+rename for existing SQLite files. - - [x] Wire `save_beam` to `BeamArchive` — in-place txn for existing SQLite, temp+rename - only for new/migrated files. Audio → packed (or referenced ≥2GB) `media` rows; - raster → PNG `media` rows keyed by keyframe id. FLAC→WAV→base64 save round-trip - deleted (now packs original bytes with their codec). - - [x] Wire `load_beam` — format dispatch: SQLite (`load_beam_sqlite`) vs legacy ZIP - (`load_beam_zip_legacy`, kept verbatim). SQLite load reconstitutes packed audio into - `embedded_data` so the existing pool loader is unchanged (streaming = Phase 1b). - - [x] Legacy ZIP `.beam` → SQLite migration: `is_sqlite()` routes load; saving a - ZIP-loaded project writes SQLite (migrates on save). Editor compiles end-to-end. - - [x] Large-media policy: packed (chunked) vs referenced — `LargeMediaMode {Ask,Pack, - Reference}`; save honors it for files ≥`LARGE_MEDIA_THRESHOLD`. Packing streams from - disk via `put_media_packed_from_path` (chunk-by-chunk, never loads the whole file). - `Ask` behaves as `Reference` at save time. - - [x] `large_media_default` user preference: persisted in `AppConfig`, editable in - Preferences → Advanced (incl. resetting to `Ask` to re-trigger the prompt). - - [x] First-import-over-threshold prompt: `note_possible_large_media` (hooked into - import_audio/video/image) queues a one-time modal; choice persists to config. - Threshold shown in the modal is derived from the constant. - - [ ] Runtime verification: save a real project, reopen it, confirm audio + raster survive - round-trip; confirm an old ZIP `.beam` still opens and migrates on save. - - [ ] (Optimization, later) FLAC-compress packed PCM/WAV audio; raster disk-dirty flag to - skip unchanged frames on in-place save (Phase 3). - -> Note: the crate's internal `#[cfg(test)]` modules (`clip.rs`, `effect_layer.rs`) have -> pre-existing compile breakage (old `Beats`/`TempoMap` API) unrelated to this work; it -> blocks `cargo test --lib`, so `beam_archive` tests live in `tests/` (integration) which -> build the lib in normal mode. Worth fixing separately. -- [x] Phase 1b — stream on project load (PACKED audio path complete & user-verified: streams on load, - waveform generates + persists, sample-accurate seeking). Referenced-path streaming + MP3 seek index - + proper video-audio reload remain as noted follow-ups. - - **Decision (user):** cross-crate packed streaming via an **inversion-of-control factory** — - daw-backend defines the interface, core implements it over `BlobReader`. Keeps the audio - engine container-agnostic. (Alternatives rejected: daw-backend owning rusqlite = layering - violation; referenced-only-first = leaves packed <2GB in RAM.) - - **Current load reality (why this is needed):** *nothing* streams on load today — every entry - is fully decoded to a PCM `Vec`. Packed audio is base64-reconstituted into `embedded_data` - (`load_beam_sqlite`) → written to a temp file → `load_file_into_pool` full-decodes; referenced - audio also full-decodes via `load_file_into_pool`; and the Phase 1a/1c disk-reader activation - never fires for loaded clips (they bypass `AddAudioClip`). - - [x] **B1/B2 foundation (DONE, headless-tested):** in `disk_reader.rs` — `trait MediaByteSource: - Read+Seek+Send+Sync { byte_len }` + `trait AudioBlobSourceFactory: Send+Sync { open(media_id) - -> Box }`; `SymphoniaByteSource` adapter (impl `MediaSource`, - is_seekable/byte_len); `CompressedReader::open_source(src, ext)` sharing probe via a - refactored `from_mss`; `enum StreamOpen { Path, Source{src,ext} }`; `StreamSource::open` and - `DiskReaderCommand::ActivateFile` now take `StreamOpen` (engine site wraps `Path`); re-exported - `AudioBlobSourceFactory`/`MediaByteSource` at `daw_backend::audio`. Test - `tests/compressed_source_stream.rs` decodes an in-memory WAV through a `Cursor`-backed - `MediaByteSource` (proves probe+decode+seek over a byte stream). daw-backend compiles clean. - - [x] **B3 (engine, DONE):** `Engine.blob_source_factory: Option>` + - `EngineController::set_blob_source_factory` (via `Query::SetBlobSourceFactory`, ordered before - `SetProject` on the same queue). `AudioFile.packed_media_id: Option` (Some ⇒ open via - factory using `original_format` as the ext hint; None ⇒ `StreamOpen::Path`). Activation factored - into `Engine::activate_streaming_for(reader_id, pool_index)`, used by `AddAudioClip` and bulk. - - [x] **C (core factory, DONE):** `file_io::blob_source_factory(beam_path)` → `BeamBlobFactory` - implementing `AudioBlobSourceFactory` over `BeamArchive::open_blob_reader`. `BlobReader` holds a - `!Sync` rusqlite `Connection`, so it's wrapped in `SyncBlobReader` (a `Mutex` used via `get_mut` - on the hot path — no runtime locking) to satisfy Symphonia's `MediaSource: Send + Sync`. Installed - by the editor between `load_audio_pool` and `set_project`. - - [x] **D (load-path, DONE — packed audio):** `load_beam_sqlite` now streams packed audio whose codec - is recognized (`is_streamable_audio_codec`) — leaves `embedded_data` empty so the pool builds a - Compressed placeholder with `packed_media_id`; no base64, no temp file, no decode. `serialize` - round-trips packed entries by media id (so in-place re-save keeps the row). Non-audio codecs - (video-container audio tracks) keep the legacy reconstitution path → **no regression**. - - [x] **E (bulk activation, DONE):** `SetProject` calls `Engine::activate_all_streaming_clips` — - walks every loaded audio clip and `activate_streaming_for` (create_buffer + `ActivateFile` + set - `read_ahead`), the loaded-clip equivalent of the Phase 1a wiring. - - [x] **Waveform-on-load for streamed audio (DONE):** streaming broke the old waveform path (it came - from the full in-RAM decode, which no longer happens). Added - `disk_reader::build_waveform_pyramid_from_source(Box, ext, B)` (load-time - counterpart of the path-based builder). On load, the editor background-generates a pyramid for any - streamed entry lacking a persisted one (opens the packed blob via a local factory), sending the - floor through the same `waveform_result` channel `update()` drains; the next save persists it. - Verified in-app: packed MP3 **streams + plays** (`Activated reader=0, kind=CompressedAudio`); the - overview now fills in shortly after load. - - **Headless tests pass** (compressed_source_stream, video_audio_stream, waveform_pyramid); all three - crates compile clean. **Needs in-app verification:** the waveform appears after load (background gen), - then instantly on subsequent loads once saved; RAM stays flat on a big project. - - [x] **Seek alignment fix (DONE):** streamed compressed audio was ~1.2s off *after seeking* - (fine from the start). `CompressedReader::seek` used `SeekMode::Coarse`, which for MP3 - byte-estimates the position and seeds the timestamp from that estimate — wrong for VBR / files - whose header padding the estimate ignores, so `actual_ts` (and thus the buffer's frame labels) - landed ~1.2s early. Switched to `SeekMode::Accurate`: Symphonia counts frame *headers* (no - decode) from a true anchor (current pos, or rewind-to-0 for backward seeks) → exact `actual_ts`; - the existing sub-frame `pending_discard` finishes the job. FLAC/OGG seek cheaply (seek tables); - a long MP3 backward seek walks headers from 0 (I/O, not decode). Tests still green. - - [ ] **Deferred (follow-up):** per-file **seek index** for elementary streams (MP3) — a one-time - header scan (ts↔byte map) to make far seeks O(1) instead of an Accurate header-walk from the - anchor. Matters for multi-hour MP3s; song-length files are fine as-is. - - [x] **Proper video-audio reload (DONE):** a video's audio is now stored as a **path reference** to - the video (never packed/embedded as audio media) and **re-probed via FFmpeg** on load into a - streaming `VideoAudio` entry — `AudioPoolEntry.is_video_audio` flag drives both `serialize` - (reference, not pack), `save_beam` (`reference_it |= is_video_audio`), and `load_from_serialized` - (`VideoAudioReader::open` → `from_video_audio`). Fixes 5.1 audio losing its channels on reload - (the old Symphonia reconstitution collapsed it); also no more decode-whole-video-to-RAM / temp - files on load. Old saves (video mis-packed as audio) self-heal on the next save. - - [ ] **Deferred (follow-up):** stream *referenced* (external-path) **audio** on load too — replace - `load_file_into_pool`'s full decode with the `do_import_audio` branching (PCM → mmap, compressed - → `from_compressed` placeholder). Higher risk (touches the working referenced path); packed - covers the common <2GB case first. - - [x] **DONE: packed video streaming.** Small videos pack into the `.beam` - (a `MediaKind::Video` blob at the clip id, `VideoClip.media_id` referencing it) and stream **both - frames and audio** from the DB blob via FFmpeg. The `AVIOContext`-over-`Read+Seek` shim lives in - the new `ffmpeg-blob-io` crate (`BlobInput`, version-pinned `=8.0.0`/`=8.0.1`), isolating the - unsafe + ABI coupling. Frames: `video.rs` `VideoSource{Path,Packed}` opens a fresh `BlobReader` - per decoder/seek/scan. Audio: `VideoAudioReader::open_source` over the same blob (the - `disk_reader.rs` `StreamSource` blocker is removed); save points the linked video-audio pool - entry's `media_id` at the video row so it streams from the same blob. Tests: ffmpeg-blob-io AVIO - unit tests (WAV via Cursor + seek + open/drop loop), core `packed_video_stream` (blob→AVIO→Input), - `beam_archive` packed-video round-trip, daw-backend `open_source` (compiles; can't link in the - container — user runtime-verifies actual A/V playback). -- [~] Phase 1c — video embedded-audio track ← **stopgap shipped; proper design next** - - [x] Stopgap: `extract_audio_from_video_to_wav` streams to a temp WAV → `import_audio_sync` - (mmap). Fixed the ~2.8GB-`Vec` OOM. But writes the whole WAV to `/tmp` (fills - small temp partitions) and the temp path doesn't survive reload. - - [~] **Proper design** (see "Phase 1c" body): stream the video's audio on demand via a new - ffmpeg `VideoAudioReader` in the disk reader — no extraction, no `/tmp`, no RAM; path - reference survives save/load. - - [x] **Step 1 (DONE):** `VideoAudioReader` (ffmpeg) + `StreamSource` enum + `SourceKind` - in `disk_reader.rs`. Sample-accurate seek (coarse seek + decode-discard to exact - frame via PTS). 2 integration tests pass (`daw-backend/tests/video_audio_stream.rs`): - in-order decode + sample-exact seek at several targets. (Found: mono frames have an - empty channel layout → must `set_channel_layout` before resampling, else swr returns - AVERROR_INPUT_CHANGED.) Lib compiles clean; `StreamSource` `#[allow(dead_code)]` - until wired. `VideoAudioReader` made `pub` for the integration test. - - [x] **Step 2 (DONE):** `AudioStorage::VideoAudio { decoded_for_waveform, decoded_frames, - total_frames }` + `AudioFile::from_video_audio` (path = the video file). `data()` - empty / `read_samples()` 0 (streamed). `Query::AddVideoAudioSync` + - `do_add_video_audio` (probes via `VideoAudioReader::open`, no decode) + - `EngineController::add_video_audio_sync`. `GetPoolAudioSamples` surfaces VideoAudio's - waveform overview too. daw-backend compiles clean; probe `total_frames` test passes. - - [x] **Step 3 (DONE):** reader thread now holds `StreamSource` (opens via - `StreamSource::open(path, kind)`, dispatches `sample_rate()/channels()/seek/decode_next`); - `ActivateFile` carries `kind: SourceKind`; `#[allow(dead_code)]` removed. `AddAudioClip` - activation maps `Compressed`→`CompressedAudio`, `VideoAudio`→`VideoAudio`, creates the - read-ahead buffer + `ActivateFile{kind}` + sets `clip.read_ahead`. Compressed path is - behaviorally identical (StreamSource::Compressed wraps the same CompressedReader). - daw-backend + editor compile clean; VideoAudioReader tests still pass. - ⚠️ Not runtime-verified — needs in-app check that compressed audio still plays (no - regression) and that an activated VideoAudio clip produces sound. - - [x] **Step 4 (DONE):** `import_video` now calls `add_video_audio_sync(video_path)` → - pool index, fetches channels/sample_rate via `get_pool_file_info`, makes the - `AudioClip` with the video's duration. **No WAV / /tmp / RAM.** Removed the stopgap - (`extract_audio_from_video_to_wav` + WAV helpers + `ExtractedAudioInfo`), dead - `add_audio_file_sync` (+ `Query::AddAudioFileSync` / `QueryResponse::AudioFileAddedSync` - / handler), and the now-unreachable `AudioExtractionResult::NoAudio`. Kept - `import_audio_sync` (still used by normal audio import). daw-backend + editor clean. - **→ Feature is live end-to-end; ready for in-app testing.** - - [x] **Step 5 (DONE):** `CompressedReader` now seeks sample-accurately too — coarse - symphonia seek + decode-discard (`pending_discard` set from `seeked.actual_ts` in - `seek`, applied in `decode_next`, which continues rather than reporting EOF when a - whole packet is discarded). So compressed clips no longer drift vs video audio after - a seek. Test `compressed_reader_seek_is_sample_accurate` passes (the WAV coarse seek - lands pre-target, exercising the discard). `CompressedReader` made `pub` for the test. - - [~] Step 6: **bounded waveform overview** — replaces today's full-resolution - `raw_audio_cache`/GPU waveform (which doesn't scale: it stores every sample at mip 0, - so a long file is multi-GB on GPU + RAM — the same memory issue, and the Phase 1a - `decode_progressive` leftover). Design below. Slices: (1a) streaming pyramid builder - + (1b) persistence + (1c) min/max GPU upload, then (2) LRU tile cache + re-decode floor. - - [x] **Slice 1a (DONE):** `daw-backend/src/audio/waveform_pyramid.rs` — - `WaveformPyramidBuilder` streams interleaved samples, accumulates the floor, and - reduces `BRANCH(4):1` at `finish` into a root-first pyramid (convention B: - `levels[0]`=root envelope, `levels.last()`=floor, `.root()`/`.floor()` accessors). - Ragged last buckets reduce over available children (no value padding). Bounded - (~22 MB/2 h @ B=256). 7 integration tests pass (`tests/waveform_pyramid.rs`): - bucket min/max, partial flush, multi-level envelope == global min/max, root-first - ordering, stereo channels, size bound, chunk-agnostic. - - [~] **Slice 1b (data layer DONE; orchestration folded into 1c):** - - [x] Generation bridge `disk_reader::build_waveform_pyramid(path, kind, B)` — streams - a decode (`StreamSource` over symphonia/ffmpeg) into the builder; bounded - memory (one chunk + the pyramid). Test: envelope matches the signal through - both backends. - - [x] Serialization `WaveformPyramid::to_bytes`/`from_bytes` (LBWF blob; f32 texels — - f16 a later size optimization). Round-trip test + rejects truncated/garbage. - - [x] `MediaKind::Waveform` in the SQLite container (keyed by the audio item's id). - - [ ] Orchestration (with 1c). - - [~] **Slice 1c (in-memory floor overview DONE; persistence next):** - - [x] `waveform_gpu`: `PendingUpload.minmax` flag + `pack_texel` helper; `upload_audio` - threads `minmax` (frame_stride 4, packs `(Lmin,Lmax,Rmin,Rmax)` directly). - The texture is already Rgba16Float and the GPU mipgen builds zoom-out levels, so - only the texel-packing differs. Render the floor at **effective rate `sr/B`** (so - time→texel maps B samples/texel) and `total_frames = floor_texel_count`. - - [x] `AppConfig.waveform_floor_samples_per_texel` (default 256, user-configurable). - - [x] App: `waveform_minmax_pools: HashMap` (pool → `B`, carries the floor rate - with full float precision) + a `(pool, packed_floor, sr, channels, B)` results channel; - drained in `update()` → `raw_audio_cache.insert(floor)` + flag pool + `waveform_gpu_dirty`. - - [x] Generation: on video-audio import Success, the same bg thread streams - `disk_reader::build_waveform_pyramid(path, VideoAudio, B)` once and sends the packed - `floor()`. (Video-audio has no in-RAM samples, so this is what makes its waveform appear.) - - [x] Threaded `waveform_minmax_pools` through the pane-context (`panes/mod.rs` + - main.rs construction) → `render_layers` → **both** render sites (collapsed-group - ~timeline.rs:3048 AND expanded-track ~3613): compute `total_frames = len/4`, - `eff_sr = sr/B`, set `minmax`. Compiles clean (editor `cargo check` = 0 errors). - - [x] Shader fix: `waveform.wgsl` now reads the **nearest integer LOD via `textureLoad`** - instead of sampling a fractional mip. Trilinear blends two levels whose row-major - linearizations differ → horizontal shift that flips each 0.5 of `mip_f` (= each 2x - zoom step), the "every other zoom level is offset" artifact. **User-confirmed fixed:** - features hold position at every zoom and line up with playback. - See memory `waveform-shader-fractional-mip-offset`. - - [x] **Persistence (done):** the full pyramid is serialized (`to_bytes`) on generation and - kept in `App.waveform_pyramid_blobs`. `save_beam` writes it as a `MediaKind::Waveform` - row keyed by a **deterministic id derived from the pool index** (`file_io::waveform_media_id`, - "LBWF" sentinel in the high 32 bits) — independent of how the audio bytes are stored, so - it works for packed/referenced/video-audio alike, and an in-place re-save reuses the row. - Carried in/out via a transient `#[serde(skip)] AudioPoolEntry.waveform_blob` and a - `waveform_blobs` field on `FileCommand::Save`. `load_beam_sqlite` reads the row back; - the editor restores `raw_audio_cache`/`waveform_minmax_pools`/`waveform_pyramid_blobs` - + flags `waveform_gpu_dirty` after the backend loads the pool (using each entry's - `sample_rate` for `eff_sr`, the stored `B` for the rate). No re-decode on load. - `register_loaded_videos` only loads frames (not audio), so there is no redundant - regeneration to suppress. Compiles clean across all three crates. - -### Waveform LOD pyramid design (step 6) -A min/max LOD pyramid (tree of zoom-level textures): fully zoomed out → envelope; fully zoomed -in → per-sample; seamless between. - -- **One streaming decode pass** builds the whole pyramid down to a configurable **floor** - `B` samples/texel (default 256), via a hierarchical reduction (each sample updates a running - per-level min/max accumulator; a filled bucket emits a texel and folds into its parent — - `branch` 4:1). Bounded memory: holds only the pyramid (~`N/B·4/3` texels ≈ **~14 MB / 2 h - stereo @ B=256**), never the full samples. Full-res (B=1 ≈ 2.7 GB) is the only level NOT - stored. -- **Persist the pyramid** in the `.beam` SQLite container (a `waveform` media kind; session - temp before first save). `B` is stored with it (preference is just the default for new gen). - Persistence is load-bearing: it makes mid-zoom a cheap **disk read**, not a re-decode. -- **Runtime = LRU tile cache** (GPU textures) loaded from the persisted pyramid on demand. - Eviction is **ancestor-closed**: only evict an LRU node with no resident children ("a node is - cleared only after its children") — so rendering can always walk up to a resident ancestor; - detail sharpens in, never blanks. Root is tiny/hot → effectively pinned for free. -- **Re-decode only below the floor** (texel < `B` samples): by then the visible window spans a - tiny time range, so decoding it (via the sample-accurate seekable readers from steps 1–5 — - the payoff) for true per-sample detail is cheap. This removes the large-span re-decode gap: - above the floor it's a disk read; below it the span is already small. -- **Why a deep floor (not a coarse cutoff):** a coarse-only pinned set would force the first - on-demand level to re-reduce a huge time span per tile. Persisting deep makes every level a - disk read; `B` is a size-vs-crossover knob (smaller B = bigger pyramid, cheaper re-decode). -- `waveform_gpu` needs a **min/max texel upload** (`Lmin,Lmax,Rmin,Rmax` per texel) instead of - min=max-per-sample; the existing compute mipgen still builds the mip chain *within* a tile. - -**Decisions (locked):** branch 4:1; floor `B≈256` samples/texel, **user-configurable** -(`AppConfig.waveform_floor_samples_per_texel`, stored per-pyramid); 8192-wide tiles; LRU ~4 -viewports of fine tiles; persist pyramid in `.beam`. -- [x] Video decoder concurrency (movie-length lag/freeze): keyframe-index scan now runs - holding no VideoManager/decoder lock (brief locks only bracket it) → no more multi-second - UI freeze on import/load; thumbnail generation uses a **dedicated** decoder and samples - at keyframes (≈1 frame each vs whole-GOP) → no playback contention. Removed dead - `VideoManager::build_keyframe_index`, `build_and_set_keyframe_index`, `downsample_rgba*`. -- [x] Phase 2a — bound video frame cache. `VideoManager.frame_cache` (was an unbounded - `HashMap<(Uuid,i64), Arc>` that grew per distinct frame during playback) is now an - `LruCache` evicted by a **byte budget** (`FRAME_CACHE_BYTE_BUDGET` = 256 MB) rather than a frame - count — robust across resolutions (a 4K frame is ~33 MB vs ~2 MB at 800×600). Byte total tracked - on insert/evict/remove; `unload_video` pops per-clip keys (LruCache has no `retain`). Decoder-level - cache was already LRU. Editor compiles clean. *(Not yet runtime-verified.)* -- [x] Phase 2b — stream export mux. `export/mod.rs::mux_video_and_audio` no longer collects every - packet into two `Vec`s before interleaving; it stream-merges the two inputs by PTS with one pending - packet per stream (O(1) memory vs O(duration)). Same tie-break (`v_us <= a_us`) and drain-on-EOF - behavior; output is byte-identical. Editor compiles clean. *(Not yet runtime-verified — needs an - in-app export to confirm A/V sync.)* -- [x] Phase 3a — lazy + async raster fault-in (`RasterStore` + background thread + image proxy) -- [x] Phase 3b — raster residency LRU + eviction (dirty-flag data-loss safety) -- [x] Phase 3c — bound raster GPU texture cache (recency LRU + F3 VRAM readout) -- [x] Phase 3d — raster undo dirty-rect diffs (+ fault-in-before-undo) -- [x] Phase 3.5 — image textures in vector scenes (fixed DCEL-broken image import; image-fill tab + picker; container-persisted) -- [x] Phase 4 — image asset paging: Tier 2 decoded-cache byte-LRU, Tier 1 lazy container bytes, playback prefetch -- [x] Phase 5 — fixed the broken `#[cfg(test)]` unit tests; **`cargo test --lib` green again** - (daw-backend 17 passed, lightningbeam-core 264 passed). Wrapped stale raw-`f64` time literals - in `Beats(...)` / passed `&TempoMap` to changed signatures (automation.rs, clip.rs, - effect_layer.rs); fixed stale test setup (register a vector clip so `get_clip_duration` resolves) - and a stale default expectation (shape `fill_color` defaults `None`). Surfaced + fixed one **real - undo bug**: `DeleteFolderAction(MoveToParent)` reparented child subfolders but never restored them - on rollback (orphaned them) — now tracked and restored. Production code otherwise untouched. diff --git a/TODO.md b/TODO.md index 4e61b42..a718f1e 100644 --- a/TODO.md +++ b/TODO.md @@ -1,305 +1,25 @@ # Lightningbeam TODO -> ⚠️ **Stale entries:** Lightningbeam was rewritten from JavaScript to Rust. Any entry below -> that cites `src/*.js` / `main.js` / `animation.js` predates that migration — the *issue* may -> or may not still exist in the Rust codebase, but the file/line references are obsolete. -> **Re-verify against the current Rust code before acting** (this covers the "Animation System -> Refactoring" section and the JS-referencing "Known Issues" entries — node editor, default -> interpolation, etc.). Items with no `.js` references are current. +## Known Issues (Rust) -## Animation System Refactoring *(STALE — JS-era migration notes; superseded by the Rust DCEL/keyframe system)* +### Animation: Tweens are broken — LOW PRIORITY +- Shape/vector interpolation between keyframes, and the `tween_after` behavior on + keyframes, don't work correctly in the current app. Needs investigation + fix. + Not urgent — revisit later. -### Completed -- ✅ Implement AnimationData curve-based system (Keyframe, AnimationCurve, AnimationData classes) -- ✅ Add GraphicsObject.currentTime property -- ✅ Migrate shape rendering to use AnimationData curves (exists, zOrder) -- ✅ Binary search optimization for keyframe lookups +## Backlog / Feature ideas -### In Progress -- Migrating from Frame-based to AnimationData curve-based system throughout codebase +### Animation curve enhancements +- [ ] Extrapolation modes, separate for start vs end: hold (default), extend, repeat, decay +- [ ] Position / scale / rotation animation curves for shapes +- [ ] Shape morphing / tweening between keyframes -### Pending Features - -#### Animation Curve Enhancements -- [ ] Implement extrapolation modes (separate for start vs end): - - "hold" (default) - hold value at first/last keyframe - - "extend" - linearly extend the curve beyond keyframes - - "repeat" - repeat the animation - - "decay" - exponential decay to a target value -- [ ] Add position, scale, rotation animation curves for shapes -- [ ] Add shape morphing/tweening between keyframes - -#### Keyframing Behavior -- [ ] Add user preference for keyframing behavior when editing objects: +### Keyframing behavior +- [ ] User preference for keyframing when editing objects: - Auto-keyframe (current default): create/update keyframe at current time - Edit previous (Flash-style): update most recent keyframe before current time - Ephemeral (Blender-style): changes don't persist without manual keyframe - - Optional: Add modifier key (e.g. Shift) to toggle between modes + - Optional modifier key (e.g. Shift) to toggle modes -#### Shape Ordering -- [ ] Add "Bring Forward" menu option (swap zOrder with shape in front) -- [ ] Add "Send Backward" menu option (swap zOrder with shape behind) -- [ ] Add "Bring to Front" menu option (set zOrder to max + 1) -- [ ] Add "Send to Back" menu option (set zOrder to min - 1) - -#### Code Cleanup -- [ ] Remove all remaining references to Frame-based system -- [ ] Remove legacy Frame class once migration is complete -- [ ] Clean up GraphicsObject.shapes[] array (shapes should only live in Layers) - -## Known Issues / Platform Limitations - -### Animation: Tweens are broken (Rust codebase) — LOW PRIORITY -- **Issue**: Animation tweening between keyframes (shape/vector interpolation, and the - `tween_after` behavior on keyframes) does not work correctly in the current Rust app. - Needs investigation + fix. Not urgent — revisit later. -- (Older JS-codebase animation entries below reference `src/*.js` and are stale.) - -### Audio: Oscillator Timbre Drift (Phase Accumulation Error) -- **Issue**: Oscillators exhibit timbre changes over time due to floating-point phase accumulation errors -- **Affected Files**: - - `daw-backend/src/effects/synth.rs:117-120` (SimpleSynth) - - `daw-backend/src/audio/node_graph/nodes/oscillator.rs:167-170` (OscillatorNode) -- **Root Cause**: Current phase wrapping uses conditional subtraction (`if phase >= 1.0 { phase -= 1.0 }`), which accumulates f32 rounding errors over time, especially for long-playing notes -- **Current Code**: - ```rust - self.phase += frequency / sample_rate; - if self.phase >= 1.0 { - self.phase -= 1.0; - } - ``` -- **Recommended Fix**: Replace with `.fract()` for numerically stable wraparound: - ```rust - self.phase += frequency / sample_rate; - self.phase = self.phase.fract(); - ``` -- **Impact**: Medium - affects audio quality for sustained notes, becomes noticeable after several seconds -- **Priority**: Medium - should be addressed before production use - -### UI: Node Connections Render Behind VoiceAllocator Child Nodes -- **Issue**: Connection lines (SVG paths) inside expanded VoiceAllocator nodes render behind child nodes due to z-index stacking -- **Affected File**: `src/styles.css:1128` -- **Root Cause**: Child nodes have `z-index: 10` while connection SVG paths have default/lower z-index -- **Current Code**: - ```css - .drawflow .drawflow-node.child-node { - opacity: 0.9; - border: 1px solid #5a5aaa !important; - box-shadow: 0 2px 8px rgba(90, 90, 170, 0.3); - z-index: 10; - } - ``` -- **Recommended Fix**: Either: - 1. Remove `z-index: 10` from `.child-node` (simplest), or - 2. Add higher z-index to connection SVG paths, or - 3. Use CSS `isolation: isolate` on the VoiceAllocator contents area to create a new stacking context -- **Impact**: Low - visual issue only, connections still function but appear to go "behind" nodes -- **Priority**: Low - cosmetic issue that doesn't affect functionality - -### UI: VoiceAllocator Child Nodes Don't Move with Parent -- **Issue**: When a VoiceAllocator node is moved, its child nodes remain in their original positions instead of moving with the parent -- **Affected File**: `src/main.js:6202-6207` -- **Root Cause**: The `nodeMoved` event handler only handles the case where a child node is moved (resizes parent), but doesn't handle when the VoiceAllocator itself is moved -- **Current Code**: - ```javascript - editor.on("nodeMoved", (nodeId) => { - const node = editor.getNodeFromId(nodeId); - if (node && node.data.parentNodeId) { - resizeVoiceAllocatorToFit(node.data.parentNodeId); - } - }); - ``` -- **Recommended Fix**: Add logic to detect when a VoiceAllocator is moved and update all child node positions: - ```javascript - editor.on("nodeMoved", (nodeId) => { - const node = editor.getNodeFromId(nodeId); - - // Case 1: A child node was moved - resize parent - if (node && node.data.parentNodeId) { - resizeVoiceAllocatorToFit(node.data.parentNodeId); - } - - // Case 2: A VoiceAllocator was moved - move all children - if (node && node.data.nodeType === 'VoiceAllocator') { - // Calculate delta from previous position (need to track) - // Update all child node positions by the delta - // Call editor.updateConnectionNodes() for parent and all children - } - }); - ``` -- **Impact**: High - child nodes become disconnected from parent visually -- **Priority**: High - breaks expected behavior of grouped nodes - -### UI: VoiceAllocator Expansion Doesn't Update Connection Positions -- **Issue**: When expanding/collapsing a VoiceAllocator, connection endpoints don't update to match the new port positions -- **Affected File**: `src/main.js:6496-6555` (handleNodeDoubleClick function) -- **Root Cause**: The expand/collapse logic shows/hides child nodes and resizes the container, but never calls `editor.updateConnectionNodes()` to refresh connection positions -- **Current Code**: In `handleNodeDoubleClick()`, after expanding or collapsing: - ```javascript - // Expand - expandedNodes.add(nodeId); - nodeElement.classList.add('expanded'); - nodeElement.style.width = '600px'; - nodeElement.style.height = '400px'; - // ... shows child nodes ... - // Missing: editor.updateConnectionNodes(`node-${nodeId}`) - ``` -- **Recommended Fix**: Call `editor.updateConnectionNodes()` after resizing: - ```javascript - // After expanding - expandedNodes.add(nodeId); - nodeElement.classList.add('expanded'); - // ... resize and show children ... - - // Update connection positions for VoiceAllocator and all children - editor.updateConnectionNodes(`node-${nodeId}`); - for (const [childId, parentId] of nodeParents.entries()) { - if (parentId === nodeId) { - editor.updateConnectionNodes(`node-${childId}`); - } - } - ``` -- **Impact**: Medium - connections appear in wrong positions until manually moved -- **Priority**: Medium - visual issue that affects usability - -### UI: Node Editor Allows Editing Without MIDI Layer Selected -- **Issue**: The node editor pane allows adding/editing instrument nodes even when no MIDI layer is selected, and always uses hardcoded `trackId: 0` -- **Affected File**: `src/main.js:6045-6920` (nodeEditor function) -- **Root Cause**: The node editor never checks if `context.activeObject.activeLayer` exists or is a MIDI track, and all backend commands use hardcoded `trackId: 0` -- **Current Code**: All graph commands hardcode track 0: - ```javascript - const commandArgs = parentNodeId - ? { - trackId: 0, // HARDCODED! - voiceAllocatorId: editor.getNodeFromId(parentNodeId).data.backendId, - nodeType: nodeType, - x: x, - y: y - } - : { - trackId: 0, // HARDCODED! - nodeType: nodeType, - x: x, - y: y - }; - ``` -- **Recommended Fix**: - 1. Check if activeLayer is a MIDI track before allowing edits: - ```javascript - function getSelectedMidiTrack() { - const activeLayer = context.activeObject?.activeLayer; - if (!activeLayer || activeLayer.type !== 'midi') { - return null; - } - return activeLayer; - } - ``` - 2. Show placeholder when no MIDI track selected: - ```javascript - function nodeEditor() { - const container = document.createElement("div"); - const midiTrack = getSelectedMidiTrack(); - - if (!midiTrack) { - container.innerHTML = '
Select a MIDI layer to edit instruments
'; - return container; - } - // ... rest of node editor code ... - } - ``` - 3. Use actual track ID instead of hardcoded 0: - ```javascript - const trackId = midiTrack.audioTrackId || 0; - const commandArgs = { trackId, nodeType, x, y }; - ``` - 4. Add listener to refresh node editor when layer selection changes -- **Impact**: High - allows editing wrong track's instrument graph, data corruption risk -- **Priority**: High - can cause confusion and data loss - -### Animation: Wrong Default Interpolation for Shape and Object Keyframes -- **Issue**: Shape index and object transform keyframes default to "linear" interpolation but should default to "hold" (step function), and there's no UI to change interpolation after creation -- **Affected Files**: - - `src/models/animation.js:124` (Keyframe constructor defaults to "linear") - - `src/main.js:2161` (shapeIndex keyframes default to "linear") - - `src/main.js:2198` (object position/rotation/scale keyframes default to "linear") - - `src/main.js:5910` (Timeline menu - missing tween options) -- **Root Cause**: - 1. The Keyframe constructor defaults interpolation to "linear" - 2. Shape index keyframes preserve existing interpolation or default to "linear" - 3. Object transform keyframes explicitly use "linear" - 4. No menu options exist to change interpolation mode after keyframe creation -- **Current Code**: - - Keyframe constructor (animation.js:124): - ```javascript - constructor(time, value, interpolation = "linear", uuid = undefined) { - ``` - - Shape index keyframes (main.js:2161): - ```javascript - const interpolationType = existingShapeIndexKf ? existingShapeIndexKf.interpolation : 'linear'; - const shapeIndexKeyframe = new Keyframe(currentTime, newShapeIndex, interpolationType); - ``` - - Object keyframes (main.js:2198): - ```javascript - const newKeyframe = new Keyframe( - currentTime, - currentValue, - 'linear' // Default to linear interpolation - ); - ``` -- **Expected Behavior**: - - Shape index keyframes should default to "hold" (shapes shouldn't morph between versions) - - Object transforms should default to "hold" (objects shouldn't move/rotate/scale between keyframes unless explicitly tweened) - - Timeline menu should have options to convert between interpolation modes -- **Recommended Fix**: - 1. Change shapeIndex default to "hold" (main.js:2161): - ```javascript - const interpolationType = existingShapeIndexKf ? existingShapeIndexKf.interpolation : 'hold'; - ``` - 2. Change object keyframe default to "hold" (main.js:2198): - ```javascript - const newKeyframe = new Keyframe(currentTime, currentValue, 'hold'); - ``` - 3. Add Timeline menu options (main.js:5910, in timelineSubmenu): - ```javascript - { - text: "Add Shape Tween", - enabled: /* check if shape is selected and has keyframes */, - action: () => { - // Find shapeIndex curve for selected shape - // Change interpolation between keyframes to "linear" - } - }, - { - text: "Add Motion Tween", - enabled: /* check if object is selected and has transform keyframes */, - action: () => { - // Find position/rotation/scale curves for selected object - // Change interpolation between keyframes to "linear" or "bezier" - } - } - ``` -- **Note**: exists and zOrder keyframes already correctly use "hold" (main.js:2139, 2150) -- **Impact**: High - causes unwanted interpolation, shapes morph unexpectedly, objects move when they shouldn't -- **Priority**: High - fundamental animation behavior is incorrect - -### Tauri Pinch-Zoom on Linux -- **Issue**: Two-finger pinch gestures zoom the entire Tauri window instead of individual canvases -- **Status**: Known Tauri limitation on Linux/GTK with no cross-platform solution -- **Tracking**: https://github.com/tauri-apps/tauri/discussions/3843 -- **Workaround attempts**: Tried `zoomHotkeysEnabled: false`, `touch-action: none`, viewport meta tags - none worked -- **Resolution**: Monitor Tauri releases for official fix - -## Notes - -### Architecture -- **GraphicsObject** contains Layers and has `currentTime` (continuous time) -- **Layer** contains `shapes[]` array and `animationData` (AnimationData instance) -- **AnimationData** contains curves dictionary, each curve identified by parameter name - - Shape curves: `shape.{uuid}.exists`, `shape.{uuid}.zOrder` - - Future: `shape.{uuid}.x`, `shape.{uuid}.y`, `shape.{uuid}.rotation`, etc. -- **Shapes render based on curves**: Layer.draw checks exists > 0, sorts by zOrder, draws in order - -### Interpolation Types -- `linear` - Linear interpolation between keyframes -- `bezier` - Cubic Bezier with easing control points -- `step`/`hold` - Step function (jumps to next value) +### Shape ordering +- [ ] Bring Forward / Send Backward / Bring to Front / Send to Back menu options diff --git a/daw-backend/Cargo.lock b/daw-backend/Cargo.lock index 6399b8d..25d2e1e 100644 --- a/daw-backend/Cargo.lock +++ b/daw-backend/Cargo.lock @@ -586,6 +586,7 @@ dependencies = [ "dasp_rms", "dasp_sample", "dasp_signal", + "ffmpeg-blob-io", "ffmpeg-next", "hound", "memmap2", @@ -661,6 +662,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "ffmpeg-blob-io" +version = "0.1.0" +dependencies = [ + "ffmpeg-next", + "ffmpeg-sys-next", + "libc", +] + [[package]] name = "ffmpeg-next" version = "8.0.0" diff --git a/lightningbeam-ui/Cargo.lock b/lightningbeam-ui/Cargo.lock index efbdbeb..97aed88 100644 --- a/lightningbeam-ui/Cargo.lock +++ b/lightningbeam-ui/Cargo.lock @@ -3628,7 +3628,7 @@ dependencies = [ [[package]] name = "lightningbeam-editor" -version = "1.0.7-alpha" +version = "1.0.8-alpha" dependencies = [ "beamdsp", "bytemuck", diff --git a/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md b/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md index 0586549..17e2d19 100644 --- a/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md +++ b/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md @@ -67,15 +67,47 @@ at the requested target res. This fixes the 4K decode wall, the 8 MB upload, *an - Result: software exports are full-quality at any export res, and document resizes re-target decode. No hardware needed; this is the correctness fix for the codecs HW can't handle anyway. -### Stage 2 — hardware decode primitive (headless-testable here, like the 8 encode tests) -- In `gpu-video-encoder` (rename → `gpu-video-codec`): `h264_vaapi`-style **decode** → VAAPI surface → - export DMA-BUF → import as a wgpu texture. Hardware test: decode a known file, verify dims/contents. +### Stage 2 — hardware decode primitive (DONE, commit 255e164) +`decoder::VaapiDecoder` in `gpu-video-encoder`: decode → VAAPI surface → DRM-PRIME DMA-BUF → +`dmabuf::import_raw` → wgpu textures. Round-trip test (encode gray → decode → readback Y≈128) passes. -### Stage 3 — wire hardware decode into `get_frame` (blind; user-verifies) -- When the source codec/driver is HW-decodable, `get_frame` returns a **GPU texture** (native res) - instead of `Arc>`; the compositor uses it directly (no `write_texture`), GPU-scaling to the - target. For the zero-copy export the frame never leaves the GPU: **decode → composite → encode** on - one device. Software path is the fallback for everything else. +### The device-affinity problem (drives the whole rest of the design) +wgpu textures can't cross devices, and a decoded frame is a wgpu texture imported from a DMA-BUF — +which **requires a device with the DMA-BUF-import extensions** (`VK_EXT_image_drm_format_modifier` ++ external-memory), built via wgpu-hal `device_from_raw` (the safe `DeviceDescriptor` can't add +them). So a hardware-decoded frame is only usable by a compositor running on **such** a device. +- **Export** composites on the encoder's custom device → already fine. +- **Preview** composites on eframe's *normal* device → can't import DMA-BUFs → can't use HW frames. + +Since **preview must HW-decode 4K** (software 4K decode ≈19 ms/frame), the resolution is a **single +shared custom device** used by eframe + preview compositor + decoder + encoder. eframe 0.33 (local +`egui-fork`) accepts it via `WgpuSetup::Existing { instance, adapter, device, queue }` — confirmed. +The earlier "separate export device" becomes redundant once this lands. + +### Stage 3a — windowed shared `DrmDevice`, injected into eframe (highest-risk; blind) +Today `vk_device::create()` is **headless**. Make a windowed variant (or extend it) that is a +**superset** device: DMA-BUF import ext **+** `VK_KHR_swapchain` (device) and the WSI surface +instance extensions, **+** everything eframe/egui/vello need — `adapter.limits()` (already; Vello +needs `max_storage_buffers_per_shader_stage` ≥ 5), `max_texture_dimension_2d` 8192, and the optional +features main.rs requests (`SHADER_F16`, `TIMESTAMP_QUERY[_INSIDE_ENCODERS]`). Pick the adapter that +is the **VAAPI GPU** (the render node must match libva's, or DMA-BUF sharing fails on multi-GPU). +- main.rs: try to build the shared device; on success pass `WgpuSetup::Existing`, else fall back to + the current `WgpuSetupCreateNew` (software decode only). Gate on Linux + VAAPI + a config/env + override; **must be bulletproof** — this device now renders *every* frame of *every* session for + Linux/VAAPI users, video or not. Milestone: editor runs normally on it with no video involved. + +### Stage 3b — VideoManager hardware decode on the shared device (blind) +- `VideoManager` holds a `VaapiDecoder` per HW-decodable clip (built on the shared device), plus the + software `VideoDecoder` fallback. `get_frame` gains a GPU-returning variant: yields an imported NV12 + texture pair (native res) instead of `Arc>`. Probe HW support per source; non-VAAPI / + unsupported codecs / non-Linux → software path (Stage 1, target-res). +- Cache native GPU textures keyed by (clip, ts); revisit the byte budget (4K NV12 ≈ 12 MB each). + +### Stage 3c — compositor consumes the GPU frame (blind; user-verifies) +- The video-instance composite path takes an NV12 texture (or a small NV12→RGB GPU pass) and blits it + to the target with the existing bilinear blit — **no `write_texture` upload**. GPU scales native→ + target (preview res or export res). Both preview and the zero-copy export become + decode→composite(→encode) with no CPU frame. Software frames still upload as today. ## Critical files - `lightningbeam-core/src/video.rs` — `VideoDecoder` (per-request output size, scaler cache), @@ -86,13 +118,21 @@ at the requested target res. This fixes the 4K decode wall, the 8 MB upload, *an - `gpu-video-encoder/` (→ `gpu-video-codec`) — `dmabuf.rs`/`vk_device.rs` reused for the decode import. ## Risks +- **Shared custom device is the editor's main device (BIGGEST risk)** — Stage 3a makes a hand-built + wgpu-hal Vulkan device render every frame for Linux/VAAPI users. It must satisfy eframe + egui + + vello + winit presentation across varied Intel/AMD/Mesa stacks, or the editor won't start. Mitigate + with a strict try-and-fall-back-to-normal-device path + an env/config kill switch. Test broadly. +- **Multi-GPU** — the shared render device must be the *same* GPU as libva's VAAPI device, or DMA-BUF + import fails. Adapter selection must match the render node to the VAAPI node (laptops with iGPU + + dGPU, PRIME). - **Codec coverage** — only some codecs are HW-decodable per GPU/driver; software must stay correct - and well-tested. Selection must probe support per source, not assume. -- **Cache memory** — native-res GPU textures (esp. 4K) are large; the frame cache budget needs revisiting. -- **Colorspace/format** — VAAPI decode surfaces are NV12/tiled; the existing import handles NV12, but - 10-bit/HDR sources (P010) need format handling. -- **Preview vs export sharing** — two live targets (preview res + export res) from the same source; the - cache/scaler design must serve both without thrashing. + and well-tested. Probe support per source, don't assume. +- **Cache memory** — native-res GPU textures (esp. 4K NV12 ≈12 MB) are large; revisit the frame cache + budget, and the two live targets (preview res + export res) shouldn't thrash. +- **Colorspace/format** — VAAPI decode surfaces are NV12/tiled; import handles NV12, but 10-bit/HDR + (P010) needs format handling. Decoded NV12 also needs the right BT.601/709 + range on the NV12→RGB + read (mirror the encoder's color tags, [[gpu-video-decode]] color-range work). +- **Non-Linux / no-VAAPI** — must cleanly run on the normal eframe device with software decode. ## Verification - Stage 0/1: visual — export above document res is now full-quality (not upscaled); profile shows diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs index 705c9d5..687ed15 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs @@ -198,7 +198,7 @@ impl Action for AddClipInstanceAction { // Calculate internal start/end from trim parameters let internal_start = self.clip_instance.trim_start; - let internal_end = self.clip_instance.trim_end.unwrap_or(clip.duration); + let internal_end = self.clip_instance.trim_end.unwrap_or(clip.content_duration().native()); let external_start = self.clip_instance.timeline_start; // Calculate external duration (for looping if timeline_duration is set). @@ -240,7 +240,7 @@ impl Action for AddClipInstanceAction { // `trim_*` / `clip.duration` are in SECONDS (audio content time), // while `timeline_*` and the backend's `duration` are in BEATS. let internal_start = self.clip_instance.trim_start; - let internal_end = self.clip_instance.trim_end.unwrap_or(clip.duration); + let internal_end = self.clip_instance.trim_end.unwrap_or(clip.content_duration().native()); let start_time = self.clip_instance.timeline_start; // `effective_duration` is in BEATS. When `timeline_duration` is set // it already is; otherwise the clip occupies its natural content diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs index b275b22..4e05db1 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/loop_clip_instances.rs @@ -129,7 +129,7 @@ impl LoopClipInstancesAction { }; let content_window = { - let trim_end = instance.trim_end.unwrap_or(clip.duration); + let trim_end = instance.trim_end.unwrap_or(clip.content_duration().native()); (trim_end - instance.trim_start).max(0.0) // seconds }; // Natural content length as a beats span at the clip's start (the diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs index 76af639..947e6e0 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/remove_clip_instances.rs @@ -170,7 +170,7 @@ impl Action for RemoveClipInstancesAction { use daw_backend::command::{Query, QueryResponse}; let internal_start = instance.trim_start; - let internal_end = instance.trim_end.unwrap_or(clip.duration); + let internal_end = instance.trim_end.unwrap_or(clip.content_duration().native()); let external_start = instance.timeline_start; // MIDI trims are beats-domain, so the fallback span is beats too. let external_duration = instance @@ -198,7 +198,7 @@ impl Action for RemoveClipInstancesAction { } AudioClipType::Sampled { audio_pool_index } => { let internal_start = instance.trim_start; - let internal_end = instance.trim_end.unwrap_or(clip.duration); + let internal_end = instance.trim_end.unwrap_or(clip.content_duration().native()); let start_time = instance.timeline_start; // Fallback span is the content seconds converted to beats at the // clip's start (not the seconds span treated as beats). diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs b/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs index fb3095e..1e2c6d4 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/split_clip_instance.rs @@ -377,7 +377,7 @@ impl Action for SplitClipInstanceAction { // 1. Trim the original (left) instance let orig_internal_start = original_instance.trim_start; - let orig_internal_end = original_instance.trim_end.unwrap_or(clip.duration); + let orig_internal_end = original_instance.trim_end.unwrap_or(clip.content_duration().native()); // Look up the original backend instance ID if let Some(crate::action::BackendClipInstanceId::Midi(orig_backend_id)) = @@ -388,7 +388,7 @@ impl Action for SplitClipInstanceAction { // 2. Add the new (right) instance let internal_start = new_instance.trim_start; - let internal_end = new_instance.trim_end.unwrap_or(clip.duration); + let internal_end = new_instance.trim_end.unwrap_or(clip.content_duration().native()); let external_start = new_instance.timeline_start; // MIDI trims are beats-domain, so the fallback span is beats too. let external_duration = new_instance @@ -425,7 +425,7 @@ impl Action for SplitClipInstanceAction { AudioClipType::Sampled { audio_pool_index } => { // 1. Trim the original (left) instance let orig_internal_start = original_instance.trim_start; - let orig_internal_end = original_instance.trim_end.unwrap_or(clip.duration); + let orig_internal_end = original_instance.trim_end.unwrap_or(clip.content_duration().native()); // Look up the original backend instance ID if let Some(crate::action::BackendClipInstanceId::Audio(orig_backend_id)) = @@ -436,7 +436,7 @@ impl Action for SplitClipInstanceAction { // 2. Add the new (right) instance let internal_start = new_instance.trim_start; - let internal_end = new_instance.trim_end.unwrap_or(clip.duration); + let internal_end = new_instance.trim_end.unwrap_or(clip.content_duration().native()); let start_time = new_instance.timeline_start; // Fallback span is the content seconds converted to beats at the // clip's start (not the seconds span treated as beats). @@ -499,7 +499,7 @@ impl Action for SplitClipInstanceAction { if let Some(instance) = al.clip_instances.iter().find(|ci| ci.id == self.instance_id) { if let Some(clip) = document.get_audio_clip(&instance.clip_id) { let orig_internal_start = instance.trim_start; - let orig_internal_end = self.original_trim_end.unwrap_or(clip.duration); + let orig_internal_end = self.original_trim_end.unwrap_or(clip.content_duration().native()); // Restore based on clip type use crate::clip::AudioClipType; diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs index 55fa95d..d4d48bc 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/trim_clip_instances.rs @@ -424,7 +424,7 @@ impl Action for TrimClipInstancesAction { // Calculate new internal_start and internal_end for backend // Note: instance already has the new trim values after execute() let internal_start = instance.trim_start; - let internal_end = instance.trim_end.unwrap_or(clip.duration); + let internal_end = instance.trim_end.unwrap_or(clip.content_duration().native()); // Handle trim based on clip type match &clip.clip_type { @@ -517,8 +517,8 @@ impl Action for TrimClipInstancesAction { TrimType::TrimRight => instance.trim_start, // trim_start wasn't changed }; let internal_end = match trim_type { - TrimType::TrimLeft => instance.trim_end.unwrap_or(clip.duration), // trim_end wasn't changed - TrimType::TrimRight => old.trim_value.unwrap_or(clip.duration), + TrimType::TrimLeft => instance.trim_end.unwrap_or(clip.content_duration().native()), // trim_end wasn't changed + TrimType::TrimRight => old.trim_value.unwrap_or(clip.content_duration().native()), }; // Handle trim based on clip type diff --git a/lightningbeam-ui/lightningbeam-core/src/clip.rs b/lightningbeam-ui/lightningbeam-core/src/clip.rs index 2c216df..dd294c4 100644 --- a/lightningbeam-ui/lightningbeam-core/src/clip.rs +++ b/lightningbeam-ui/lightningbeam-core/src/clip.rs @@ -468,6 +468,37 @@ pub enum AudioClipType { Recording, } +/// A clip's content duration, tagged by its native unit. +/// +/// Sampled/recording audio and video measure content in wall-clock **seconds**; MIDI measures +/// it in **beats** (tempo-independent musical length). Carrying the domain in the type means a +/// duration can't be silently read in the wrong unit. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ClipDuration { + Seconds(Seconds), + Beats(Beats), +} + +impl ClipDuration { + /// Wall-clock seconds. Beats are converted as a length from beat 0 (exact under constant + /// tempo; a reasonable approximation otherwise — durations are position-independent here). + pub fn to_seconds(self, tempo_map: &daw_backend::TempoMap) -> Seconds { + match self { + ClipDuration::Seconds(s) => s, + ClipDuration::Beats(b) => tempo_map.beats_to_seconds(b), + } + } + + /// The raw magnitude in the clip's native unit. Use only in code that already works in that + /// domain (e.g. trim math, whose values share the clip's native domain). + pub fn native(self) -> f64 { + match self { + ClipDuration::Seconds(s) => s.seconds_to_f64(), + ClipDuration::Beats(b) => b.beats_to_f64(), + } + } +} + /// Audio clip /// /// This is compatible with daw-backend's audio system: @@ -481,9 +512,13 @@ pub struct AudioClip { /// Clip name pub name: String, - /// Duration in seconds - /// For sampled audio, this can be set to trim the audio shorter than the source file - pub duration: f64, + /// Raw content duration in the clip's **native domain** — SECONDS for sampled/recording + /// audio, BEATS for MIDI (musical length, tempo-independent). Private on purpose: the domain + /// depends on `clip_type`, so all access goes through [`AudioClip::content_duration`] / + /// [`AudioClip::set_content_duration`], which keep it type-safe. Stored as a bare `f64` + /// because the `.beam` format serializes it as a plain number (serde derives over private + /// fields fine); a domain-tagged newtype would change the on-disk shape. + duration: f64, /// Audio clip type (sampled or MIDI) pub clip_type: AudioClipType, @@ -494,6 +529,31 @@ pub struct AudioClip { } impl AudioClip { + /// The clip's content duration, tagged with its native domain (seconds for sampled/recording, + /// beats for MIDI). This is the only sanctioned way to read the raw `duration` field. + pub fn content_duration(&self) -> ClipDuration { + match self.clip_type { + AudioClipType::Midi { .. } => ClipDuration::Beats(Beats(self.duration)), + AudioClipType::Sampled { .. } | AudioClipType::Recording => { + ClipDuration::Seconds(Seconds(self.duration)) + } + } + } + + /// Set the content duration. Debug-asserts the value's domain matches the clip type so a + /// beats duration can't be stored on a seconds clip (or vice-versa). + pub fn set_content_duration(&mut self, duration: ClipDuration) { + debug_assert!( + matches!( + (&self.clip_type, duration), + (AudioClipType::Midi { .. }, ClipDuration::Beats(_)) + | (AudioClipType::Sampled { .. } | AudioClipType::Recording, ClipDuration::Seconds(_)) + ), + "clip duration domain must match clip type", + ); + self.duration = duration.native(); + } + /// Create a new sampled audio clip /// /// # Arguments diff --git a/lightningbeam-ui/lightningbeam-core/src/document.rs b/lightningbeam-ui/lightningbeam-core/src/document.rs index 9596f74..a26b86b 100644 --- a/lightningbeam-ui/lightningbeam-core/src/document.rs +++ b/lightningbeam-ui/lightningbeam-core/src/document.rs @@ -472,8 +472,10 @@ impl Document { } crate::layer::AnyLayer::Audio(audio_layer) => { for instance in &audio_layer.clip_instances { - if let Some(clip) = self.audio_clips.get(&instance.clip_id) { - let end_time = calculate_instance_end(instance, clip.duration); + // get_clip_duration yields seconds (converting MIDI's beats duration), + // which is what the closure expects. + if let Some(clip_duration) = self.get_clip_duration(&instance.clip_id) { + let end_time = calculate_instance_end(instance, clip_duration.seconds_to_f64()); max_end_time = max_end_time.max(end_time); } } @@ -516,8 +518,8 @@ impl Document { } crate::layer::AnyLayer::Audio(al) => { for inst in &al.clip_instances { - if let Some(clip) = doc.audio_clips.get(&inst.clip_id) { - *max_end = max_end.max(calc_end(inst, clip.duration)); + if let Some(clip_duration) = doc.get_clip_duration(&inst.clip_id) { + *max_end = max_end.max(calc_end(inst, clip_duration.seconds_to_f64())); } } } @@ -908,7 +910,7 @@ impl Document { // Avoid deep recursion — use stored duration for nested vector clips Some(vc.content_duration(self.framerate, tempo_map)) } else if let Some(ac) = self.audio_clips.get(id) { - Some(ac.duration) + Some(ac.content_duration().to_seconds(tempo_map).seconds_to_f64()) } else if let Some(vc) = self.video_clips.get(id) { Some(vc.duration) } else if self.effect_definitions.contains_key(id) { @@ -921,15 +923,9 @@ impl Document { } else if let Some(clip) = self.video_clips.get(clip_id) { Some(Seconds(clip.duration)) } else if let Some(clip) = self.audio_clips.get(clip_id) { - // MIDI clips store `duration` in BEATS (they share the AudioClip struct with - // sampled clips, whose duration is seconds). Convert to wall-clock seconds so - // the content-window sizing works uniformly. - match clip.clip_type { - crate::clip::AudioClipType::Midi { .. } => { - Some(self.tempo_map().beats_to_seconds(Beats(clip.duration))) - } - _ => Some(Seconds(clip.duration)), - } + // Interpret the clip's native-domain duration as wall-clock seconds (MIDI stores + // beats, sampled stores seconds — content_duration keeps that straight). + Some(clip.content_duration().to_seconds(self.tempo_map())) } else if self.effect_definitions.contains_key(clip_id) { // Effects have infinite internal duration - their timeline length // is controlled by ClipInstance.trim_end diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 7f98849..6ad2a5b 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -2,6 +2,7 @@ use eframe::egui; use daw_backend::{Beats, Seconds}; +use lightningbeam_core::clip::ClipDuration; use lightningbeam_core::layer::{AnyLayer, AudioLayer}; use lightningbeam_core::layout::{LayoutDefinition, LayoutNode}; use lightningbeam_core::pane::PaneType; @@ -5879,7 +5880,7 @@ impl EditorApp { // Get audio clip duration for logging let duration = self.action_executor.document().audio_clips .get(&audio_clip_id) - .map(|c| c.duration) + .map(|c| c.content_duration().native()) .unwrap_or(0.0); println!("✅ Extracted audio from '{}' ({:.1}s, {}ch, {}Hz) - AudioClip ID: {}", @@ -6477,7 +6478,7 @@ impl eframe::App for EditorApp { if let Some(doc_clip_id) = doc_clip_id { if let Some(clip) = self.action_executor.document_mut().audio_clips.get_mut(&doc_clip_id) { if clip.is_recording() { - clip.duration = duration.seconds_to_f64(); + clip.set_content_duration(ClipDuration::Seconds(duration)); } } } @@ -6650,7 +6651,7 @@ impl eframe::App for EditorApp { } // Update the clip's duration so the timeline bar grows if let Some(clip) = self.action_executor.document_mut().audio_clips.get_mut(&doc_clip_id) { - clip.duration = duration.beats_to_f64(); + clip.set_content_duration(ClipDuration::Beats(duration)); } } } @@ -6686,7 +6687,7 @@ impl eframe::App for EditorApp { .map(|(id, _)| id); if let Some(doc_clip_id) = doc_clip_id { if let Some(clip) = self.action_executor.document_mut().audio_clips.get_mut(&doc_clip_id) { - clip.duration = midi_clip_data.duration; + clip.set_content_duration(ClipDuration::Beats(Beats(midi_clip_data.duration))); clip.name = format!("MIDI Recording {}", clip_id); } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs index cc39985..d3d0079 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs @@ -932,7 +932,7 @@ impl AssetLibraryPane { name: clip.name.clone(), category: AssetCategory::Audio, drag_clip_type, - duration: clip.duration, + duration: clip.content_duration().native(), dimensions: None, extra_info, is_builtin: false, @@ -1136,7 +1136,7 @@ impl AssetLibraryPane { name: clip.name.clone(), category: AssetCategory::Audio, drag_clip_type, - duration: clip.duration, + duration: clip.content_duration().native(), dimensions: None, extra_info, is_builtin: false, @@ -1802,7 +1802,7 @@ impl AssetLibraryPane { AudioClipType::Midi { midi_clip_id } => { let note_color = egui::Color32::from_rgb(100, 200, 100); if let Some(events) = shared.midi_event_cache.get(midi_clip_id) { - Some(generate_midi_thumbnail(events, clip.duration, bg_color, note_color)) + Some(generate_midi_thumbnail(events, clip.content_duration().native(), bg_color, note_color)) } else { Some(generate_placeholder_thumbnail(AssetCategory::Audio, 200)) } @@ -2358,7 +2358,7 @@ impl AssetLibraryPane { AudioClipType::Midi { midi_clip_id } => { let note_color = egui::Color32::from_rgb(100, 200, 100); if let Some(events) = shared.midi_event_cache.get(midi_clip_id) { - Some(generate_midi_thumbnail(events, clip.duration, bg_color, note_color)) + Some(generate_midi_thumbnail(events, clip.content_duration().native(), bg_color, note_color)) } else { Some(generate_placeholder_thumbnail(AssetCategory::Audio, 200)) } @@ -2495,7 +2495,7 @@ impl AssetLibraryPane { AudioClipType::Midi { midi_clip_id } => { let note_color = egui::Color32::from_rgb(100, 200, 100); if let Some(events) = shared.midi_event_cache.get(midi_clip_id) { - Some(generate_midi_thumbnail(events, clip.duration, bg_color, note_color)) + Some(generate_midi_thumbnail(events, clip.content_duration().native(), bg_color, note_color)) } else { Some(generate_placeholder_thumbnail(AssetCategory::Audio, 200)) } @@ -2858,7 +2858,7 @@ impl AssetLibraryPane { let note_color = egui::Color32::from_rgb(100, 200, 100); if let Some(events) = shared.midi_event_cache.get(midi_clip_id) { - Some(generate_midi_thumbnail(events, clip.duration, bg_color, note_color)) + Some(generate_midi_thumbnail(events, clip.content_duration().native(), bg_color, note_color)) } else { Some(generate_placeholder_thumbnail(AssetCategory::Audio, 200)) } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs index d29c26b..1522f5b 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs @@ -1808,7 +1808,7 @@ impl InfopanelPane { }); ui.horizontal(|ui| { ui.label("Duration:"); - ui.label(format!("{:.2}s", clip.duration)); + ui.label(format!("{:.2}s", clip.content_duration().to_seconds(document.tempo_map()).seconds_to_f64())); }); } else { // Could be an image asset or effect — show ID diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs index 34b2145..2505ef3 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs @@ -466,7 +466,7 @@ impl PianoRollPane { for instance in &audio_layer.clip_instances { if let Some(clip) = document.audio_clips.get(&instance.clip_id) { if let AudioClipType::Midi { midi_clip_id } = clip.clip_type { - let duration = instance.effective_duration(Seconds(clip.duration), document.tempo_map()); + let duration = instance.effective_duration(clip.content_duration().to_seconds(document.tempo_map()), document.tempo_map()); clip_data.push((midi_clip_id, instance.timeline_start.beats_to_f64(), instance.trim_start, duration.beats_to_f64(), instance.id)); } } @@ -2459,7 +2459,7 @@ impl PianoRollPane { // length converted to beats at the clip's start. let duration = instance.timeline_duration.unwrap_or_else(|| { let tmap = document.tempo_map(); - tmap.seconds_to_beats(tmap.beats_to_seconds(instance.timeline_start) + Seconds(clip.duration)) - instance.timeline_start + tmap.seconds_to_beats(tmap.beats_to_seconds(instance.timeline_start) + clip.content_duration().to_seconds(tmap)) - instance.timeline_start }); // Get sample rate from raw_audio_cache if let Some((_samples, sr, _ch)) = shared.raw_audio_cache.get(&audio_pool_index) { diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index 2ed636d..c19b724 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -209,7 +209,9 @@ fn effective_clip_duration( document.get_clip_duration(&clip_instance.clip_id) } } - AnyLayer::Audio(_) => document.get_audio_clip(&clip_instance.clip_id).map(|c| Seconds(c.duration)), + // Delegate to get_clip_duration so MIDI clips (whose `duration` is stored in beats, + // not seconds) are converted correctly rather than read as raw seconds. + AnyLayer::Audio(_) => document.get_clip_duration(&clip_instance.clip_id), AnyLayer::Video(_) => document.get_video_clip(&clip_instance.clip_id).map(|c| Seconds(c.duration)), AnyLayer::Effect(_) => Some(Seconds(lightningbeam_core::effect::EFFECT_DURATION)), AnyLayer::Group(_) => None, @@ -3082,7 +3084,7 @@ impl TimelinePane { }; let audio_file_duration = total_frames as f64 / eff_sr as f64; - let clip_dur = Seconds(audio_clip.duration); + let clip_dur = audio_clip.content_duration().to_seconds(document.tempo_map()); let mut ci_start = ci.effective_start(); if is_move_drag && selection.contains_clip_instance(&ci.id) { ci_start = self.moved_start(ci_start, document.tempo_map(), &document.time_signature, document.framerate); @@ -4699,7 +4701,7 @@ impl TimelinePane { if selection.contains_clip_instance(&clip_instance.id) { let clip_duration = match layer { lightningbeam_core::layer::AnyLayer::Audio(_) => { - document.get_audio_clip(&clip_instance.clip_id).map(|c| c.duration) + document.get_audio_clip(&clip_instance.clip_id).map(|c| c.content_duration().native()) } _ => continue, }; @@ -4773,7 +4775,7 @@ impl TimelinePane { if selection.contains_clip_instance(&clip_instance.id) { let clip_duration = match layer { lightningbeam_core::layer::AnyLayer::Audio(_) => { - document.get_audio_clip(&clip_instance.clip_id).map(|c| c.duration) + document.get_audio_clip(&clip_instance.clip_id).map(|c| c.content_duration().native()) } _ => continue, }; From ba4395602d7d22eb54ca648f844a92227d276dd1 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Sat, 11 Jul 2026 19:23:46 -0400 Subject: [PATCH 7/8] Make recording an undoable action; fixes MIDI recording not marking doc modified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recording (audio and MIDI) mutated the document directly in the AudioEvent handlers, outside the action system — so it was never undoable, and dirty- tracking leaned on an ad-hoc `media_modified` flag that the MIDI stop handler forgot to set (hence: recording a MIDI clip didn't trigger the save-on-close prompt). Recording is temporal (the clip streams into the document live over the take), so it can't be applied by one synchronous execute(). Instead, commit the finished take as an *already-applied* action: - ActionExecutor::push_applied(action) — registers an action whose effect is already present (clears redo, bumps the epoch so the doc reads as modified, pushes to the undo stack) WITHOUT re-running execute()/execute_backend(). Undo then removes the content via rollback/rollback_backend; redo re-adds it. - AddClipInstanceAction::already_applied(...) — constructs the action pre-seeded into its post-execute state (executed + the existing backend clip id) so the first undo can remove the live-recorded clip from both doc and backend, and redo re-adds it through the normal path. - Both recording stop handlers now finalize, then push_applied this action. Keeping the clip in the document (not a transient) matters for streaming-to- disk and keeps the doc the single source of truth. Recordings now bump the epoch like every other edit, so the media_modified flag is dropped for recordings (kept only as a defensive fallback if the action can't be built). Whole workspace compiles; 299 core tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../lightningbeam-core/src/action.rs | 18 ++++++ .../src/actions/add_clip_instance.rs | 25 ++++++++ .../lightningbeam-editor/src/main.rs | 62 ++++++++++++++++--- 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/action.rs b/lightningbeam-ui/lightningbeam-core/src/action.rs index 1e4bf10..226ddfb 100644 --- a/lightningbeam-ui/lightningbeam-core/src/action.rs +++ b/lightningbeam-ui/lightningbeam-core/src/action.rs @@ -236,6 +236,24 @@ impl ActionExecutor { Ok(()) } + /// Register an action whose effect has **already been applied** to the document (and backend) + /// outside the executor — e.g. a recording, which streams its content into the document live + /// over time and can't be applied by a single synchronous `execute()`. + /// + /// Unlike `execute`, this does NOT run `execute()`/`execute_backend()` (the effect is already + /// present). It clears the redo stack, bumps the epoch (so the document reads as modified), and + /// pushes the action so it becomes undoable: undo runs `rollback`/`rollback_backend` to remove + /// the content, redo runs `execute`/`execute_backend` to bring it back. The action must be + /// constructed already in its post-execute state (see e.g. `AddClipInstanceAction::already_applied`). + pub fn push_applied(&mut self, action: Box) { + self.redo_stack.clear(); + self.epoch = self.epoch.wrapping_add(1); + self.undo_stack.push(action); + if self.undo_stack.len() > self.max_undo_depth { + self.undo_stack.remove(0); + } + } + /// Undo the last action /// /// Returns Ok(true) if an action was undone, Ok(false) if undo stack is empty, diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs index 687ed15..f39d8ad 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs @@ -47,6 +47,31 @@ impl AddClipInstanceAction { } } + /// Construct the action in its **already-applied** state: the clip instance is already in the + /// document and its backend clip already exists (e.g. a finished recording). Pair with + /// `ActionExecutor::push_applied` so the recording becomes undoable without re-adding anything. + /// Undo will `rollback`/`rollback_backend` (removing the clip from doc + backend via the seeded + /// ids); redo re-adds it via the normal `execute`/`execute_backend` path. + pub fn already_applied( + layer_id: Uuid, + clip_instance: ClipInstance, + backend_track_id: daw_backend::TrackId, + backend_id: crate::action::BackendClipInstanceId, + ) -> Self { + let (backend_midi_instance_id, backend_audio_instance_id) = match backend_id { + crate::action::BackendClipInstanceId::Midi(id) => (Some(id), None), + crate::action::BackendClipInstanceId::Audio(id) => (None, Some(id)), + }; + Self { + layer_id, + clip_instance, + executed: true, // already present in the document + backend_track_id: Some(backend_track_id), + backend_midi_instance_id, + backend_audio_instance_id, + } + } + /// Get the ID of the clip instance that will be/was added pub fn clip_instance_id(&self) -> Uuid { self.clip_instance.id diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 6ad2a5b..e97c10e 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -6547,10 +6547,8 @@ impl eframe::App for EditorApp { }; if !clip_id.is_nil() { - // Finalize the clip (update pool_index and duration) - // A finished recording (samples in the pool) needs capturing. + // Finalize the clip (update pool_index and duration). self.autosave.pending_event = true; - self.media_modified = true; if let Some(clip) = self.action_executor.document_mut().audio_clips.get_mut(&clip_id) { if clip.finalize_recording(pool_index, duration) { clip.name = format!("Recording {}", pool_index); @@ -6563,11 +6561,31 @@ impl eframe::App for EditorApp { // Map the document instance_id → the existing backend clip so that // delete/move/trim actions can reference it correctly. // DO NOT call AddAudioClipSync — that would create a duplicate clip. - self.clip_instance_to_backend_map.insert( - instance_id, - lightningbeam_core::action::BackendClipInstanceId::Audio(_backend_clip_id), - ); + let backend_id = lightningbeam_core::action::BackendClipInstanceId::Audio(_backend_clip_id); + self.clip_instance_to_backend_map.insert(instance_id, backend_id); eprintln!("[AUDIO] Mapped doc instance {} → backend clip {}", instance_id, _backend_clip_id); + + // Register the finished recording as an already-applied action so + // it bumps the epoch (marks the document modified for save-on-close + // and autosave) and can be undone/redone like any other edit. + let clip_instance = self.layer_to_track_map.get(&layer_id).copied().and_then(|track_id| { + self.action_executor.document() + .get_layer(&layer_id) + .and_then(|l| if let AnyLayer::Audio(al) = l { + al.clip_instances.iter().find(|ci| ci.id == instance_id).cloned() + } else { None }) + .map(|ci| (track_id, ci)) + }); + if let Some((track_id, clip_instance)) = clip_instance { + let action = lightningbeam_core::actions::AddClipInstanceAction::already_applied( + layer_id, clip_instance, track_id, backend_id, + ); + self.action_executor.push_applied(Box::new(action)); + } else { + // Couldn't build the action; still mark modified so the recording + // isn't silently lost on close. + self.media_modified = true; + } } } @@ -6702,9 +6720,33 @@ impl eframe::App for EditorApp { } } - // TODO: Store clip_instance_to_backend_map entry for this MIDI clip. - // The backend created the instance in create_midi_clip(), but doesn't - // report the instance_id back. Needed for move/trim operations later. + // Register the finished MIDI recording as an already-applied action so it + // marks the document modified (save-on-close / autosave) and is undoable, + // like the audio path. The backend instance id was mapped during + // MidiRecordingProgress. + if let Some(&layer_id) = self.track_to_layer_map.get(&track_id) { + let doc_clip_id = self.action_executor.document() + .audio_clip_by_midi_clip_id(clip_id).map(|(id, _)| id); + if let Some(doc_clip_id) = doc_clip_id { + let instance = self.action_executor.document() + .get_layer(&layer_id) + .and_then(|l| if let AnyLayer::Audio(al) = l { + al.clip_instances.iter().find(|ci| ci.clip_id == doc_clip_id).cloned() + } else { None }); + if let Some(instance) = instance { + if let Some(&backend_id) = self.clip_instance_to_backend_map.get(&instance.id) { + let action = lightningbeam_core::actions::AddClipInstanceAction::already_applied( + layer_id, instance, track_id, backend_id, + ); + self.action_executor.push_applied(Box::new(action)); + } else { + // No backend mapping (e.g. snapshot lookup missed); still mark + // modified so the recording isn't silently lost on close. + self.media_modified = true; + } + } + } + } // Remove this MIDI layer from active recordings if let Some(&layer_id) = self.track_to_layer_map.get(&track_id) { From a3706dd2c47206ed78f08c18e1453bcdb7db40cb Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Sun, 12 Jul 2026 09:09:17 -0400 Subject: [PATCH 8/8] Bump version to 1.0.9-alpha --- Changelog.md | 8 ++++++++ lightningbeam-ui/lightningbeam-editor/Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index d331668..ef65494 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,11 @@ +# 1.0.9-alpha: +Bugfixes: +- Fix audio recording placement: a second recording landed at the wrong spot (and clicking/dragging clips was similarly off) at any tempo other than 60 BPM — recordings now start exactly at the playhead at any tempo +- While recording a second audio clip, the live bar showed as a zero-length clip until you stopped; it now grows as you record +- MIDI clips were drawn with the wrong length (their end grew too fast) at tempos other than 120 BPM +- Recording a MIDI clip didn't mark the project as having unsaved changes, so closing or starting a new file didn't prompt to save +- Audio and MIDI recording can now be undone (and redone) + # 1.0.8-alpha: Changes: - Mobile/touch UI (experimental, testing only — not built or packaged for mobile yet; enabled on desktop with the LB_MOBILE_UI environment variable): work-in-progress phone-friendly interface with a vertical sliding-window pane stack you drag to reveal panes, a new-file intent picker, a selection inspector sheet, a keyboard-primary music surface, a Focus/Patch node editor, long-press context menus, a command palette, and landscape/orientation support diff --git a/lightningbeam-ui/lightningbeam-editor/Cargo.toml b/lightningbeam-ui/lightningbeam-editor/Cargo.toml index 2ec49a3..a24b1c6 100644 --- a/lightningbeam-ui/lightningbeam-editor/Cargo.toml +++ b/lightningbeam-ui/lightningbeam-editor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightningbeam-editor" -version = "1.0.8-alpha" +version = "1.0.9-alpha" edition = "2021" description = "Multimedia editor for audio, video and 2D animation" license = "GPL-3.0-or-later"