diff --git a/daw-backend/src/audio/disk_reader.rs b/daw-backend/src/audio/disk_reader.rs index 377f4f0..f810667 100644 --- a/daw-backend/src/audio/disk_reader.rs +++ b/daw-backend/src/audio/disk_reader.rs @@ -307,23 +307,96 @@ impl ReadAheadBuffer { // --------------------------------------------------------------------------- /// Wraps a Symphonia decoder for streaming a single compressed audio file. -struct CompressedReader { +/// +/// Public (like [`VideoAudioReader`]) only so integration tests can exercise it +/// directly; treat it as crate-internal. +pub struct CompressedReader { format_reader: Box, decoder: Box, track_id: u32, /// Current decoder position in frames. current_frame: u64, + /// Frames still to drop from the front of decoded output, so that after a + /// (coarse) seek the next emitted sample lands exactly on the target frame. + pending_discard: u64, sample_rate: u32, channels: u32, - #[allow(dead_code)] total_frames: u64, /// Temporary decode buffer. sample_buf: Option>, } +/// A seekable byte stream for packed media held in the host's project container. +/// +/// `daw-backend` stays container-agnostic: it never references the `.beam` SQLite +/// store directly. Instead the host (lightningbeam-core) implements this trait over +/// its incremental blob reader and installs a factory ([`AudioBlobSourceFactory`]) +/// into the engine, so packed compressed audio can be stream-decoded without ever +/// being fully loaded into RAM. +pub trait MediaByteSource: std::io::Read + std::io::Seek + Send + Sync { + /// Total length of the stream in bytes (Symphonia needs this for seeking). + fn byte_len(&self) -> u64; +} + +/// Opens fresh byte streams for packed media by id. Installed into the engine by +/// the host; invoked when activating a clip backed by container-packed audio. +/// (`Debug` so it can ride in the `Query` enum, which derives `Debug`.) +pub trait AudioBlobSourceFactory: Send + Sync + std::fmt::Debug { + /// Open a new independent reader for the packed media item `media_id` + /// (the UUID string stored on the audio pool entry). + fn open(&self, media_id: &str) -> Result, String>; +} + +/// Adapts a [`MediaByteSource`] to Symphonia's `MediaSource` (adds the seekable + +/// byte-length metadata Symphonia's probe/seek require). +struct SymphoniaByteSource(Box); + +impl std::io::Read for SymphoniaByteSource { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.read(buf) + } +} +impl std::io::Seek for SymphoniaByteSource { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.0.seek(pos) + } +} +impl symphonia::core::io::MediaSource for SymphoniaByteSource { + fn is_seekable(&self) -> bool { + true + } + fn byte_len(&self) -> Option { + Some(self.0.byte_len()) + } +} + +/// How to open a streaming audio source: a filesystem path (referenced media or a +/// video file) or a host-provided byte stream (container-packed media). +pub enum StreamOpen { + Path(PathBuf), + Source { + src: Box, + /// Codec/extension hint for the Symphonia probe (e.g. `"mp3"`, `"flac"`). + ext: Option, + }, +} + impl CompressedReader { + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } + + pub fn channels(&self) -> u32 { + self.channels + } + + /// Total frames from the codec header (0 if the format doesn't report it). + pub fn total_frames(&self) -> u64 { + self.total_frames + } + /// Open a compressed audio file and prepare for streaming decode. - fn open(path: &Path) -> Result { + pub fn open(path: &Path) -> Result { let file = std::fs::File::open(path).map_err(|e| format!("Failed to open file: {}", e))?; let mss = MediaSourceStream::new(Box::new(file), Default::default()); @@ -332,7 +405,21 @@ impl CompressedReader { if let Some(ext) = path.extension().and_then(|e| e.to_str()) { hint.with_extension(ext); } + Self::from_mss(mss, hint) + } + /// Open a compressed stream from a host-provided byte source (packed media). + pub fn open_source(src: Box, ext: Option<&str>) -> Result { + let mss = MediaSourceStream::new(Box::new(SymphoniaByteSource(src)), Default::default()); + let mut hint = Hint::new(); + if let Some(ext) = ext { + hint.with_extension(ext); + } + Self::from_mss(mss, hint) + } + + /// Shared probe + decoder setup over an already-built media stream. + fn from_mss(mss: MediaSourceStream, hint: Hint) -> Result { let probed = symphonia::default::get_probe() .format( &hint, @@ -368,6 +455,7 @@ impl CompressedReader { decoder, track_id, current_frame: 0, + pending_discard: 0, sample_rate, channels, total_frames, @@ -375,9 +463,17 @@ impl CompressedReader { }) } - /// Seek to a specific frame. Returns the actual frame reached (may differ - /// for compressed formats that can only seek to keyframes). - fn seek(&mut self, target_frame: u64) -> Result { + /// Seek to `target_frame`, **sample-accurately**. Uses `SeekMode::Accurate`: + /// for an elementary stream like MP3 a *coarse* seek byte-estimates the + /// position and seeds the timestamp from that estimate — which for VBR (or a + /// file whose header padding the estimate ignores) lands off by up to ~1s. + /// Accurate mode instead counts frame *headers* (no decode) from a true anchor + /// (the current position, or a rewind to the start for backward seeks), so the + /// returned `actual_ts` is exact; the small residual to `target_frame` is then + /// dropped in `decode_next`. Container formats with seek tables (FLAC/OGG) seek + /// cheaply; a long MP3 walks headers from the anchor (I/O, not decode) — a + /// per-file seek index would make that O(1) (future work). + pub fn seek(&mut self, target_frame: u64) -> Result { let seek_to = SeekTo::TimeStamp { ts: target_frame, track_id: self.track_id, @@ -385,21 +481,23 @@ impl CompressedReader { let seeked = self .format_reader - .seek(SeekMode::Coarse, seek_to) + .seek(SeekMode::Accurate, seek_to) .map_err(|e| format!("Seek failed: {}", e))?; let actual_frame = seeked.actual_ts; self.current_frame = actual_frame; + // Drop the frames between where the coarse seek landed and the target. + self.pending_discard = target_frame.saturating_sub(actual_frame); // Reset the decoder after seeking. self.decoder.reset(); - Ok(actual_frame) + Ok(target_frame) } /// Decode the next chunk of audio into `out`. Returns the number of frames /// decoded. Returns `Ok(0)` at end-of-file. - fn decode_next(&mut self, out: &mut Vec) -> Result { + pub fn decode_next(&mut self, out: &mut Vec) -> Result { out.clear(); loop { @@ -428,10 +526,22 @@ impl CompressedReader { if let Some(ref mut buf) = self.sample_buf { buf.copy_interleaved_ref(decoded); let samples = buf.samples(); - out.extend_from_slice(samples); - let frames = samples.len() / self.channels as usize; - self.current_frame += frames as u64; - return Ok(frames); + let ch = self.channels as usize; + let frames = samples.len() / ch; + + // Drop leading frames for sample-accurate seek alignment. + let discard = self.pending_discard.min(frames as u64) as usize; + self.pending_discard -= discard as u64; + out.extend_from_slice(&samples[discard * ch..]); + let emitted = frames - discard; + self.current_frame += emitted as u64; + + if emitted > 0 { + return Ok(emitted); + } + // Whole packet discarded for alignment — keep decoding so + // we never falsely report EOF (Ok(0)). + continue; } return Ok(0); @@ -445,16 +555,383 @@ impl CompressedReader { } } +// --------------------------------------------------------------------------- +// VideoAudioReader +// --------------------------------------------------------------------------- + +/// Streams the audio track out of a media file (a video container, or any audio +/// file) using FFmpeg, decoding on demand. Mirrors [`CompressedReader`]'s +/// interface so the disk reader can drive either through [`StreamSource`]. +/// +/// Seeking is **sample-accurate**: after `seek(target)`, the next `decode_next` +/// yields samples beginning at exactly `target`. FFmpeg's container seek only +/// lands at-or-before the target, so we decode forward and discard the leading +/// samples to hit the frame precisely — this keeps video audio frame-synced with +/// other (mmap/in-memory) clips. +/// +/// Public (vs. the private `CompressedReader`) only so integration tests can +/// exercise it directly; treat it as crate-internal. +pub struct VideoAudioReader { + input: ffmpeg_next::format::context::Input, + decoder: ffmpeg_next::decoder::Audio, + /// Built lazily from the first decoded frame's format/layout → interleaved f32. + resampler: Option, + stream_index: usize, + /// Seconds per stream-timestamp unit. + time_base: f64, + sample_rate: u32, + channels: u32, + total_frames: u64, + /// Absolute frame index of the next sample `decode_next` will output. + current_frame: u64, + /// Frames still to drop from the front of decoded output (seek alignment). + pending_discard: u64, + /// When set, the next decoded frame establishes the discard needed to land on + /// this absolute target frame (sample-accurate seek). + align_to: Option, +} + +impl VideoAudioReader { + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } + + pub fn channels(&self) -> u32 { + self.channels + } + + /// Estimated total audio frames (from the stream/container duration). + pub fn total_frames(&self) -> u64 { + self.total_frames + } + + pub fn open(path: &Path) -> Result { + ffmpeg_next::init().map_err(|e| e.to_string())?; + let input = ffmpeg_next::format::input(&path) + .map_err(|e| format!("Failed to open media: {}", e))?; + + // Pull stream scalars + build the decoder inside a scope so the stream + // borrow of `input` ends before we use `input` again. + let (stream_index, time_base, stream_duration, decoder) = { + let stream = input + .streams() + .best(ffmpeg_next::media::Type::Audio) + .ok_or_else(|| "No audio stream found".to_string())?; + let stream_index = stream.index(); + let time_base = f64::from(stream.time_base()); + let stream_duration = stream.duration(); + let ctx = ffmpeg_next::codec::context::Context::from_parameters(stream.parameters()) + .map_err(|e| e.to_string())?; + let decoder = ctx.decoder().audio().map_err(|e| e.to_string())?; + (stream_index, time_base, stream_duration, decoder) + }; + + let sample_rate = decoder.rate(); + let channels = decoder.channels() as u32; + + let duration_secs = if stream_duration > 0 { + stream_duration as f64 * time_base + } else if input.duration() > 0 { + input.duration() as f64 / f64::from(ffmpeg_next::ffi::AV_TIME_BASE) + } else { + 0.0 + }; + let total_frames = (duration_secs * sample_rate as f64).max(0.0) as u64; + + Ok(Self { + input, + decoder, + resampler: None, + stream_index, + time_base, + sample_rate, + channels, + total_frames, + current_frame: 0, + pending_discard: 0, + align_to: None, + }) + } + + pub fn seek(&mut self, target_frame: u64) -> Result { + let seconds = target_frame as f64 / self.sample_rate.max(1) as f64; + let ts_av = (seconds * f64::from(ffmpeg_next::ffi::AV_TIME_BASE)) as i64; + // Seek to at-or-before the target (max_ts = ts_av) so we can decode + // forward to it exactly. ffmpeg-next's `seek` wants a bounded range. + self.input + .seek(ts_av, 0..(ts_av + 1)) + .map_err(|e| format!("Seek failed: {}", e))?; + self.decoder.flush(); + self.pending_discard = 0; + self.align_to = Some(target_frame); + self.current_frame = target_frame; + // We align to the exact frame below, so the effective position IS target. + Ok(target_frame) + } + + pub fn decode_next(&mut self, out: &mut Vec) -> Result { + out.clear(); + loop { + // Drain a decoded frame if one is ready. + let mut decoded = ffmpeg_next::frame::Audio::empty(); + if self.decoder.receive_frame(&mut decoded).is_ok() { + self.ensure_layout(&mut decoded); + let n = self.emit(&decoded, out); + if n > 0 { + return Ok(n); + } + continue; // frame fully discarded by seek-alignment; keep going + } + + // Read one packet (owned), releasing the `input` borrow before decoding. + let packet = self.input.packets().next().map(|(_, p)| p); + match packet { + Some(packet) => { + if packet.stream() == self.stream_index { + self.decoder + .send_packet(&packet) + .map_err(|e| e.to_string())?; + } + } + None => { + // EOF: flush and drain whatever remains. + let _ = self.decoder.send_eof(); + let mut decoded = ffmpeg_next::frame::Audio::empty(); + if self.decoder.receive_frame(&mut decoded).is_ok() { + self.ensure_layout(&mut decoded); + return Ok(self.emit(&decoded, out)); + } + return Ok(0); + } + } + } + } + + /// Decoders for some formats (e.g. raw mono WAV) leave the frame's channel + /// layout unset. The resampler needs a concrete layout that matches the + /// frame, so fill one in from the channel count when it's missing. + fn ensure_layout(&self, frame: &mut ffmpeg_next::frame::Audio) { + if frame.channel_layout().is_empty() { + frame.set_channel_layout( + ffmpeg_next::channel_layout::ChannelLayout::default(self.channels as i32), + ); + } + } + + /// Resample one decoded frame to interleaved f32, apply any pending + /// seek-alignment discard, append to `out`, return frames emitted. + fn emit(&mut self, frame: &ffmpeg_next::frame::Audio, out: &mut Vec) -> usize { + // `frame` already has a non-empty channel layout (set by `ensure_layout` + // before this call), so the resampler config and the actual frame agree + // — otherwise swr fails with AVERROR_INPUT_CHANGED. + if self.resampler.is_none() { + match ffmpeg_next::software::resampling::Context::get( + frame.format(), + frame.channel_layout(), + self.sample_rate, + ffmpeg_next::format::Sample::F32(ffmpeg_next::format::sample::Type::Packed), + frame.channel_layout(), + self.sample_rate, + ) { + Ok(r) => self.resampler = Some(r), + Err(_) => return 0, + } + } + + let mut resampled = ffmpeg_next::frame::Audio::empty(); + if self + .resampler + .as_mut() + .unwrap() + .run(frame, &mut resampled) + .is_err() + { + return 0; + } + + // The output is packed (interleaved) f32. Read it from the raw byte plane + // `data(0)` — its length is correct (`frames * channels * 4`), whereas + // `plane::(0)` is a known ffmpeg-next footgun that reports only + // `samples()` elements (ignoring channels) and would slice out of range + // for multi-channel audio. + let ch = self.channels.max(1) as usize; + let bytes = resampled.data(0); + let n_frames = (bytes.len() / 4) / ch; + if n_frames == 0 { + return 0; + } + + // On the first frame after a seek, compute how many leading frames to + // drop so output begins exactly at the seek target. + if let Some(target) = self.align_to.take() { + let frame_start = self.pts_to_frame(frame.pts()); + self.pending_discard = target.saturating_sub(frame_start); + } + + let discard = (self.pending_discard.min(n_frames as u64)) as usize; + self.pending_discard -= discard as u64; + + let start_byte = discard * ch * 4; + let end_byte = n_frames * ch * 4; + out.extend( + bytes[start_byte..end_byte] + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])), + ); + let emitted = n_frames - discard; + self.current_frame += emitted as u64; + emitted + } + + /// Convert a stream PTS to an absolute audio frame index. + fn pts_to_frame(&self, pts: Option) -> u64 { + match pts { + Some(p) if p >= 0 => { + ((p as f64 * self.time_base) * self.sample_rate as f64).round() as u64 + } + _ => self.current_frame, + } + } +} + +// --------------------------------------------------------------------------- +// StreamSource — dispatches the disk reader over either decoder backend. +// (Wired into the reader thread in a later step.) +// --------------------------------------------------------------------------- + +/// Which decoder backend a streaming source uses. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceKind { + /// Symphonia, for compressed audio files (MP3, FLAC, OGG, …). + CompressedAudio, + /// FFmpeg, for the audio track of a video container. + VideoAudio, +} + +/// A streaming audio source backing one active clip: either Symphonia +/// ([`CompressedReader`]) or FFmpeg ([`VideoAudioReader`]). +enum StreamSource { + Compressed(CompressedReader), + Video(VideoAudioReader), +} + +impl StreamSource { + fn open(open: StreamOpen, kind: SourceKind) -> Result { + match (kind, open) { + (SourceKind::CompressedAudio, StreamOpen::Path(p)) => { + Ok(StreamSource::Compressed(CompressedReader::open(&p)?)) + } + (SourceKind::CompressedAudio, StreamOpen::Source { src, ext }) => { + Ok(StreamSource::Compressed(CompressedReader::open_source(src, ext.as_deref())?)) + } + (SourceKind::VideoAudio, StreamOpen::Path(p)) => { + Ok(StreamSource::Video(VideoAudioReader::open(&p)?)) + } + (SourceKind::VideoAudio, StreamOpen::Source { .. }) => { + Err("VideoAudio cannot be opened from a packed byte source".to_string()) + } + } + } + + fn sample_rate(&self) -> u32 { + match self { + StreamSource::Compressed(r) => r.sample_rate, + StreamSource::Video(r) => r.sample_rate, + } + } + + fn channels(&self) -> u32 { + match self { + StreamSource::Compressed(r) => r.channels, + StreamSource::Video(r) => r.channels, + } + } + + fn seek(&mut self, target_frame: u64) -> Result { + match self { + StreamSource::Compressed(r) => r.seek(target_frame), + StreamSource::Video(r) => r.seek(target_frame), + } + } + + fn decode_next(&mut self, out: &mut Vec) -> Result { + match self { + StreamSource::Compressed(r) => r.decode_next(out), + StreamSource::Video(r) => r.decode_next(out), + } + } + + fn total_frames(&self) -> u64 { + match self { + StreamSource::Compressed(r) => r.total_frames(), + StreamSource::Video(r) => r.total_frames(), + } + } +} + +/// Decode a media source end-to-end and build its [`WaveformPyramid`] overview, +/// streaming — only one decode chunk plus the (bounded) pyramid are ever in +/// memory, never the full sample buffer. `floor_samples_per_texel` is the +/// finest-level resolution (see [`crate::audio::waveform_pyramid`]). +pub fn build_waveform_pyramid( + path: &Path, + kind: SourceKind, + floor_samples_per_texel: u32, +) -> Result { + let src = StreamSource::open(StreamOpen::Path(path.to_path_buf()), kind)?; + build_pyramid_from_streamsource(src, floor_samples_per_texel) +} + +/// Build a waveform pyramid from a host-provided byte source (container-packed +/// compressed audio) — the load-time counterpart of [`build_waveform_pyramid`] +/// for media that has no filesystem path. +pub fn build_waveform_pyramid_from_source( + src: Box, + ext: Option<&str>, + floor_samples_per_texel: u32, +) -> Result { + let src = StreamSource::open( + StreamOpen::Source { src, ext: ext.map(|s| s.to_string()) }, + SourceKind::CompressedAudio, + )?; + build_pyramid_from_streamsource(src, floor_samples_per_texel) +} + +fn build_pyramid_from_streamsource( + mut src: StreamSource, + floor_samples_per_texel: u32, +) -> Result { + use crate::audio::waveform_pyramid::WaveformPyramidBuilder; + + let channels = src.channels(); + let mut builder = WaveformPyramidBuilder::new(channels, floor_samples_per_texel); + builder.reserve_for_frames(src.total_frames()); + + let mut buf = Vec::new(); + loop { + let frames = src.decode_next(&mut buf)?; + if frames == 0 { + break; + } + builder.push_interleaved(&buf); + } + Ok(builder.finish()) +} + // --------------------------------------------------------------------------- // DiskReaderCommand // --------------------------------------------------------------------------- /// Commands sent from the engine to the disk reader thread. pub enum DiskReaderCommand { - /// Start streaming a compressed file for a clip instance. + /// Start streaming a file for a clip instance, using the decoder backend + /// selected by `kind` (compressed audio vs. a video's audio track). `open` + /// is either a filesystem path (referenced media / video) or a host-provided + /// byte stream (container-packed media). ActivateFile { reader_id: u64, - path: PathBuf, + open: StreamOpen, + kind: SourceKind, buffer: Arc, }, /// Stop streaming for a clip instance. @@ -529,7 +1006,7 @@ impl DiskReader { mut command_rx: rtrb::Consumer, running: Arc, ) { - let mut active_files: HashMap)> = + let mut active_files: HashMap)> = HashMap::new(); let mut decode_buf = Vec::with_capacity(8192); @@ -539,18 +1016,19 @@ impl DiskReader { match cmd { DiskReaderCommand::ActivateFile { reader_id, - path, + open, + kind, buffer, - } => match CompressedReader::open(&path) { + } => match StreamSource::open(open, kind) { Ok(reader) => { - eprintln!("[DiskReader] Activated reader={}, ch={}, sr={}, path={:?}", - reader_id, reader.channels, reader.sample_rate, path); + eprintln!("[DiskReader] Activated reader={}, kind={:?}, ch={}, sr={}", + reader_id, kind, reader.channels(), reader.sample_rate()); active_files.insert(reader_id, (reader, buffer)); } Err(e) => { eprintln!( - "[DiskReader] Failed to open compressed file {:?}: {}", - path, e + "[DiskReader] Failed to open reader={} ({:?}): {}", + reader_id, kind, e ); } }, @@ -588,7 +1066,7 @@ impl DiskReader { // If the target has jumped behind or far ahead of the buffer, // seek the decoder and reset. - if target < buf_start || target > buf_end + reader.sample_rate as u64 { + if target < buf_start || target > buf_end + reader.sample_rate() as u64 { buffer.reset(target); let _ = reader.seek(target); continue; @@ -607,7 +1085,7 @@ impl DiskReader { let buf_valid = buffer.valid_frames_count(); let buf_end = buf_start + buf_valid; let prefetch_target = - target + (PREFETCH_SECONDS * reader.sample_rate as f64) as u64; + target + (PREFETCH_SECONDS * reader.sample_rate() as f64) as u64; if buf_end >= prefetch_target { continue; // Already filled far enough ahead. @@ -649,3 +1127,7 @@ impl Drop for DiskReader { } } } + +// Tests for VideoAudioReader live in `daw-backend/tests/video_audio_stream.rs` +// (integration tests) so they build the lib in normal mode, independent of +// pre-existing breakage in the crate's `#[cfg(test)]` unit tests (automation.rs). diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index 8d59698..4991346 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -91,6 +91,10 @@ pub struct Engine { // Disk reader for streaming playback of compressed files disk_reader: Option, + // Host-installed factory for opening container-packed audio as a byte stream + // (set on load, before the project's clips are bulk-activated for streaming). + blob_source_factory: Option>, + // Input monitoring and metering input_monitoring: bool, input_gain: f32, @@ -176,6 +180,7 @@ impl Engine { metronome: Metronome::new(sample_rate), recording_sample_buffer: Vec::with_capacity(4096), disk_reader: Some(disk_reader), + blob_source_factory: None, input_monitoring: false, input_gain: 1.0, input_level_peak: 0.0, @@ -273,6 +278,97 @@ impl Engine { /// Rebuild the clip snapshot from the current project state. /// Call this after any command that adds, removes, or modifies clip instances. + /// Set up disk streaming for a clip backed by a streaming pool entry + /// (Compressed or VideoAudio). Sends `ActivateFile` to the disk reader and + /// returns the read-ahead buffer to attach to the clip. Returns `None` if the + /// entry isn't streamed, or a packed source can't be opened. + fn activate_streaming_for( + &mut self, + reader_id: u64, + pool_index: usize, + ) -> Option> { + use crate::audio::pool::AudioStorage; + use crate::audio::disk_reader::{DiskReader, DiskReaderCommand, SourceKind, StreamOpen}; + + // Decide how to open the source. `Packed` ⇒ via the host factory (bytes in + // the container); otherwise stream from the file path. Extract owned values + // first so the immutable pool borrow ends before we touch the disk reader. + enum OpenDesc { + Path(std::path::PathBuf), + Packed { media_id: String, ext: Option }, + } + let (kind, sample_rate, channels, desc) = { + let file = self.audio_pool.get_file(pool_index)?; + let kind = match file.storage { + AudioStorage::Compressed { .. } => SourceKind::CompressedAudio, + AudioStorage::VideoAudio { .. } => SourceKind::VideoAudio, + _ => return None, + }; + let desc = match &file.packed_media_id { + Some(id) => OpenDesc::Packed { media_id: id.clone(), ext: file.original_format.clone() }, + None => OpenDesc::Path(file.path.clone()), + }; + (kind, file.sample_rate, file.channels, desc) + }; + + let open = match desc { + OpenDesc::Path(p) => StreamOpen::Path(p), + OpenDesc::Packed { media_id, ext } => { + let factory = match self.blob_source_factory.as_ref() { + Some(f) => f, + None => { + eprintln!("[Engine] packed audio (pool {}) but no blob factory installed", pool_index); + return None; + } + }; + match factory.open(&media_id) { + Ok(src) => StreamOpen::Source { src, ext }, + Err(e) => { + eprintln!("[Engine] blob factory open({}) failed: {}", media_id, e); + return None; + } + } + } + }; + + let buffer = DiskReader::create_buffer(sample_rate, channels); + if let Some(ref mut dr) = self.disk_reader { + dr.send(DiskReaderCommand::ActivateFile { reader_id, open, kind, buffer: buffer.clone() }); + } + Some(buffer) + } + + /// Activate disk streaming for every loaded clip backed by a streaming pool + /// entry. Called after `SetProject` since loaded clips bypass `AddAudioClip`. + fn activate_all_streaming_clips(&mut self) { + use crate::audio::track::TrackNode; + // Collect (track, clip, pool) first via an immutable walk, then activate + // (needs &mut self) and attach the buffer back to the clip. + let targets: Vec<(TrackId, u64, usize)> = self + .project + .track_iter() + .filter_map(|(track_id, node)| match node { + TrackNode::Audio(t) => Some((track_id, t)), + _ => None, + }) + .flat_map(|(track_id, t)| { + t.clips + .iter() + .map(move |c| (track_id, c.id as u64, c.audio_pool_index)) + }) + .collect(); + + for (track_id, clip_id, pool_index) in targets { + if let Some(buffer) = self.activate_streaming_for(clip_id, pool_index) { + if let Some(TrackNode::Audio(track)) = self.project.get_track_mut(track_id) { + if let Some(clip) = track.clips.iter_mut().find(|c| c.id as u64 == clip_id) { + clip.read_ahead = Some(buffer); + } + } + } + } + } + fn refresh_clip_snapshot(&self) { let mut snap = self.clip_snapshot.write().unwrap(); snap.audio.clear(); @@ -914,7 +1010,7 @@ impl Engine { let start_secs = self.tempo_map.beats_to_seconds(start_beats); let end_secs = self.tempo_map.beats_to_seconds(end_beats); let content_dur_secs = (end_secs - start_secs).seconds_to_f64(); - let clip = AudioClipInstance::new( + let mut clip = AudioClipInstance::new( clip_id, pool_index, Seconds(offset), @@ -923,6 +1019,13 @@ impl Engine { Beats(duration), ); + // If the source is streamed (a compressed audio file, or a video's + // audio track), set up disk streaming instead of an in-memory decode. + // Each clip instance gets its own read-ahead buffer keyed by clip_id. + if let Some(buffer) = self.activate_streaming_for(clip_id as u64, pool_index) { + clip.read_ahead = Some(buffer); + } + // Add clip to track if let Some(crate::audio::track::TrackNode::Audio(track)) = self.project.get_track_mut(track_id) { track.clips.push(clip); @@ -2313,6 +2416,34 @@ impl Engine { } } + /// Add a video file's audio track as a streaming pool entry (FFmpeg, decoded + /// on demand — no extraction). Probes the audio track for channels/rate/frames + /// without decoding, and returns the pool index. Playback activation + /// (per-clip read-ahead) is wired separately when a clip references it. + fn do_add_video_audio(&mut self, path: &std::path::Path) -> Result { + use crate::audio::disk_reader::VideoAudioReader; + + let reader = VideoAudioReader::open(path)?; + let channels = reader.channels(); + let sample_rate = reader.sample_rate(); + let total_frames = reader.total_frames(); + drop(reader); + + let audio_file = crate::audio::pool::AudioFile::from_video_audio( + path.to_path_buf(), + channels, + sample_rate, + total_frames, + ); + let pool_index = self.audio_pool.add_file(audio_file); + + eprintln!( + "[ENGINE] AddVideoAudio: ch={}, sr={}, total_frames={}, pool_index={}, path={:?}", + channels, sample_rate, total_frames, pool_index, path + ); + Ok(pool_index) + } + /// Import an audio file into the pool: mmap for PCM, streaming for compressed. /// Returns the pool index on success. Emits AudioFileReady event. fn do_import_audio(&mut self, path: &std::path::Path) -> Result { @@ -2738,10 +2869,14 @@ impl Engine { Query::GetPoolAudioSamples(pool_index) => { match self.audio_pool.get_file(pool_index) { Some(file) => { - // For Compressed storage, return decoded_for_waveform if available + // For streamed (Compressed/VideoAudio) storage, return the + // progressively-decoded waveform overview if available. let samples = match &file.storage { crate::audio::pool::AudioStorage::Compressed { decoded_for_waveform, decoded_frames, .. + } + | crate::audio::pool::AudioStorage::VideoAudio { + decoded_for_waveform, decoded_frames, .. } if *decoded_frames > 0 => { decoded_for_waveform.clone() } @@ -2794,77 +2929,12 @@ impl Engine { self.refresh_clip_snapshot(); result } - Query::AddAudioFileSync(path, data, channels, sample_rate) => { - // Add audio file to pool and return the pool index - // Detect original format from file extension - let path_buf = std::path::PathBuf::from(&path); - let original_format = path_buf.extension() - .and_then(|ext| ext.to_str()) - .map(|s| s.to_lowercase()); - - // Create AudioFile and add to pool - let audio_file = crate::audio::pool::AudioFile::with_format( - path_buf.clone(), - data.clone(), // Clone data for background thread - channels, - sample_rate, - original_format, - ); - let pool_index = self.audio_pool.add_file(audio_file); - - // Generate Level 0 (overview) waveform chunks asynchronously in background thread - let chunk_tx = self.chunk_generation_tx.clone(); - let duration = data.len() as f64 / (sample_rate as f64 * channels as f64); - println!("🔄 [ENGINE] Spawning background thread to generate Level 0 chunks for pool {}", pool_index); - std::thread::spawn(move || { - // Create temporary AudioFile for chunk generation - let temp_audio_file = crate::audio::pool::AudioFile::with_format( - path_buf, - data, - channels, - sample_rate, - None, - ); - - // Generate Level 0 chunks - let chunk_count = crate::audio::waveform_cache::WaveformCache::calculate_chunk_count(duration, 0); - println!("🔄 [BACKGROUND] Generating {} Level 0 chunks for pool {}", chunk_count, pool_index); - let chunks = crate::audio::waveform_cache::WaveformCache::generate_chunks( - &temp_audio_file, - pool_index, - 0, // Level 0 (overview) - &(0..chunk_count).collect::>(), - ); - - // Send chunks via MPSC channel (will be forwarded by audio thread) - if !chunks.is_empty() { - println!("📤 [BACKGROUND] Generated {} chunks, sending to audio thread (pool {})", chunks.len(), pool_index); - let event_chunks: Vec<(u32, (f64, f64), Vec)> = chunks - .into_iter() - .map(|chunk| (chunk.chunk_index, chunk.time_range, chunk.peaks)) - .collect(); - - match chunk_tx.send(AudioEvent::WaveformChunksReady { - pool_index, - detail_level: 0, - chunks: event_chunks, - }) { - Ok(_) => println!("✅ [BACKGROUND] Chunks sent successfully for pool {}", pool_index), - Err(e) => eprintln!("❌ [BACKGROUND] Failed to send chunks: {}", e), - } - } else { - eprintln!("⚠️ [BACKGROUND] No chunks generated for pool {}", pool_index); - } - }); - - // Notify UI about the new audio file (for event listeners) - let _ = self.event_tx.push(AudioEvent::AudioFileAdded(pool_index, path)); - - QueryResponse::AudioFileAddedSync(Ok(pool_index)) - } Query::ImportAudioSync(path) => { QueryResponse::AudioImportedSync(self.do_import_audio(&path)) } + Query::AddVideoAudioSync(path) => { + QueryResponse::AudioImportedSync(self.do_add_video_audio(&path)) + } Query::GetProject => { // Save graph presets before cloning — AudioTrack::clone() creates // a fresh default graph (not a copy), so the preset must be populated @@ -2879,11 +2949,19 @@ impl Engine { match project.rebuild_audio_graphs(self.buffer_pool.buffer_size()) { Ok(()) => { self.project = project; + // Loaded clips bypass AddAudioClip, so their disk streaming was + // never activated — do it now for every streaming-backed clip. + self.activate_all_streaming_clips(); + self.refresh_clip_snapshot(); QueryResponse::ProjectSet(Ok(())) } Err(e) => QueryResponse::ProjectSet(Err(format!("Failed to rebuild audio graphs: {}", e))), } } + Query::SetBlobSourceFactory(factory) => { + self.blob_source_factory = Some(factory); + QueryResponse::BlobSourceFactorySet(Ok(())) + } Query::DuplicateMidiClipSync(clip_id) => { match self.project.midi_clip_pool.duplicate_clip(clip_id) { Some(new_id) => QueryResponse::MidiClipDuplicated(Ok(new_id)), @@ -3395,16 +3473,6 @@ impl EngineController { } } - /// Add an audio file to the pool synchronously and get the pool index - /// Returns the pool index where the audio file was added - pub fn add_audio_file_sync(&mut self, path: String, data: Vec, channels: u32, sample_rate: u32) -> Result { - let query = Query::AddAudioFileSync(path, data, channels, sample_rate); - match self.send_query(query)? { - QueryResponse::AudioFileAddedSync(result) => result, - _ => Err("Unexpected query response".to_string()), - } - } - /// Import an audio file asynchronously. The engine will memory-map WAV/AIFF /// files for instant availability, or set up stream decoding for compressed /// formats. Listen for `AudioEvent::AudioFileReady` to get the pool index. @@ -3427,6 +3495,30 @@ impl EngineController { } } + /// Add a video file's audio track as a streaming pool entry (decoded on + /// demand via FFmpeg — no extraction to disk or RAM). Probes the audio track + /// and returns the pool index. Use this for a video clip's embedded audio. + pub fn add_video_audio_sync(&mut self, path: std::path::PathBuf) -> Result { + let query = Query::AddVideoAudioSync(path); + match self.send_query(query)? { + QueryResponse::AudioImportedSync(result) => result, + _ => Err("Unexpected query response".to_string()), + } + } + + /// Install the host's packed-media byte-source factory. Must be called before + /// loading a project so its container-packed audio can be streamed (the disk + /// reader opens packed entries through this factory). + pub fn set_blob_source_factory( + &mut self, + factory: std::sync::Arc, + ) -> Result<(), String> { + match self.send_query(Query::SetBlobSourceFactory(factory))? { + QueryResponse::BlobSourceFactorySet(result) => result, + _ => Err("Unexpected query response".to_string()), + } + } + /// Generate the next unique audio clip instance ID (atomic, thread-safe) pub fn next_audio_clip_id(&self) -> AudioClipInstanceId { self.next_audio_clip_id.fetch_add(1, Ordering::Relaxed) diff --git a/daw-backend/src/audio/mod.rs b/daw-backend/src/audio/mod.rs index 6d7d771..8d79ad8 100644 --- a/daw-backend/src/audio/mod.rs +++ b/daw-backend/src/audio/mod.rs @@ -15,10 +15,12 @@ pub mod recording; pub mod sample_loader; pub mod track; pub mod waveform_cache; +pub mod waveform_pyramid; pub use automation::{AutomationLane, AutomationLaneId, AutomationPoint, CurveType, ParameterId}; pub use buffer_pool::BufferPool; pub use clip::{AudioClipInstance, AudioClipInstanceId, Clip, ClipId}; +pub use disk_reader::{AudioBlobSourceFactory, MediaByteSource}; pub use engine::{AudioClipSnapshot, Engine, EngineController}; pub use export::{export_audio, ExportFormat, ExportSettings}; pub use metronome::Metronome; diff --git a/daw-backend/src/audio/pool.rs b/daw-backend/src/audio/pool.rs index ecf0745..8748859 100644 --- a/daw-backend/src/audio/pool.rs +++ b/daw-backend/src/audio/pool.rs @@ -83,6 +83,16 @@ pub enum AudioStorage { decoded_frames: u64, total_frames: u64, }, + + /// Audio track of a video container, decoded on demand via FFmpeg + /// (`VideoAudioReader`). The source video file is `AudioFile::path`. Like + /// `Compressed`, playback is streamed through the disk reader and + /// `decoded_for_waveform` is filled progressively for the overview. + VideoAudio { + decoded_for_waveform: Vec, + decoded_frames: u64, + total_frames: u64, + }, } /// Audio file stored in the pool @@ -98,6 +108,10 @@ pub struct AudioFile { pub original_format: Option, /// Original compressed file bytes (preserved across save/load to avoid re-encoding) pub original_bytes: Option>, + /// When `Some`, this entry's bytes are packed in the project container (not on + /// disk at `path`); the disk reader opens them via the host's + /// `AudioBlobSourceFactory` using this media id. `None` ⇒ stream from `path`. + pub packed_media_id: Option, } impl AudioFile { @@ -112,6 +126,7 @@ impl AudioFile { frames, original_format: None, original_bytes: None, + packed_media_id: None, } } @@ -126,6 +141,7 @@ impl AudioFile { frames, original_format, original_bytes: None, + packed_media_id: None, } } @@ -158,6 +174,7 @@ impl AudioFile { frames: total_frames, original_format: Some("wav".to_string()), original_bytes: None, + packed_media_id: None, } } @@ -181,6 +198,32 @@ impl AudioFile { frames: total_frames, original_format, original_bytes: None, + packed_media_id: None, + } + } + + /// Create a placeholder AudioFile for a video's audio track. `path` is the + /// source video file; the audio is streamed on demand by the disk reader's + /// FFmpeg-backed `VideoAudioReader`. + pub fn from_video_audio( + path: PathBuf, + channels: u32, + sample_rate: u32, + total_frames: u64, + ) -> Self { + Self { + path, + storage: AudioStorage::VideoAudio { + decoded_for_waveform: Vec::new(), + decoded_frames: 0, + total_frames, + }, + channels, + sample_rate, + frames: total_frames, + original_format: None, + original_bytes: None, + packed_media_id: None, } } @@ -274,8 +317,8 @@ impl AudioFile { } written } - AudioStorage::Compressed { .. } => { - // Compressed files are read through the disk reader + AudioStorage::Compressed { .. } | AudioStorage::VideoAudio { .. } => { + // Streamed through the disk reader, not via read_samples(). 0 } } @@ -786,6 +829,18 @@ pub struct AudioPoolEntry { pub channels: u32, /// Embedded audio data (for files < 10MB) pub embedded_data: Option, + /// Stable media id (UUID string) for the SQLite `.beam` container. When set, + /// the audio bytes live in the container's `media` table keyed by this id + /// (packed storage). `None` for referenced entries (use `relative_path`) or + /// legacy ZIP-loaded entries. Populated by the file_io save/load layer. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_id: Option, + /// Transient carrier for this entry's serialized waveform-pyramid blob (LBWF + /// bytes). Never serialized into project.json — the bytes live in the + /// container's `media` table (kind `Waveform`). Set by the file_io save layer + /// (in) and load layer (out); `None` everywhere else. + #[serde(skip)] + pub waveform_blob: Option>, } impl AudioClipPool { @@ -800,6 +855,28 @@ impl AudioClipPool { let mut entries = Vec::new(); for (index, file) in self.files.iter().enumerate() { + // Packed-in-container streaming entry: its bytes already live in the + // `.beam` media table (kept in place across re-saves). Emit just the + // media id — no path, no embedded bytes, nothing to decode. + if let Some(media_id) = &file.packed_media_id { + entries.push(AudioPoolEntry { + pool_index: index, + waveform_blob: None, + name: file + .path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| format!("file_{}", index)), + relative_path: None, + duration: file.duration_seconds(), + sample_rate: file.sample_rate, + channels: file.channels, + embedded_data: None, + media_id: Some(media_id.clone()), + }); + continue; + } + let file_path = &file.path; let file_path_str = file_path.to_string_lossy(); @@ -830,6 +907,7 @@ impl AudioClipPool { let entry = AudioPoolEntry { pool_index: index, + waveform_blob: None, name: file_path .file_name() .map(|n| n.to_string_lossy().to_string()) @@ -839,6 +917,7 @@ impl AudioClipPool { sample_rate: file.sample_rate, channels: file.channels, embedded_data, + media_id: None, }; entries.push(entry); @@ -977,7 +1056,31 @@ impl AudioClipPool { let entry_start = std::time::Instant::now(); eprintln!("📊 [LOAD_SERIALIZED] Processing entry {}/{}: '{}'", i + 1, entries.len(), entry.name); - let success = if let Some(ref embedded) = entry.embedded_data { + let success = if entry.media_id.is_some() && entry.embedded_data.is_none() { + // Packed-in-container streaming entry: build a Compressed placeholder + // backed by the host blob factory (opened at clip-activation time). + // No decode here — playback streams through the disk reader. + let media_id = entry.media_id.clone().unwrap(); + let ext = std::path::Path::new(&entry.name) + .extension() + .and_then(|e| e.to_str()) + .map(|s| s.to_lowercase()); + let total_frames = (entry.duration * entry.sample_rate as f64).ceil() as u64; + let mut file = AudioFile::from_compressed( + PathBuf::from(&entry.name), + entry.channels, + entry.sample_rate, + total_frames, + ext, + ); + file.packed_media_id = Some(media_id); + if entry.pool_index < self.files.len() { + self.files[entry.pool_index] = file; + true + } else { + false + } + } else if let Some(ref embedded) = entry.embedded_data { // Load from embedded data eprintln!("📊 [LOAD_SERIALIZED] Entry has embedded data (format: {})", embedded.format); match Self::load_from_embedded_into_pool(self, entry.pool_index, embedded.clone(), &entry.name) { diff --git a/daw-backend/src/audio/waveform_pyramid.rs b/daw-backend/src/audio/waveform_pyramid.rs new file mode 100644 index 0000000..6d58d42 --- /dev/null +++ b/daw-backend/src/audio/waveform_pyramid.rs @@ -0,0 +1,292 @@ +//! Streaming min/max waveform LOD pyramid. +//! +//! A waveform pyramid is a tree of zoom levels. **Index = tree depth:** +//! `levels[0]` is the **root** (a single texel — the min/max envelope of the +//! whole file, lowest resolution); each deeper level is `BRANCH`× finer, and +//! `levels.last()` is the **floor** (one texel per `floor_samples_per_texel` +//! source frames — the finest *persisted* level). A node's children live at +//! `index + 1`, so the residency invariant ("a node is cleared only after its +//! children") reads straight off the index. +//! +//! Below the floor (finer than the floor bucket) is *not* stored; the caller +//! re-decodes the source window on demand for true per-sample detail. +//! +//! The builder is **streaming**: samples are pushed once, in order, and only the +//! finest level is accumulated (~`total_frames / floor` texels); the coarser +//! levels are derived by repeated `BRANCH:1` min/max reduction in [`finish`]. +//! This yields the identical pyramid to an in-stream cascade (each parent = the +//! min/max of its children) without ever holding the full sample buffer. +//! +//! **Ragged edges are handled by reducing over available children:** a bucket +//! whose group is partial (1..BRANCH children, or `< floor` samples at the floor) +//! simply takes the min/max of what's there — no value padding. Padding to a +//! regular shape, if ever needed, is a GPU-texture/tile concern, not the data's. +//! +//! Each texel carries per-channel min/max for up to two channels +//! (`Lmin,Lmax,Rmin,Rmax`), matching the GPU waveform texture; mono mirrors the +//! left channel into the right. +//! +//! [`finish`]: WaveformPyramidBuilder::finish + +/// Reduction factor between adjacent pyramid levels. +pub const BRANCH: u32 = 4; + +/// Default finest-level resolution (source frames per floor texel). Trades +/// on-disk pyramid size against how soon zoom-in must re-decode the source. +pub const DEFAULT_FLOOR_SAMPLES_PER_TEXEL: u32 = 256; + +/// One waveform texel: per-channel min/max (stereo; mono duplicates left→right). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Texel { + pub l_min: f32, + pub l_max: f32, + pub r_min: f32, + pub r_max: f32, +} + +impl Texel { + const EMPTY: Texel = Texel { + l_min: f32::INFINITY, + l_max: f32::NEG_INFINITY, + r_min: f32::INFINITY, + r_max: f32::NEG_INFINITY, + }; + + #[inline] + fn include_sample(&mut self, l: f32, r: f32) { + self.l_min = self.l_min.min(l); + self.l_max = self.l_max.max(l); + self.r_min = self.r_min.min(r); + self.r_max = self.r_max.max(r); + } + + #[inline] + fn include_texel(&mut self, c: &Texel) { + self.l_min = self.l_min.min(c.l_min); + self.l_max = self.l_max.max(c.l_max); + self.r_min = self.r_min.min(c.r_min); + self.r_max = self.r_max.max(c.r_max); + } +} + +/// A built min/max LOD pyramid, **root-first**: `levels[0]` is the coarsest +/// (whole-file envelope), `levels.last()` is the finest persisted (floor). +#[derive(Clone, Debug)] +pub struct WaveformPyramid { + pub floor_samples_per_texel: u32, + pub branch: u32, + pub channels: u32, + pub total_frames: u64, + pub levels: Vec>, +} + +impl WaveformPyramid { + /// Coarsest level — a single texel (whole-file envelope), or empty if no + /// samples were pushed. + pub fn root(&self) -> &[Texel] { + self.levels.first().map_or(&[][..], |v| v) + } + + /// Finest persisted level (`floor_samples_per_texel` frames per texel). + pub fn floor(&self) -> &[Texel] { + self.levels.last().map_or(&[][..], |v| v) + } + + /// Number of levels (tree depth + 1). + pub fn depth(&self) -> usize { + self.levels.len() + } + + /// Serialize to a compact binary blob (for persisting in the `.beam` + /// container). Header carries `B`/branch/channels/total_frames + per-level + /// lengths, then root-first texel data (`f32` min/max). + pub fn to_bytes(&self) -> Vec { + let total_texels: usize = self.levels.iter().map(|l| l.len()).sum(); + let mut out = Vec::with_capacity(32 + self.levels.len() * 4 + total_texels * 16); + out.extend_from_slice(b"LBWF"); + out.extend_from_slice(&1u32.to_le_bytes()); // format version + out.extend_from_slice(&self.floor_samples_per_texel.to_le_bytes()); + out.extend_from_slice(&self.branch.to_le_bytes()); + out.extend_from_slice(&self.channels.to_le_bytes()); + out.extend_from_slice(&self.total_frames.to_le_bytes()); + out.extend_from_slice(&(self.levels.len() as u32).to_le_bytes()); + for level in &self.levels { + out.extend_from_slice(&(level.len() as u32).to_le_bytes()); + } + for level in &self.levels { + for t in level { + out.extend_from_slice(&t.l_min.to_le_bytes()); + out.extend_from_slice(&t.l_max.to_le_bytes()); + out.extend_from_slice(&t.r_min.to_le_bytes()); + out.extend_from_slice(&t.r_max.to_le_bytes()); + } + } + out + } + + /// Reconstruct from [`WaveformPyramid::to_bytes`]. + pub fn from_bytes(data: &[u8]) -> Result { + let mut r = ByteReader::new(data); + if r.take(4)? != b"LBWF" { + return Err("Not a waveform pyramid blob".to_string()); + } + let version = r.u32()?; + if version != 1 { + return Err(format!("Unsupported waveform pyramid version {}", version)); + } + let floor_samples_per_texel = r.u32()?; + let branch = r.u32()?; + let channels = r.u32()?; + let total_frames = r.u64()?; + let num_levels = r.u32()? as usize; + let mut level_lens = Vec::with_capacity(num_levels); + for _ in 0..num_levels { + level_lens.push(r.u32()? as usize); + } + let mut levels = Vec::with_capacity(num_levels); + for &len in &level_lens { + let mut level = Vec::with_capacity(len); + for _ in 0..len { + level.push(Texel { + l_min: r.f32()?, + l_max: r.f32()?, + r_min: r.f32()?, + r_max: r.f32()?, + }); + } + levels.push(level); + } + Ok(WaveformPyramid { + floor_samples_per_texel, + branch, + channels, + total_frames, + levels, + }) + } +} + +/// Minimal little-endian byte cursor for [`WaveformPyramid::from_bytes`]. +struct ByteReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> ByteReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + fn take(&mut self, n: usize) -> Result<&'a [u8], String> { + let end = self.pos.checked_add(n).ok_or("overflow")?; + if end > self.data.len() { + return Err("Truncated waveform pyramid blob".to_string()); + } + let s = &self.data[self.pos..end]; + self.pos = end; + Ok(s) + } + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap())) + } + fn u64(&mut self) -> Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().unwrap())) + } + fn f32(&mut self) -> Result { + Ok(f32::from_le_bytes(self.take(4)?.try_into().unwrap())) + } +} + +/// Streaming builder for a [`WaveformPyramid`]. See the module docs. +pub struct WaveformPyramidBuilder { + floor: u32, + branch: u32, + channels: u32, + total_frames: u64, + floor_level: Vec, + acc: Texel, + acc_count: u32, +} + +impl WaveformPyramidBuilder { + pub fn new(channels: u32, floor_samples_per_texel: u32) -> Self { + Self { + floor: floor_samples_per_texel.max(1), + branch: BRANCH, + channels: channels.max(1), + total_frames: 0, + floor_level: Vec::new(), + acc: Texel::EMPTY, + acc_count: 0, + } + } + + /// Pre-reserve the floor `Vec` from an estimated total frame count (e.g. the + /// probe's `total_frames`), to avoid reallocations during streaming. Purely a + /// hint — the final size is set by the actual number of frames pushed. + pub fn reserve_for_frames(&mut self, estimated_frames: u64) { + let est_texels = (estimated_frames / self.floor as u64).saturating_add(1); + self.floor_level.reserve(est_texels.min(usize::MAX as u64) as usize); + } + + /// Push a block of interleaved samples (`channels` per frame). Partial + /// trailing frames (fewer than `channels`) are ignored. + pub fn push_interleaved(&mut self, samples: &[f32]) { + let ch = self.channels as usize; + for frame in samples.chunks_exact(ch) { + let l = frame[0]; + let r = if ch >= 2 { frame[1] } else { l }; + self.push_frame(l, r); + } + } + + #[inline] + fn push_frame(&mut self, l: f32, r: f32) { + self.total_frames += 1; + self.acc.include_sample(l, r); + self.acc_count += 1; + if self.acc_count >= self.floor { + self.floor_level.push(std::mem::replace(&mut self.acc, Texel::EMPTY)); + self.acc_count = 0; + } + } + + /// Flush the trailing partial bucket and reduce up to the root. + pub fn finish(mut self) -> WaveformPyramid { + if self.acc_count > 0 { + self.floor_level.push(self.acc); + } + + // Build finest-first by repeated BRANCH:1 reduction until one texel. + // The shape is fully determined by the floor texel count; the last group + // at each level may be ragged (1..BRANCH children) and reduces over what + // it has. + let mut levels = vec![std::mem::take(&mut self.floor_level)]; + let branch = self.branch as usize; + while levels.last().map_or(0, |l| l.len()) > 1 { + let prev = levels.last().unwrap(); + let mut next = Vec::with_capacity(prev.len().div_ceil(branch)); + for chunk in prev.chunks(branch) { + let mut t = Texel::EMPTY; + for c in chunk { + t.include_texel(c); + } + next.push(t); + } + levels.push(next); + } + // Output is root-first (convention B): levels[0] = root, last = floor. + levels.reverse(); + + WaveformPyramid { + floor_samples_per_texel: self.floor, + branch: self.branch, + channels: self.channels, + total_frames: self.total_frames, + levels, + } + } +} + +// Tests live in `daw-backend/tests/waveform_pyramid.rs` (integration tests) so +// they build the lib in normal mode, independent of the crate's pre-existing +// broken `#[cfg(test)]` unit tests (automation.rs). diff --git a/daw-backend/src/command/types.rs b/daw-backend/src/command/types.rs index 5d481cb..cf29ccc 100644 --- a/daw-backend/src/command/types.rs +++ b/daw-backend/src/command/types.rs @@ -430,8 +430,6 @@ pub enum Query { /// Add a MIDI clip instance to a track synchronously (track_id, instance) - returns instance ID /// The clip must already exist in the MidiClipPool AddMidiClipInstanceSync(TrackId, crate::audio::midi::MidiClipInstance), - /// Add an audio file to the pool synchronously (path, data, channels, sample_rate) - returns pool index - AddAudioFileSync(String, Vec, u32, u32), /// Import an audio file synchronously (path) - returns pool index. /// Does the same work as Command::ImportAudio (mmap for PCM, streaming /// setup for compressed) but returns the real pool index in the response. @@ -440,12 +438,20 @@ pub enum Query { /// problem for very large files, switch to async import with event-based /// pool index reconciliation. ImportAudioSync(std::path::PathBuf), + /// Add the audio track of a video file as a streaming pool entry (FFmpeg, + /// decoded on demand — no extraction). Probes the audio track and returns + /// the pool index. Response: `AudioImportedSync`. + AddVideoAudioSync(std::path::PathBuf), /// Get raw audio samples from pool (pool_index) - returns (samples, sample_rate, channels) GetPoolAudioSamples(usize), /// Get a clone of the current project for serialization GetProject, /// Set the project (replaces current project state) SetProject(Box), + /// Install the host's packed-media byte-source factory (for streaming + /// container-packed audio on load). Sent before `SetProject` so bulk + /// activation can open packed sources. + SetBlobSourceFactory(std::sync::Arc), /// Duplicate a MIDI clip in the pool, returning the new clip's ID DuplicateMidiClipSync(MidiClipId), /// Get whether a track's graph is still the auto-generated default @@ -516,10 +522,10 @@ pub enum QueryResponse { AudioExported(Result<(), String>), /// MIDI clip instance added (returns instance ID) MidiClipInstanceAdded(Result), - /// Audio file added to pool (returns pool index) - AudioFileAddedSync(Result), /// Audio file imported to pool (returns pool index) AudioImportedSync(Result), + /// Packed-media byte-source factory installed + BlobSourceFactorySet(Result<(), String>), /// Raw audio samples from pool (samples, sample_rate, channels) PoolAudioSamples(Result<(Vec, u32, u32), String>), /// Project retrieved diff --git a/daw-backend/tests/compressed_source_stream.rs b/daw-backend/tests/compressed_source_stream.rs new file mode 100644 index 0000000..c9ab89d --- /dev/null +++ b/daw-backend/tests/compressed_source_stream.rs @@ -0,0 +1,89 @@ +//! Integration test for `CompressedReader::open_source` — decoding a streaming +//! audio source from an in-memory byte stream (the packed-in-container path) +//! rather than a filesystem path. Proves the `MediaByteSource` adapter feeds +//! Symphonia correctly (probe + decode + seekable byte length). + +use std::io::{Cursor, Read, Seek, SeekFrom}; + +use daw_backend::audio::disk_reader::{CompressedReader, MediaByteSource}; + +/// A `MediaByteSource` over an in-memory buffer (stands in for core's BlobReader). +struct VecSource(Cursor>); + +impl Read for VecSource { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.read(buf) + } +} +impl Seek for VecSource { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.0.seek(pos) + } +} +impl MediaByteSource for VecSource { + fn byte_len(&self) -> u64 { + self.0.get_ref().len() as u64 + } +} + +/// Build a minimal PCM16 mono WAV byte buffer holding `frames` samples of a ramp. +fn make_wav(sample_rate: u32, frames: u32) -> Vec { + let channels: u16 = 1; + let bits: u16 = 16; + let block_align: u16 = channels * bits / 8; + let byte_rate: u32 = sample_rate * block_align as u32; + let data_len: u32 = frames * block_align as u32; + + let mut v = Vec::new(); + v.extend_from_slice(b"RIFF"); + v.extend_from_slice(&(36 + data_len).to_le_bytes()); + v.extend_from_slice(b"WAVE"); + v.extend_from_slice(b"fmt "); + v.extend_from_slice(&16u32.to_le_bytes()); + v.extend_from_slice(&1u16.to_le_bytes()); // PCM + v.extend_from_slice(&channels.to_le_bytes()); + v.extend_from_slice(&sample_rate.to_le_bytes()); + v.extend_from_slice(&byte_rate.to_le_bytes()); + v.extend_from_slice(&block_align.to_le_bytes()); + v.extend_from_slice(&bits.to_le_bytes()); + v.extend_from_slice(b"data"); + v.extend_from_slice(&data_len.to_le_bytes()); + for i in 0..frames { + // A ramp from -16000..16000 so values are recognizable. + let s = (((i % 1000) as i32 - 500) * 32) as i16; + v.extend_from_slice(&s.to_le_bytes()); + } + v +} + +#[test] +fn open_source_decodes_in_memory_wav() { + let sample_rate = 8000; + let frames = 4096; + let bytes = make_wav(sample_rate, frames); + + let src = Box::new(VecSource(Cursor::new(bytes))); + let mut reader = CompressedReader::open_source(src, Some("wav")) + .expect("open_source should probe the in-memory WAV"); + + assert_eq!(reader.sample_rate(), sample_rate); + assert_eq!(reader.channels(), 1); + + // Decode the whole stream and count emitted frames. + let mut buf = Vec::new(); + let mut decoded = 0usize; + loop { + let n = reader.decode_next(&mut buf).expect("decode_next"); + if n == 0 { + break; + } + decoded += n; + } + // Should recover (approximately) all frames — codec frame counts can round. + assert!( + (decoded as i64 - frames as i64).abs() < 64, + "decoded {} vs expected {}", + decoded, + frames + ); +} diff --git a/daw-backend/tests/video_audio_stream.rs b/daw-backend/tests/video_audio_stream.rs new file mode 100644 index 0000000..066ae62 --- /dev/null +++ b/daw-backend/tests/video_audio_stream.rs @@ -0,0 +1,253 @@ +//! Integration tests for `VideoAudioReader` (FFmpeg streaming audio source). +//! +//! These build the daw-backend lib in normal mode, so they're independent of +//! the crate's pre-existing broken `#[cfg(test)]` unit tests (automation.rs). +//! They synthesize a mono 32-bit-float WAV whose sample `i` has value `i/n`, so +//! a decoded sample's value identifies its frame index — letting us assert both +//! in-order decoding and **sample-accurate seeking** (the property video audio +//! needs to stay synced with other clips). + +use daw_backend::audio::disk_reader::{ + build_waveform_pyramid, CompressedReader, SourceKind, VideoAudioReader, +}; +use std::io::Write; +use std::path::Path; + +fn write_ramp_wav(path: &Path, n: u32, sample_rate: u32) { + let channels = 1u16; + let bytes_per_sample = 4u32; + let data_size = n * bytes_per_sample; + let mut buf: Vec = Vec::with_capacity(44 + data_size as usize); + buf.extend_from_slice(b"RIFF"); + buf.extend_from_slice(&(36 + data_size).to_le_bytes()); + buf.extend_from_slice(b"WAVE"); + buf.extend_from_slice(b"fmt "); + buf.extend_from_slice(&16u32.to_le_bytes()); + buf.extend_from_slice(&3u16.to_le_bytes()); // IEEE float + buf.extend_from_slice(&channels.to_le_bytes()); + buf.extend_from_slice(&sample_rate.to_le_bytes()); + buf.extend_from_slice(&(sample_rate * channels as u32 * bytes_per_sample).to_le_bytes()); + buf.extend_from_slice(&((channels as u32 * bytes_per_sample) as u16).to_le_bytes()); + buf.extend_from_slice(&32u16.to_le_bytes()); + buf.extend_from_slice(b"data"); + buf.extend_from_slice(&data_size.to_le_bytes()); + for i in 0..n { + buf.extend_from_slice(&((i as f32) / (n as f32)).to_le_bytes()); + } + let mut f = std::fs::File::create(path).unwrap(); + f.write_all(&buf).unwrap(); +} + +/// Stereo ramp: frame `i` has left = `i/n`, right = `0.5 - i/n` (distinct per +/// channel), interleaved `[L0,R0,L1,R1,…]`. Exercises the channels>1 path. +fn write_stereo_ramp_wav(path: &Path, n: u32, sample_rate: u32) { + let channels = 2u16; + let bytes_per_sample = 4u32; + let data_size = n * channels as u32 * bytes_per_sample; + let mut buf: Vec = Vec::with_capacity(44 + data_size as usize); + buf.extend_from_slice(b"RIFF"); + buf.extend_from_slice(&(36 + data_size).to_le_bytes()); + buf.extend_from_slice(b"WAVE"); + buf.extend_from_slice(b"fmt "); + buf.extend_from_slice(&16u32.to_le_bytes()); + buf.extend_from_slice(&3u16.to_le_bytes()); // IEEE float + buf.extend_from_slice(&channels.to_le_bytes()); + buf.extend_from_slice(&sample_rate.to_le_bytes()); + buf.extend_from_slice(&(sample_rate * channels as u32 * bytes_per_sample).to_le_bytes()); + buf.extend_from_slice(&((channels as u32 * bytes_per_sample) as u16).to_le_bytes()); + buf.extend_from_slice(&32u16.to_le_bytes()); + buf.extend_from_slice(b"data"); + buf.extend_from_slice(&data_size.to_le_bytes()); + for i in 0..n { + let l = i as f32 / n as f32; + let r = 0.5 - i as f32 / n as f32; + buf.extend_from_slice(&l.to_le_bytes()); + buf.extend_from_slice(&r.to_le_bytes()); + } + let mut f = std::fs::File::create(path).unwrap(); + f.write_all(&buf).unwrap(); +} + +fn temp_path(tag: &str) -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!("lb_videoaudio_test_{}_{}.wav", std::process::id(), tag)); + p +} + +#[test] +fn decodes_samples_in_order() { + let n = 4000u32; + let sr = 8000u32; + let path = temp_path("seq"); + write_ramp_wav(&path, n, sr); + + let mut reader = VideoAudioReader::open(&path).unwrap(); + assert_eq!(reader.channels(), 1); + assert_eq!(reader.sample_rate(), sr); + // Probe estimate (used by add_video_audio_sync) should be ~n frames. + let tf = reader.total_frames() as f64; + assert!( + (tf - n as f64).abs() < n as f64 * 0.1, + "total_frames {} not ~{}", + tf, n + ); + + let mut all = Vec::new(); + let mut buf = Vec::new(); + loop { + let frames = reader.decode_next(&mut buf).unwrap(); + if frames == 0 { + break; + } + all.extend_from_slice(&buf); + } + + // Allow a couple of priming/flush samples of slack at the very end. + assert!(all.len() + 4 >= n as usize, "decoded too few samples: {}", all.len()); + for (i, &v) in all.iter().enumerate().take(n as usize) { + let expected = i as f32 / n as f32; + assert!((v - expected).abs() < 1e-3, "sample {} = {}, expected {}", i, v, expected); + } + let _ = std::fs::remove_file(&path); +} + +/// CompressedReader (symphonia) must seek **sample-accurately** too, so compressed +/// audio stays frame-synced with video audio. Symphonia decodes WAV via the same +/// path; its coarse seek lands on packet boundaries, exercising the decode-discard. +#[test] +fn compressed_reader_seek_is_sample_accurate() { + let n = 4000u32; + let sr = 8000u32; + let path = temp_path("comp_seek"); + write_ramp_wav(&path, n, sr); + + let mut reader = CompressedReader::open(&path).unwrap(); + assert_eq!(reader.channels(), 1); + assert_eq!(reader.sample_rate(), sr); + + for &target in &[2000u64, 137, 3500, 0] { + let actual = reader.seek(target).unwrap(); + assert_eq!(actual, target, "seek should report the exact target"); + + let mut buf = Vec::new(); + let mut frames = 0; + for _ in 0..128 { + frames = reader.decode_next(&mut buf).unwrap(); + if frames > 0 { + break; + } + } + assert!(frames > 0, "no samples after seek to {}", target); + let expected = target as f32 / n as f32; + assert!( + (buf[0] - expected).abs() < 1e-3, + "compressed seek to {}: first sample = {}, expected {}", + target, buf[0], expected + ); + } + let _ = std::fs::remove_file(&path); +} + +/// The decode→pyramid bridge should produce an envelope matching the signal, +/// through both reader backends (symphonia + ffmpeg), with bounded memory. +#[test] +fn waveform_pyramid_from_decode_matches_signal() { + let n = 5000u32; + let sr = 8000u32; + let path = temp_path("pyr"); + write_ramp_wav(&path, n, sr); // ramp 0 .. (n-1)/n, all positive + + for kind in [SourceKind::CompressedAudio, SourceKind::VideoAudio] { + let p = build_waveform_pyramid(&path, kind, 256).unwrap(); + assert_eq!(p.channels, 1); + assert_eq!(p.root().len(), 1, "{:?}: root should be one texel", kind); + let root = p.root()[0]; + assert!(root.l_min.abs() < 1e-2, "{:?}: root min {} ~ 0", kind, root.l_min); + let expected_max = (n - 1) as f32 / n as f32; + assert!( + (root.l_max - expected_max).abs() < 1e-2, + "{:?}: root max {} ~ {}", kind, root.l_max, expected_max + ); + // Frame count is approximate across decoders (priming/resampler overhead); + // the envelope above is the real check. Just confirm it's about right. + assert!((p.total_frames as i64 - n as i64).abs() < 128, "{:?}: frames {}", kind, p.total_frames); + } + let _ = std::fs::remove_file(&path); +} + +#[test] +fn decodes_stereo_interleaved() { + let n = 2000u32; + let sr = 8000u32; + let path = temp_path("stereo"); + write_stereo_ramp_wav(&path, n, sr); + + let mut reader = VideoAudioReader::open(&path).unwrap(); + assert_eq!(reader.channels(), 2); + + let mut all = Vec::new(); + let mut buf = Vec::new(); + loop { + let frames = reader.decode_next(&mut buf).unwrap(); + if frames == 0 { + break; + } + // Each decode_next returns whole interleaved frames. + assert_eq!(buf.len() % 2, 0, "stereo decode returned a partial frame"); + all.extend_from_slice(&buf); + } + + // Interleaved L/R, ~n frames. + assert!(all.len() + 8 >= (n * 2) as usize, "decoded too few samples: {}", all.len()); + for i in 0..n as usize { + let l = all[2 * i]; + let r = all[2 * i + 1]; + assert!((l - i as f32 / n as f32).abs() < 1e-3, "L[{}]={} expected {}", i, l, i as f32 / n as f32); + assert!( + (r - (0.5 - i as f32 / n as f32)).abs() < 1e-3, + "R[{}]={} expected {}", i, r, 0.5 - i as f32 / n as f32 + ); + } + let _ = std::fs::remove_file(&path); +} + +#[test] +fn seek_is_sample_accurate() { + let n = 4000u32; + let sr = 8000u32; + let path = temp_path("seek"); + write_ramp_wav(&path, n, sr); + + let mut reader = VideoAudioReader::open(&path).unwrap(); + + for &target in &[2000u64, 137, 3500, 0] { + let actual = reader.seek(target).unwrap(); + assert_eq!(actual, target); + + // Pull the first non-empty decode after the seek. + let mut buf = Vec::new(); + let mut frames = 0; + for _ in 0..64 { + frames = reader.decode_next(&mut buf).unwrap(); + if frames > 0 { + break; + } + } + assert!(frames > 0, "no samples after seek to {}", target); + + let expected = target as f32 / n as f32; + assert!( + (buf[0] - expected).abs() < 1e-3, + "after seek to {}: first sample = {}, expected {}", + target, + buf[0], + expected + ); + // And the next few advance in order. + for k in 0..frames.min(8) { + let exp = (target as usize + k) as f32 / n as f32; + assert!((buf[k] - exp).abs() < 1e-3, "seek {}+{}: {} vs {}", target, k, buf[k], exp); + } + } + let _ = std::fs::remove_file(&path); +} diff --git a/daw-backend/tests/waveform_pyramid.rs b/daw-backend/tests/waveform_pyramid.rs new file mode 100644 index 0000000..fe67ef9 --- /dev/null +++ b/daw-backend/tests/waveform_pyramid.rs @@ -0,0 +1,135 @@ +//! Integration tests for the streaming waveform LOD pyramid builder. +//! +//! Convention B: `levels[0]` is the root (coarsest), `levels.last()` the floor +//! (finest). Tests use the `.root()` / `.floor()` accessors so they don't depend +//! on the raw index ordering. + +use daw_backend::audio::waveform_pyramid::{Texel, WaveformPyramid, WaveformPyramidBuilder}; + +fn build_mono(samples: &[f32], floor: u32) -> WaveformPyramid { + let mut b = WaveformPyramidBuilder::new(1, floor); + b.push_interleaved(samples); + b.finish() +} + +#[test] +fn floor_level_min_max_per_bucket() { + // 8 samples, floor 4 → two floor texels covering [0..4) and [4..8). + let s: Vec = (0..8).map(|i| i as f32).collect(); + let p = build_mono(&s, 4); + assert_eq!(p.floor().len(), 2); + assert_eq!(p.floor()[0], Texel { l_min: 0.0, l_max: 3.0, r_min: 0.0, r_max: 3.0 }); + assert_eq!(p.floor()[1], Texel { l_min: 4.0, l_max: 7.0, r_min: 4.0, r_max: 7.0 }); + // Root reduces the two floor texels into the envelope [0..8). + assert_eq!(p.root().len(), 1); + assert_eq!(p.root()[0], Texel { l_min: 0.0, l_max: 7.0, r_min: 0.0, r_max: 7.0 }); +} + +#[test] +fn partial_trailing_bucket_is_flushed() { + // 6 samples, floor 4 → texels [0..4) and a ragged [4..6). + let s: Vec = (0..6).map(|i| i as f32).collect(); + let p = build_mono(&s, 4); + assert_eq!(p.floor().len(), 2); + assert_eq!(p.floor()[1], Texel { l_min: 4.0, l_max: 5.0, r_min: 4.0, r_max: 5.0 }); + assert_eq!(p.total_frames, 6); +} + +#[test] +fn multi_level_envelope_matches_global_min_max() { + let s: Vec = (0..1000).map(|i| ((i as f32) * 0.01).sin()).collect(); + let g_min = s.iter().cloned().fold(f32::INFINITY, f32::min); + let g_max = s.iter().cloned().fold(f32::NEG_INFINITY, f32::max); + let p = build_mono(&s, 16); + assert_eq!(p.root().len(), 1); + assert!((p.root()[0].l_min - g_min).abs() < 1e-6); + assert!((p.root()[0].l_max - g_max).abs() < 1e-6); + // Every level's overall min/max equals the global (extremes are lossless). + for level in &p.levels { + let lmin = level.iter().map(|t| t.l_min).fold(f32::INFINITY, f32::min); + let lmax = level.iter().map(|t| t.l_max).fold(f32::NEG_INFINITY, f32::max); + assert!((lmin - g_min).abs() < 1e-6); + assert!((lmax - g_max).abs() < 1e-6); + } +} + +#[test] +fn levels_are_root_first_and_get_finer() { + let s: Vec = (0..1000).map(|i| i as f32).collect(); + let p = build_mono(&s, 16); + // Root first, floor last; strictly finer (more texels) as depth increases. + assert_eq!(p.root().len(), 1); + assert!(p.depth() >= 3); + for w in p.levels.windows(2) { + assert!(w[1].len() >= w[0].len(), "deeper level should be finer"); + } + // Floor has ceil(1000/16) = 63 texels. + assert_eq!(p.floor().len(), 63); +} + +#[test] +fn stereo_channels_tracked_separately() { + // L ramps up, R ramps down; interleaved. + let n = 64; + let mut s = Vec::new(); + for i in 0..n { + s.push(i as f32); // L + s.push(-(i as f32)); // R + } + let mut b = WaveformPyramidBuilder::new(2, 16); + b.push_interleaved(&s); + let p = b.finish(); + assert_eq!(p.root().len(), 1); + assert_eq!(p.root()[0].l_min, 0.0); + assert_eq!(p.root()[0].l_max, (n - 1) as f32); + assert_eq!(p.root()[0].r_min, -((n - 1) as f32)); + assert_eq!(p.root()[0].r_max, 0.0); +} + +#[test] +fn pyramid_size_is_bounded() { + let n = 100_000usize; + let s: Vec = (0..n).map(|i| (i % 7) as f32).collect(); + let floor = 256u32; + let p = build_mono(&s, floor); + let total: usize = p.levels.iter().map(|l| l.len()).sum(); + let floor_texels = (n as u32).div_ceil(floor) as usize; + // Geometric bound: < floor_texels * branch/(branch-1) + small per-level slack. + let bound = floor_texels * 4 / 3 + p.depth() + 2; + assert!(total <= bound, "pyramid too big: {} > {}", total, bound); +} + +#[test] +fn bytes_round_trip() { + let s: Vec = (0..3333).map(|i| ((i as f32) * 0.013).sin()).collect(); + let p = build_mono(&s, 64); + let bytes = p.to_bytes(); + let q = WaveformPyramid::from_bytes(&bytes).unwrap(); + assert_eq!(p.floor_samples_per_texel, q.floor_samples_per_texel); + assert_eq!(p.branch, q.branch); + assert_eq!(p.channels, q.channels); + assert_eq!(p.total_frames, q.total_frames); + assert_eq!(p.levels, q.levels); + // Truncated/garbage input is rejected, not panicking. + assert!(WaveformPyramid::from_bytes(&bytes[..bytes.len() - 4]).is_err()); + assert!(WaveformPyramid::from_bytes(b"nope").is_err()); +} + +#[test] +fn pushing_in_arbitrary_chunks_matches() { + // The streaming builder must be agnostic to how samples are chunked. + let s: Vec = (0..5000).map(|i| ((i * 13) % 97) as f32 - 48.0).collect(); + let whole = build_mono(&s, 32); + + let mut b = WaveformPyramidBuilder::new(1, 32); + b.reserve_for_frames(5000); + for chunk in s.chunks(37) { + b.push_interleaved(chunk); + } + let chunked = b.finish(); + + assert_eq!(whole.depth(), chunked.depth()); + for (a, c) in whole.levels.iter().zip(chunked.levels.iter()) { + assert_eq!(a, c); + } +} diff --git a/lightningbeam-ui/Cargo.lock b/lightningbeam-ui/Cargo.lock index f2aa015..4505926 100644 --- a/lightningbeam-ui/Cargo.lock +++ b/lightningbeam-ui/Cargo.lock @@ -2160,6 +2160,18 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.16.2" @@ -2925,6 +2937,9 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.12", +] [[package]] name = "hashbrown" @@ -2946,6 +2961,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heapless" version = "0.8.0" @@ -3415,6 +3439,17 @@ dependencies = [ "redox_syscall 0.5.18", ] +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libxdo" version = "0.6.0" @@ -3455,6 +3490,7 @@ dependencies = [ "objc2-foundation 0.3.2", "pathdiff", "rstar", + "rusqlite", "serde", "serde_json", "tiny-skia", @@ -3469,7 +3505,7 @@ dependencies = [ [[package]] name = "lightningbeam-editor" -version = "1.0.3-alpha" +version = "1.0.4-alpha" dependencies = [ "beamdsp", "bytemuck", @@ -5478,6 +5514,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad8388ea1a9e0ea807e442e8263a699e7edcb320ecbcd21b4fa8ff859acce3ba" +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags 2.10.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/lightningbeam-ui/lightningbeam-core/Cargo.toml b/lightningbeam-ui/lightningbeam-core/Cargo.toml index 7dea28c..99a68c9 100644 --- a/lightningbeam-ui/lightningbeam-core/Cargo.toml +++ b/lightningbeam-ui/lightningbeam-core/Cargo.toml @@ -36,6 +36,9 @@ zip = "0.6" chrono = "0.4" base64 = "0.21" pathdiff = "0.2" +# .beam container: SQLite database file. `bundled` compiles SQLite from source +# (no system dependency); `blob` enables incremental blob I/O for streaming. +rusqlite = { version = "0.31", features = ["bundled", "blob"] } # Audio encoding for embedded files flacenc = "0.4" # For FLAC encoding (lossless) diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs index b952cdb..86fb996 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/add_clip_instance.rs @@ -235,11 +235,22 @@ impl Action for AddClipInstanceAction { } } AudioClipType::Sampled { audio_pool_index } => { + // `trim_*` / `clip.duration` are in SECONDS (audio content time), + // while `timeline_*` and the backend's `duration` are in BEATS. let internal_start = self.clip_instance.trim_start; let internal_end = self.clip_instance.trim_end.unwrap_or(clip.duration); - let effective_duration = self.clip_instance.timeline_duration - .unwrap_or(internal_end - internal_start); let start_time = self.clip_instance.timeline_start; + // `effective_duration` is in BEATS. When `timeline_duration` is set + // it already is; otherwise the clip occupies its natural content + // length, so convert that seconds-span to beats at the clip's start + // (NOT `internal_end - internal_start`, which is seconds — that was + // the seconds-as-beats bug that made clips stop early off 60 BPM). + let effective_duration = self.clip_instance.timeline_duration.unwrap_or_else(|| { + let tempo_map = document.tempo_map(); + let content_secs = internal_end - internal_start; + tempo_map.inverse_transform(tempo_map.transform(start_time) + content_secs) + - start_time + }); let instance_id = controller.add_audio_clip( *backend_track_id, diff --git a/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs new file mode 100644 index 0000000..7f27258 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs @@ -0,0 +1,778 @@ +//! SQLite-backed `.beam` project container. +//! +//! The `.beam` format is a single SQLite database file. It replaces the older +//! ZIP-archive format. SQLite gives us, in one file: +//! +//! - **Streaming reads** — packed media is split into chunk rows and read on +//! demand through [`BlobReader`] (`Read + Seek`), so arbitrary-length audio / +//! video never has to be fully decoded into RAM on load. +//! - **In-place, crash-safe mutation** — raster frame write-back and re-save are +//! transactional `UPDATE`s rather than rewriting a whole archive. +//! - **Single-file UX** — behaves like a file on every platform. +//! +//! ## Media storage +//! +//! Each media item is one row in `media` plus, when *packed*, N rows in +//! `media_chunk`: +//! +//! - **Packed** (`MediaStorage::Packed`) — bytes live in the database, split +//! into [`CHUNK_SIZE`]-byte chunks. Chunking keeps each blob well under +//! SQLite's ~2 GB per-blob ceiling and bounds the working set of a streaming +//! reader to a single chunk. +//! - **Referenced** (`MediaStorage::Referenced`) — only an external path is +//! stored; the bytes stay on disk (useful for shared media on a network drive, +//! or media too large/volatile to pack). Callers open the path directly. +//! +//! `project.json` (the serialized `BeamProject`) is stored verbatim in the +//! single-row `project_json` table; only the container and media storage change +//! relative to the legacy format. + +use rusqlite::blob::Blob; +use rusqlite::{Connection, DatabaseName, OpenFlags, OptionalExtension}; +use std::io::{self, Read, Seek, SeekFrom}; +use std::path::Path; +use uuid::Uuid; + +/// Default packed-media chunk size: 4 MiB. +/// +/// Small enough to bound a streaming reader's per-chunk work and any +/// whole-chunk buffering, large enough to keep row counts modest (a 1 GB file +/// is 256 rows). Comfortably under SQLite's ~2 GB per-blob limit. +pub const CHUNK_SIZE: u64 = 4 * 1024 * 1024; + +/// Files at or above this size prompt the user to pick packed vs referenced +/// (and the choice is then persisted as the default). Matches SQLite's +/// practical large-blob threshold. +pub const LARGE_MEDIA_THRESHOLD: u64 = 2 * 1024 * 1024 * 1024; + +/// Kind of media stored in the `media` table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MediaKind { + Audio = 0, + Video = 1, + Raster = 2, + ImageAsset = 3, + /// A precomputed waveform LOD pyramid blob for an audio item (keyed by the + /// same id as the audio it describes). See `daw_backend::audio::waveform_pyramid`. + Waveform = 4, +} + +impl MediaKind { + fn from_i64(v: i64) -> Option { + match v { + 0 => Some(Self::Audio), + 1 => Some(Self::Video), + 2 => Some(Self::Raster), + 3 => Some(Self::ImageAsset), + 4 => Some(Self::Waveform), + _ => None, + } + } +} + +/// How a media item's bytes are stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MediaStorage { + /// Bytes are chunked into `media_chunk` rows inside the database. + Packed = 0, + /// Only an external path is stored; bytes live on disk. + Referenced = 1, +} + +impl MediaStorage { + fn from_i64(v: i64) -> Option { + match v { + 0 => Some(Self::Packed), + 1 => Some(Self::Referenced), + _ => None, + } + } +} + +/// Metadata row for a media item (no bytes). +#[derive(Debug, Clone)] +pub struct MediaInfo { + pub id: Uuid, + pub kind: MediaKind, + /// Original codec / container extension, e.g. `"flac"`, `"mp3"`, `"png"`. + pub codec: String, + pub storage: MediaStorage, + /// Set when `storage == Referenced`. + pub ext_path: Option, + /// Total byte length of the media payload (packed only; 0 for referenced). + pub total_len: u64, + // Kind-specific metadata (nullable; meaning depends on `kind`). + pub channels: Option, + pub sample_rate: Option, + pub width: Option, + pub height: Option, +} + +/// Optional kind-specific metadata supplied when writing a media item. +#[derive(Debug, Clone, Copy, Default)] +pub struct MediaMeta { + pub channels: Option, + pub sample_rate: Option, + pub width: Option, + pub height: Option, +} + +/// A `.beam` project container backed by a SQLite database. +pub struct BeamArchive { + conn: Connection, + chunk_size: u64, +} + +impl BeamArchive { + /// Schema version stored in `meta` under `"schema_version"`. + pub const SCHEMA_VERSION: i64 = 1; + + /// Create a new (empty) archive at `path`, replacing any existing file. + pub fn create(path: &Path) -> Result { + // Remove any existing file so we start from a clean schema. + if path.exists() { + std::fs::remove_file(path).map_err(|e| format!("Failed to replace {:?}: {}", path, e))?; + } + let conn = Connection::open(path).map_err(map_sql)?; + let mut archive = Self { conn, chunk_size: CHUNK_SIZE }; + archive.init_schema()?; + Ok(archive) + } + + /// Open an existing archive for read/write. + pub fn open(path: &Path) -> Result { + let conn = Connection::open(path).map_err(map_sql)?; + let archive = Self { conn, chunk_size: CHUNK_SIZE }; + archive.verify_schema()?; + Ok(archive) + } + + /// Quick check: does `path` look like a SQLite database (vs. a legacy ZIP)? + /// Reads the 16-byte SQLite header magic. Used to route between the SQLite + /// loader and the legacy-ZIP migration path. + pub fn is_sqlite(path: &Path) -> bool { + use std::io::Read as _; + let mut f = match std::fs::File::open(path) { + Ok(f) => f, + Err(_) => return false, + }; + let mut magic = [0u8; 16]; + if f.read_exact(&mut magic).is_err() { + return false; + } + &magic == b"SQLite format 3\0" + } + + fn init_schema(&mut self) -> Result<(), String> { + self.conn + .execute_batch( + "BEGIN; + CREATE TABLE media ( + id BLOB PRIMARY KEY, -- 16-byte Uuid + kind INTEGER NOT NULL, + codec TEXT NOT NULL, + storage INTEGER NOT NULL, + ext_path TEXT, + total_len INTEGER NOT NULL DEFAULT 0, + channels INTEGER, + sample_rate INTEGER, + width INTEGER, + height INTEGER + ); + CREATE TABLE media_chunk ( + id INTEGER PRIMARY KEY, -- rowid, for blob_open + media_id BLOB NOT NULL, + chunk_index INTEGER NOT NULL, + bytes BLOB NOT NULL, + UNIQUE (media_id, chunk_index) + ); + CREATE TABLE project_json ( + id INTEGER PRIMARY KEY CHECK (id = 0), + data TEXT NOT NULL + ); + CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + COMMIT;", + ) + .map_err(map_sql)?; + self.set_meta("schema_version", &Self::SCHEMA_VERSION.to_string())?; + Ok(()) + } + + fn verify_schema(&self) -> Result<(), String> { + let v: Option = self.get_meta("schema_version")?; + match v.as_deref().and_then(|s| s.parse::().ok()) { + Some(n) if n <= Self::SCHEMA_VERSION => Ok(()), + Some(n) => Err(format!( + "Unsupported .beam schema version {} (this build supports up to {})", + n, + Self::SCHEMA_VERSION + )), + None => Err("Not a valid .beam archive (missing schema_version)".to_string()), + } + } + + /// Begin a write transaction grouping multiple media/json writes into one + /// atomic, crash-safe commit. Used by saves so unchanged (large) media is + /// never rewritten — only dirty rows are touched, in place. + pub fn transaction(&mut self) -> Result, String> { + let tx = self.conn.transaction().map_err(map_sql)?; + Ok(BeamTxn { tx, chunk_size: self.chunk_size }) + } + + // -- meta key/value -------------------------------------------------- + + pub fn set_meta(&self, key: &str, value: &str) -> Result<(), String> { + set_meta_conn(&self.conn, key, value) + } + + pub fn get_meta(&self, key: &str) -> Result, String> { + self.conn + .query_row("SELECT value FROM meta WHERE key = ?1", [key], |r| r.get(0)) + .optional() + .map_err(map_sql) + } + + // -- project.json ---------------------------------------------------- + + /// Store the serialized `project.json` (single row). + pub fn set_project_json(&self, json: &str) -> Result<(), String> { + set_project_json_conn(&self.conn, json) + } + + /// Read the serialized `project.json`. + pub fn get_project_json(&self) -> Result { + self.conn + .query_row("SELECT data FROM project_json WHERE id = 0", [], |r| r.get(0)) + .optional() + .map_err(map_sql)? + .ok_or_else(|| "Archive has no project.json".to_string()) + } + + // -- media write ----------------------------------------------------- + + /// Write a media item whose bytes are packed (chunked) into the database. + /// Replaces any existing rows for `id`. + pub fn put_media_packed( + &mut self, + id: Uuid, + kind: MediaKind, + codec: &str, + bytes: &[u8], + meta: MediaMeta, + ) -> Result<(), String> { + let tx = self.conn.transaction().map_err(map_sql)?; + write_media_packed(&tx, self.chunk_size, id, kind, codec, bytes, meta)?; + tx.commit().map_err(map_sql)?; + Ok(()) + } + + /// Write a media item that references an external file by path (no bytes + /// stored). Replaces any existing rows for `id`. + pub fn put_media_referenced( + &mut self, + id: Uuid, + kind: MediaKind, + codec: &str, + ext_path: &str, + meta: MediaMeta, + ) -> Result<(), String> { + let tx = self.conn.transaction().map_err(map_sql)?; + write_media_referenced(&tx, id, kind, codec, ext_path, meta)?; + tx.commit().map_err(map_sql)?; + Ok(()) + } + + // -- media read ------------------------------------------------------ + + /// Look up a media item's metadata. + pub fn media_info(&self, id: Uuid) -> Result, String> { + let id_bytes = id.as_bytes().to_vec(); + self.conn + .query_row( + "SELECT kind, codec, storage, ext_path, total_len, channels, sample_rate, width, height + FROM media WHERE id = ?1", + [&id_bytes], + |r| { + Ok(( + r.get::<_, i64>(0)?, + r.get::<_, String>(1)?, + r.get::<_, i64>(2)?, + r.get::<_, Option>(3)?, + r.get::<_, i64>(4)?, + r.get::<_, Option>(5)?, + r.get::<_, Option>(6)?, + r.get::<_, Option>(7)?, + r.get::<_, Option>(8)?, + )) + }, + ) + .optional() + .map_err(map_sql)? + .map(|(kind, codec, storage, ext_path, total_len, channels, sample_rate, width, height)| { + Ok(MediaInfo { + id, + kind: MediaKind::from_i64(kind) + .ok_or_else(|| format!("Unknown media kind {}", kind))?, + codec, + storage: MediaStorage::from_i64(storage) + .ok_or_else(|| format!("Unknown media storage {}", storage))?, + ext_path, + total_len: total_len.max(0) as u64, + channels, + sample_rate, + width, + height, + }) + }) + .transpose() + } + + /// List every media item of a given kind. + pub fn media_ids_of_kind(&self, kind: MediaKind) -> Result, String> { + let mut stmt = self + .conn + .prepare("SELECT id FROM media WHERE kind = ?1") + .map_err(map_sql)?; + let rows = stmt + .query_map([kind as i64], |r| r.get::<_, Vec>(0)) + .map_err(map_sql)?; + let mut out = Vec::new(); + for row in rows { + let bytes = row.map_err(map_sql)?; + out.push(uuid_from_bytes(&bytes)?); + } + Ok(out) + } + + /// Read an entire packed media item into memory. Convenience for small + /// media (raster frames, image assets); large media should stream via + /// [`BeamArchive::open_blob_reader`] instead. + pub fn read_media_full(&self, id: Uuid) -> Result, String> { + let info = self + .media_info(id)? + .ok_or_else(|| format!("Media {} not found", id))?; + if info.storage != MediaStorage::Packed { + return Err(format!("Media {} is referenced, not packed", id)); + } + let id_bytes = id.as_bytes().to_vec(); + let mut stmt = self + .conn + .prepare("SELECT bytes FROM media_chunk WHERE media_id = ?1 ORDER BY chunk_index") + .map_err(map_sql)?; + let rows = stmt + .query_map([&id_bytes], |r| r.get::<_, Vec>(0)) + .map_err(map_sql)?; + let mut out = Vec::with_capacity(info.total_len as usize); + for row in rows { + out.extend_from_slice(&row.map_err(map_sql)?); + } + Ok(out) + } + + /// Open a streaming reader over a packed media item. The reader owns its own + /// SQLite connection (read-only) so it can live on a separate thread (e.g. + /// the audio disk reader) independent of this archive handle. + pub fn open_blob_reader(&self, db_path: &Path, id: Uuid) -> Result { + let info = self + .media_info(id)? + .ok_or_else(|| format!("Media {} not found", id))?; + if info.storage != MediaStorage::Packed { + return Err(format!("Media {} is referenced, not packed", id)); + } + BlobReader::open(db_path, id, info.total_len, self.chunk_size) + } + + /// Override the chunk size (testing / tuning). Affects subsequent writes. + #[doc(hidden)] + pub fn set_chunk_size(&mut self, chunk_size: u64) { + assert!(chunk_size > 0); + self.chunk_size = chunk_size; + } +} + +/// Streaming reader (`Read + Seek`) over a packed media item's chunk rows. +/// +/// Owns a dedicated read-only SQLite connection so it is independent of the +/// writing [`BeamArchive`] handle and can be moved to another thread. Each +/// `read` opens a blob handle on the current chunk's row via `blob_open` (no +/// per-read query — chunk rowids are resolved once up front) and reads up to the +/// chunk boundary; callers that issue many tiny reads should wrap this in a +/// `BufReader`. +pub struct BlobReader { + conn: Connection, + chunk_rowids: Vec, + chunk_size: u64, + total_len: u64, + pos: u64, +} + +impl BlobReader { + fn open(db_path: &Path, id: Uuid, total_len: u64, chunk_size: u64) -> Result { + let conn = Connection::open_with_flags( + db_path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(map_sql)?; + let id_bytes = id.as_bytes().to_vec(); + let mut stmt = conn + .prepare("SELECT id FROM media_chunk WHERE media_id = ?1 ORDER BY chunk_index") + .map_err(map_sql)?; + let rows = stmt + .query_map([&id_bytes], |r| r.get::<_, i64>(0)) + .map_err(map_sql)?; + let mut chunk_rowids = Vec::new(); + for row in rows { + chunk_rowids.push(row.map_err(map_sql)?); + } + drop(stmt); + Ok(Self { conn, chunk_rowids, chunk_size, total_len, pos: 0 }) + } + + /// Total length of the media payload in bytes. + pub fn len(&self) -> u64 { + self.total_len + } + + pub fn is_empty(&self) -> bool { + self.total_len == 0 + } + + fn chunk_blob(&self, chunk_index: usize) -> io::Result> { + let rowid = *self + .chunk_rowids + .get(chunk_index) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "chunk index out of range"))?; + self.conn + .blob_open(DatabaseName::Main, "media_chunk", "bytes", rowid, true) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + } +} + +impl Read for BlobReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.pos >= self.total_len || buf.is_empty() { + return Ok(0); + } + let chunk_index = (self.pos / self.chunk_size) as usize; + let off_in_chunk = self.pos % self.chunk_size; + + // The chunk's length is derivable from total_len/chunk_size, so we don't + // depend on Blob::len(): every chunk but the last is exactly chunk_size. + let chunk_start = chunk_index as u64 * self.chunk_size; + let chunk_len = (self.total_len - chunk_start).min(self.chunk_size); + let avail_in_chunk = (chunk_len - off_in_chunk) as usize; + let avail_total = (self.total_len - self.pos) as usize; + let want = buf.len().min(avail_in_chunk).min(avail_total); + + // Scope the blob borrow (it borrows `self.conn`) so it ends before we + // mutate `self.pos`. + let n = { + let mut blob = self.chunk_blob(chunk_index)?; + blob.seek(SeekFrom::Start(off_in_chunk))?; + blob.read(&mut buf[..want])? + }; + self.pos += n as u64; + Ok(n) + } +} + +impl Seek for BlobReader { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let new_pos = match pos { + SeekFrom::Start(n) => n as i64, + SeekFrom::End(n) => self.total_len as i64 + n, + SeekFrom::Current(n) => self.pos as i64 + n, + }; + if new_pos < 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "seek before start of media", + )); + } + // Allow seeking to/past end (reads then return 0), matching File semantics. + self.pos = new_pos as u64; + Ok(self.pos) + } +} + +/// A write transaction over a [`BeamArchive`]. All writes are buffered until +/// [`BeamTxn::commit`]; dropping without committing rolls back. Lets a save +/// touch only the rows that changed, in place, without rewriting unchanged media. +pub struct BeamTxn<'a> { + tx: rusqlite::Transaction<'a>, + chunk_size: u64, +} + +impl BeamTxn<'_> { + pub fn put_media_packed( + &self, + id: Uuid, + kind: MediaKind, + codec: &str, + bytes: &[u8], + meta: MediaMeta, + ) -> Result<(), String> { + write_media_packed(&self.tx, self.chunk_size, id, kind, codec, bytes, meta) + } + + pub fn put_media_referenced( + &self, + id: Uuid, + kind: MediaKind, + codec: &str, + ext_path: &str, + meta: MediaMeta, + ) -> Result<(), String> { + write_media_referenced(&self.tx, id, kind, codec, ext_path, meta) + } + + /// Like [`BeamTxn::put_media_packed`] but streams the bytes from a file on + /// disk chunk-by-chunk, so an arbitrarily large file is never fully loaded + /// into memory. `total_len` is taken from the bytes actually read. + pub fn put_media_packed_from_path( + &self, + id: Uuid, + kind: MediaKind, + codec: &str, + path: &Path, + meta: MediaMeta, + ) -> Result<(), String> { + write_media_packed_from_path(&self.tx, self.chunk_size, id, kind, codec, path, meta) + } + + pub fn set_project_json(&self, json: &str) -> Result<(), String> { + set_project_json_conn(&self.tx, json) + } + + pub fn set_meta(&self, key: &str, value: &str) -> Result<(), String> { + set_meta_conn(&self.tx, key, value) + } + + /// Does a media row with this id already exist? + pub fn media_exists(&self, id: Uuid) -> Result { + let id_bytes = id.as_bytes().to_vec(); + let n: i64 = self + .tx + .query_row("SELECT COUNT(*) FROM media WHERE id = ?1", [&id_bytes], |r| r.get(0)) + .map_err(map_sql)?; + Ok(n > 0) + } + + /// Every media id currently in the archive. + pub fn all_media_ids(&self) -> Result, String> { + let mut stmt = self.tx.prepare("SELECT id FROM media").map_err(map_sql)?; + let rows = stmt.query_map([], |r| r.get::<_, Vec>(0)).map_err(map_sql)?; + let mut out = Vec::new(); + for row in rows { + out.push(uuid_from_bytes(&row.map_err(map_sql)?)?); + } + Ok(out) + } + + /// Delete a media row (and its chunks). + pub fn delete_media(&self, id: Uuid) -> Result<(), String> { + let id_bytes = id.as_bytes().to_vec(); + self.tx + .execute("DELETE FROM media_chunk WHERE media_id = ?1", [&id_bytes]) + .map_err(map_sql)?; + self.tx + .execute("DELETE FROM media WHERE id = ?1", [&id_bytes]) + .map_err(map_sql)?; + Ok(()) + } + + /// Delete every media row whose id is not in `keep` (orphan cleanup). + pub fn retain_media(&self, keep: &std::collections::HashSet) -> Result { + let mut removed = 0; + for id in self.all_media_ids()? { + if !keep.contains(&id) { + self.delete_media(id)?; + removed += 1; + } + } + Ok(removed) + } + + pub fn commit(self) -> Result<(), String> { + self.tx.commit().map_err(map_sql) + } +} + +// -- shared write helpers (used by both BeamArchive and BeamTxn) -------------- + +fn write_media_packed( + conn: &Connection, + chunk_size: u64, + id: Uuid, + kind: MediaKind, + codec: &str, + bytes: &[u8], + meta: MediaMeta, +) -> Result<(), String> { + let id_bytes = id.as_bytes().to_vec(); + conn.execute("DELETE FROM media WHERE id = ?1", [&id_bytes]).map_err(map_sql)?; + conn.execute("DELETE FROM media_chunk WHERE media_id = ?1", [&id_bytes]) + .map_err(map_sql)?; + conn.execute( + "INSERT INTO media + (id, kind, codec, storage, ext_path, total_len, channels, sample_rate, width, height) + VALUES (?1, ?2, ?3, ?4, NULL, ?5, ?6, ?7, ?8, ?9)", + rusqlite::params![ + id_bytes, + kind as i64, + codec, + MediaStorage::Packed as i64, + bytes.len() as i64, + meta.channels, + meta.sample_rate, + meta.width, + meta.height, + ], + ) + .map_err(map_sql)?; + for (chunk_index, chunk) in bytes.chunks(chunk_size as usize).enumerate() { + conn.execute( + "INSERT INTO media_chunk (media_id, chunk_index, bytes) VALUES (?1, ?2, ?3)", + rusqlite::params![id_bytes, chunk_index as i64, chunk], + ) + .map_err(map_sql)?; + } + Ok(()) +} + +fn write_media_packed_from_path( + conn: &Connection, + chunk_size: u64, + id: Uuid, + kind: MediaKind, + codec: &str, + path: &Path, + meta: MediaMeta, +) -> Result<(), String> { + let id_bytes = id.as_bytes().to_vec(); + conn.execute("DELETE FROM media WHERE id = ?1", [&id_bytes]).map_err(map_sql)?; + conn.execute("DELETE FROM media_chunk WHERE media_id = ?1", [&id_bytes]) + .map_err(map_sql)?; + + let file = std::fs::File::open(path).map_err(|e| format!("Failed to open {:?}: {}", path, e))?; + let mut reader = std::io::BufReader::new(file); + let mut buf = vec![0u8; chunk_size as usize]; + let mut chunk_index: i64 = 0; + let mut total_len: u64 = 0; + + loop { + // Fill `buf` up to chunk_size, tolerating short reads. + let mut filled = 0usize; + while filled < buf.len() { + let n = reader + .read(&mut buf[filled..]) + .map_err(|e| format!("Failed to read {:?}: {}", path, e))?; + if n == 0 { + break; + } + filled += n; + } + if filled == 0 { + break; + } + conn.execute( + "INSERT INTO media_chunk (media_id, chunk_index, bytes) VALUES (?1, ?2, ?3)", + rusqlite::params![id_bytes, chunk_index, &buf[..filled]], + ) + .map_err(map_sql)?; + chunk_index += 1; + total_len += filled as u64; + if filled < buf.len() { + break; // reached EOF + } + } + + conn.execute( + "INSERT INTO media + (id, kind, codec, storage, ext_path, total_len, channels, sample_rate, width, height) + VALUES (?1, ?2, ?3, ?4, NULL, ?5, ?6, ?7, ?8, ?9)", + rusqlite::params![ + id_bytes, + kind as i64, + codec, + MediaStorage::Packed as i64, + total_len as i64, + meta.channels, + meta.sample_rate, + meta.width, + meta.height, + ], + ) + .map_err(map_sql)?; + Ok(()) +} + +fn write_media_referenced( + conn: &Connection, + id: Uuid, + kind: MediaKind, + codec: &str, + ext_path: &str, + meta: MediaMeta, +) -> Result<(), String> { + let id_bytes = id.as_bytes().to_vec(); + conn.execute("DELETE FROM media WHERE id = ?1", [&id_bytes]).map_err(map_sql)?; + conn.execute("DELETE FROM media_chunk WHERE media_id = ?1", [&id_bytes]) + .map_err(map_sql)?; + conn.execute( + "INSERT INTO media + (id, kind, codec, storage, ext_path, total_len, channels, sample_rate, width, height) + VALUES (?1, ?2, ?3, ?4, ?5, 0, ?6, ?7, ?8, ?9)", + rusqlite::params![ + id_bytes, + kind as i64, + codec, + MediaStorage::Referenced as i64, + ext_path, + meta.channels, + meta.sample_rate, + meta.width, + meta.height, + ], + ) + .map_err(map_sql)?; + Ok(()) +} + +fn set_project_json_conn(conn: &Connection, json: &str) -> Result<(), String> { + conn.execute( + "INSERT INTO project_json (id, data) VALUES (0, ?1) + ON CONFLICT(id) DO UPDATE SET data = excluded.data", + [json], + ) + .map_err(map_sql)?; + Ok(()) +} + +fn set_meta_conn(conn: &Connection, key: &str, value: &str) -> Result<(), String> { + conn.execute( + "INSERT INTO meta (key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + rusqlite::params![key, value], + ) + .map_err(map_sql)?; + Ok(()) +} + +fn map_sql(e: rusqlite::Error) -> String { + format!("SQLite error: {}", e) +} + +fn uuid_from_bytes(bytes: &[u8]) -> Result { + let arr: [u8; 16] = bytes + .try_into() + .map_err(|_| format!("Invalid uuid blob length {}", bytes.len()))?; + Ok(Uuid::from_bytes(arr)) +} + +// Tests live in `tests/beam_archive.rs` (integration tests), so they compile the +// library in non-test mode and don't depend on the crate's other `#[cfg(test)]` +// modules. diff --git a/lightningbeam-ui/lightningbeam-core/src/file_io.rs b/lightningbeam-ui/lightningbeam-core/src/file_io.rs index fe5fb74..71fce50 100644 --- a/lightningbeam-ui/lightningbeam-core/src/file_io.rs +++ b/lightningbeam-ui/lightningbeam-core/src/file_io.rs @@ -1,20 +1,25 @@ //! File I/O for .beam project files //! -//! This module handles saving and loading Lightningbeam projects in the .beam format, -//! which is a ZIP archive containing: -//! - project.json (compressed) - Project metadata and structure -//! - media/ directory (uncompressed) - Embedded media files (FLAC for audio) +//! The `.beam` format is a single **SQLite database** (see [`crate::beam_archive`]): +//! - `project_json` table — serialized project metadata and structure +//! - `media` / `media_chunk` tables — audio and raster media (packed as chunked +//! blobs, or referenced by external path) +//! +//! Older `.beam` files are ZIP archives; [`load_beam`] detects and reads those +//! too (via [`load_beam_zip_legacy`]). Saving always writes the SQLite form, so +//! opening a legacy file and saving migrates it. +use crate::beam_archive::{BeamArchive, MediaKind, MediaMeta, LARGE_MEDIA_THRESHOLD}; use crate::document::Document; use daw_backend::audio::pool::AudioPoolEntry; use daw_backend::audio::project::Project as AudioProject; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use std::fs::File; -use std::io::{Read, Write}; +use std::io::Read; use std::path::{Path, PathBuf}; -use zip::write::FileOptions; -use zip::{CompressionMethod, ZipArchive, ZipWriter}; -use flacenc::error::Verify; +use uuid::Uuid; +use zip::ZipArchive; use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; /// File format version @@ -51,9 +56,7 @@ pub struct SerializedAudioBackend { /// Audio project (tracks, MIDI clips, etc.) pub project: AudioProject, - /// Audio pool entries (metadata and paths for audio files) - /// Note: embedded_data field from daw-backend is ignored; embedded files - /// are stored as FLAC in the ZIP's media/audio/ directory instead + /// Audio pool entries (metadata and media references for audio files) pub audio_pool_entries: Vec, /// Mapping from UI layer UUIDs to backend TrackIds @@ -63,6 +66,25 @@ pub struct SerializedAudioBackend { } +/// How to store a media file at or above [`LARGE_MEDIA_THRESHOLD`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LargeMediaMode { + /// Not yet decided — prompt the user the first time a large file is imported. + /// Treated as [`LargeMediaMode::Reference`] at save time. Resetting the + /// preference to `Ask` re-triggers the prompt (useful for testing). + Ask, + /// Pack the bytes into the `.beam` container (chunked, streamed from disk). + Pack, + /// Keep the file external and store only a path reference. + Reference, +} + +impl Default for LargeMediaMode { + fn default() -> Self { + LargeMediaMode::Ask + } +} + /// Settings for saving a project #[derive(Debug, Clone)] pub struct SaveSettings { @@ -74,6 +96,10 @@ pub struct SaveSettings { /// Force linking all media files (don't embed any) pub force_link_all: bool, + + /// How to store files at/above [`LARGE_MEDIA_THRESHOLD`] (pack vs reference). + /// `Ask` behaves as `Reference` here (safe default: don't bloat the DB). + pub large_media_mode: LargeMediaMode, } impl Default for SaveSettings { @@ -82,6 +108,7 @@ impl Default for SaveSettings { auto_embed_threshold_bytes: 10_000_000, // 10 MB force_embed_all: false, force_link_all: false, + large_media_mode: LargeMediaMode::Ask, } } } @@ -125,23 +152,93 @@ pub enum MediaFileType { Image, } -/// Save a project to a .beam file +/// Save a project to a `.beam` file (SQLite container). /// -/// This function: -/// 1. Prepares audio project for save (saves AudioGraph presets) -/// 2. Serializes project data to JSON -/// 3. Creates ZIP archive with compressed project.json -/// 4. Embeds media files as FLAC (for audio) in media/ directory +/// Re-saving an existing SQLite `.beam` updates it **in place** inside a single +/// transaction: unchanged (large) media is never rewritten, only changed rows +/// are touched, and the commit is atomic/crash-safe. A brand-new file or a +/// legacy-ZIP migration is written to a temp file and atomically renamed (there +/// is no large existing container to copy in that case). /// -/// # Arguments -/// * `path` - Path to save the .beam file -/// * `document` - UI document state -/// * `audio_project` - Audio backend project -/// * `audio_pool_entries` - Serialized audio pool entries -/// * `settings` - Save settings (embedding preferences) -/// -/// # Returns -/// Ok(()) on success, or error message +/// Audio and raster media become rows in the `media` table — packed as chunked +/// blobs, or referenced by external path for files at/above +/// [`LARGE_MEDIA_THRESHOLD`]. `project.json` goes in the `project_json` table. +/// Whether a stored media codec is an audio format the disk reader (Symphonia) +/// can stream directly from a packed blob. Video-container audio tracks and any +/// unknown formats fall back to the legacy reconstitution-and-decode path. +fn is_streamable_audio_codec(codec: &str) -> bool { + matches!( + codec.to_lowercase().as_str(), + "mp3" | "flac" | "ogg" | "oga" | "wav" | "wave" | "aiff" | "aif" + | "aac" | "m4a" | "opus" | "alac" | "caf" + ) +} + +/// A `Sync` wrapper over core's `BlobReader` so it satisfies Symphonia's +/// `MediaSource: Send + Sync`. `BlobReader` holds a rusqlite `Connection` +/// (`Send` but `!Sync`); the disk reader uses it single-threaded, so the +/// hot Read/Seek path goes through `Mutex::get_mut` (no runtime locking). +struct SyncBlobReader { + inner: std::sync::Mutex, + len: u64, +} + +impl std::io::Read for SyncBlobReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.inner.get_mut().unwrap().read(buf) + } +} +impl std::io::Seek for SyncBlobReader { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.inner.get_mut().unwrap().seek(pos) + } +} +impl daw_backend::audio::MediaByteSource for SyncBlobReader { + fn byte_len(&self) -> u64 { + self.len + } +} + +/// The host's packed-media byte-source factory: opens an independent streaming +/// reader over a `.beam` container's packed audio by media id. Installed into the +/// engine on load so container-packed audio streams without a full decode. +#[derive(Debug)] +struct BeamBlobFactory { + db_path: PathBuf, +} + +impl daw_backend::audio::AudioBlobSourceFactory for BeamBlobFactory { + fn open( + &self, + media_id: &str, + ) -> Result, String> { + let id = Uuid::parse_str(media_id).map_err(|e| format!("bad media id {}: {}", media_id, e))?; + let archive = BeamArchive::open(&self.db_path)?; + let reader = archive.open_blob_reader(&self.db_path, id)?; + let len = reader.len(); + Ok(Box::new(SyncBlobReader { inner: std::sync::Mutex::new(reader), len })) + } +} + +/// Build a packed-media byte-source factory for a `.beam` file, to install into +/// the engine (`EngineController::set_blob_source_factory`) before loading so its +/// packed audio can be streamed. +pub fn blob_source_factory( + beam_path: &Path, +) -> std::sync::Arc { + std::sync::Arc::new(BeamBlobFactory { db_path: beam_path.to_path_buf() }) +} + +/// Deterministic id for the waveform-pyramid media row of audio pool entry +/// `pool_index`, within a single project container. Stable across saves (so an +/// in-place re-save reuses the row instead of orphaning/rewriting it) and +/// independent of how the audio bytes are stored. The top 32 bits are a fixed +/// "LBWF" sentinel so it can't collide with the random v4 ids used elsewhere. +fn waveform_media_id(pool_index: usize) -> Uuid { + const SENTINEL: u128 = 0x4C42_5746u128 << 96; // "LBWF" in the high 32 bits + Uuid::from_u128(SENTINEL | (pool_index as u128)) +} + pub fn save_beam( path: &Path, document: &Document, @@ -151,256 +248,166 @@ pub fn save_beam( _settings: &SaveSettings, ) -> Result<(), String> { let fn_start = std::time::Instant::now(); - eprintln!("📊 [SAVE_BEAM] Starting save_beam()..."); + eprintln!("📊 [SAVE_BEAM] Starting save_beam() (SQLite container)..."); - // 1. Create backup if file exists and open it for reading old audio files - let step1_start = std::time::Instant::now(); - let mut old_zip = if path.exists() { - let backup_path = path.with_extension("beam.backup"); - std::fs::copy(path, &backup_path) - .map_err(|e| format!("Failed to create backup: {}", e))?; + let project_dir = path.parent().unwrap_or_else(|| Path::new(".")); + let in_place = path.exists() && BeamArchive::is_sqlite(path); - // Open the backup as a ZIP archive for reading - match File::open(&backup_path) { - Ok(file) => match ZipArchive::new(file) { - Ok(archive) => { - eprintln!("📊 [SAVE_BEAM] Step 1: Create backup and open for reading took {:.2}ms", step1_start.elapsed().as_secs_f64() * 1000.0); - Some(archive) - } - Err(e) => { - eprintln!("⚠️ [SAVE_BEAM] Failed to open backup as ZIP: {}, will not copy old audio files", e); - eprintln!("📊 [SAVE_BEAM] Step 1: Create backup took {:.2}ms", step1_start.elapsed().as_secs_f64() * 1000.0); - None - } - }, - Err(e) => { - eprintln!("⚠️ [SAVE_BEAM] Failed to open backup: {}, will not copy old audio files", e); - eprintln!("📊 [SAVE_BEAM] Step 1: Create backup took {:.2}ms", step1_start.elapsed().as_secs_f64() * 1000.0); - None - } - } + // In-place for an existing SQLite container (don't rewrite unchanged media); + // temp + atomic rename for new files / legacy-ZIP migration. + let tmp_path = path.with_extension("beam.tmp"); + let mut archive = if in_place { + BeamArchive::open(path)? } else { - eprintln!("📊 [SAVE_BEAM] Step 1: No backup needed (new file)"); - None + BeamArchive::create(&tmp_path)? }; - // 2. Graph presets are already populated by the engine thread (in GetProject handler) - // before cloning. Do NOT call prepare_for_save() here — the cloned project has - // default empty graphs (AudioTrack::clone() doesn't copy the graph), so calling - // prepare_for_save() would overwrite the good presets with empty ones. - let step2_start = std::time::Instant::now(); - eprintln!("📊 [SAVE_BEAM] Step 2: (graph presets already prepared) took {:.2}ms", step2_start.elapsed().as_secs_f64() * 1000.0); + let now = chrono::Utc::now().to_rfc3339(); + let created = if in_place { + archive.get_meta("created").ok().flatten().unwrap_or_else(|| now.clone()) + } else { + now.clone() + }; - // 3. Create ZIP writer - let step3_start = std::time::Instant::now(); - let file = File::create(path) - .map_err(|e| format!("Failed to create file: {}", e))?; - let mut zip = ZipWriter::new(file); - eprintln!("📊 [SAVE_BEAM] Step 3: Create ZIP writer took {:.2}ms", step3_start.elapsed().as_secs_f64() * 1000.0); + let txn = archive.transaction()?; - // 4. Process audio pool entries and write embedded audio files to ZIP - // Priority: old ZIP file > external file > encode PCM as FLAC - let step4_start = std::time::Instant::now(); - let mut modified_entries = Vec::new(); - let mut flac_encode_time = 0.0; - let mut zip_write_time = 0.0; - let project_dir = path.parent().unwrap_or_else(|| Path::new(".")); + // --- audio pool entries -> media rows (packed) or external references --- + let mut modified_entries = Vec::with_capacity(audio_pool_entries.len()); + let mut live_media: HashSet = HashSet::new(); for entry in &audio_pool_entries { - let mut modified_entry = entry.clone(); + let mut e = entry.clone(); + let existing_id = entry.media_id.as_ref().and_then(|s| Uuid::parse_str(s).ok()); - // Try to get audio data from various sources (in priority order) - let audio_source: Option<(Vec, String)> = if let Some(ref rel_path) = entry.relative_path { - // Priority 1: Check if file is in the old ZIP - if rel_path.starts_with("media/audio/") { - if let Some(ref mut old_zip_archive) = old_zip { - match old_zip_archive.by_name(rel_path) { - Ok(mut file) => { - let mut bytes = Vec::new(); - if file.read_to_end(&mut bytes).is_ok() { - let extension = rel_path.split('.').last().unwrap_or("bin").to_string(); - eprintln!("📊 [SAVE_BEAM] Copying from old ZIP: {}", rel_path); - Some((bytes, extension)) - } else { - eprintln!("⚠️ [SAVE_BEAM] Failed to read {} from old ZIP", rel_path); - None - } - } - Err(_) => { - eprintln!("⚠️ [SAVE_BEAM] File {} not found in old ZIP", rel_path); - None - } - } - } else { - None - } + // Already packed in this archive (in-place re-save): leave the bytes + // untouched, just keep the reference. + if let Some(id) = existing_id { + if txn.media_exists(id)? { + live_media.insert(id); + e.media_id = Some(id.to_string()); + e.relative_path = None; + e.embedded_data = None; + modified_entries.push(e); + continue; } - // Priority 2: Check external filesystem - else { - let full_path = project_dir.join(rel_path); - if full_path.exists() { - match std::fs::read(&full_path) { - Ok(bytes) => { - let extension = full_path.extension() - .and_then(|e| e.to_str()) - .unwrap_or("bin") - .to_string(); - eprintln!("📊 [SAVE_BEAM] Using external file: {:?}", full_path); - Some((bytes, extension)) - } - Err(e) => { - eprintln!("⚠️ [SAVE_BEAM] Failed to read {:?}: {}", full_path, e); - None - } - } - } else { - eprintln!("⚠️ [SAVE_BEAM] External file not found: {:?}", full_path); - None - } - } - } else { - None - }; - - if let Some((audio_bytes, extension)) = audio_source { - // We have the original file - copy it directly - let zip_filename = format!("media/audio/{}.{}", entry.pool_index, extension); - - let file_options = FileOptions::default() - .compression_method(CompressionMethod::Stored); - - zip.start_file(&zip_filename, file_options) - .map_err(|e| format!("Failed to create {} in ZIP: {}", zip_filename, e))?; - - let write_start = std::time::Instant::now(); - zip.write_all(&audio_bytes) - .map_err(|e| format!("Failed to write {}: {}", zip_filename, e))?; - zip_write_time += write_start.elapsed().as_secs_f64() * 1000.0; - - // Update entry to point to ZIP file - modified_entry.embedded_data = None; - modified_entry.relative_path = Some(zip_filename); - - } else if let Some(ref embedded_data) = entry.embedded_data { - // Priority 3: No original file - encode PCM as FLAC - eprintln!("📊 [SAVE_BEAM] Encoding PCM to FLAC for pool {} (no original file)", entry.pool_index); - // Embedded data is always PCM - encode as FLAC - let audio_bytes = BASE64_STANDARD.decode(&embedded_data.data_base64) - .map_err(|e| format!("Failed to decode base64 audio data for pool index {}: {}", entry.pool_index, e))?; - - let zip_filename = format!("media/audio/{}.flac", entry.pool_index); - - let file_options = FileOptions::default() - .compression_method(CompressionMethod::Stored); - - zip.start_file(&zip_filename, file_options) - .map_err(|e| format!("Failed to create {} in ZIP: {}", zip_filename, e))?; - - // Encode PCM samples to FLAC - let flac_start = std::time::Instant::now(); - - // The audio_bytes are raw PCM samples (interleaved f32 little-endian) - let samples: Vec = audio_bytes - .chunks_exact(4) - .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) - .collect(); - - // Convert f32 samples to i32 for FLAC encoding - let samples_i32: Vec = samples - .iter() - .map(|&s| { - let clamped = s.clamp(-1.0, 1.0); - (clamped * 8388607.0) as i32 - }) - .collect(); - - // Configure FLAC encoder - let config = flacenc::config::Encoder::default() - .into_verified() - .map_err(|(_, e)| format!("FLAC encoder config error: {:?}", e))?; - - let source = flacenc::source::MemSource::from_samples( - &samples_i32, - entry.channels as usize, - 24, - entry.sample_rate as usize, - ); - - // Encode to FLAC - let flac_stream = flacenc::encode_with_fixed_block_size( - &config, - source, - config.block_size, - ).map_err(|e| format!("FLAC encoding failed: {:?}", e))?; - - // Convert stream to bytes - use flacenc::component::BitRepr; - let mut sink = flacenc::bitsink::ByteSink::new(); - flac_stream.write(&mut sink) - .map_err(|e| format!("Failed to write FLAC stream: {:?}", e))?; - let flac_bytes = sink.as_slice(); - - flac_encode_time += flac_start.elapsed().as_secs_f64() * 1000.0; - - let write_start = std::time::Instant::now(); - zip.write_all(flac_bytes) - .map_err(|e| format!("Failed to write {}: {}", zip_filename, e))?; - zip_write_time += write_start.elapsed().as_secs_f64() * 1000.0; - - // Update entry to point to ZIP file instead of embedding data - modified_entry.embedded_data = None; - modified_entry.relative_path = Some(zip_filename); } - modified_entries.push(modified_entry); - } - eprintln!("📊 [SAVE_BEAM] Step 4: Process audio pool ({} entries) took {:.2}ms", - audio_pool_entries.len(), step4_start.elapsed().as_secs_f64() * 1000.0); - if flac_encode_time > 0.0 { - eprintln!("📊 [SAVE_BEAM] - FLAC encoding: {:.2}ms", flac_encode_time); - } - if zip_write_time > 0.0 { - eprintln!("📊 [SAVE_BEAM] - ZIP writing: {:.2}ms", zip_write_time); + // Otherwise resolve the source: external file (Priority 2, streamed from + // disk so a huge file is never fully loaded), or embedded data (Priority 3). + let meta = MediaMeta { + channels: Some(entry.channels), + sample_rate: Some(entry.sample_rate), + ..Default::default() + }; + let mut wrote_packed: Option = None; + let mut referenced: Option = None; + + if let Some(rel) = entry.relative_path.as_ref() { + let full = if Path::new(rel).is_absolute() { + PathBuf::from(rel) + } else { + project_dir.join(rel) + }; + if full.exists() { + let size = std::fs::metadata(&full).map(|m| m.len()).unwrap_or(0); + let codec = full + .extension() + .and_then(|x| x.to_str()) + .unwrap_or("bin") + .to_lowercase(); + // Large files honor the user's pack-vs-reference choice (`Ask` == + // reference); smaller files are always packed. + let reference_it = size >= LARGE_MEDIA_THRESHOLD + && _settings.large_media_mode != LargeMediaMode::Pack; + if reference_it { + referenced = Some(rel.clone()); + } else { + let id = existing_id.unwrap_or_else(Uuid::new_v4); + txn.put_media_packed_from_path(id, MediaKind::Audio, &codec, &full, meta)?; + wrote_packed = Some(id); + } + } + } + + if wrote_packed.is_none() && referenced.is_none() { + if let Some(ed) = entry.embedded_data.as_ref() { + if let Ok(bytes) = BASE64_STANDARD.decode(&ed.data_base64) { + let id = existing_id.unwrap_or_else(Uuid::new_v4); + txn.put_media_packed(id, MediaKind::Audio, &ed.format.to_lowercase(), &bytes, meta)?; + wrote_packed = Some(id); + } + } + } + + if let Some(id) = wrote_packed { + live_media.insert(id); + e.media_id = Some(id.to_string()); + e.relative_path = None; + e.embedded_data = None; + } else if let Some(rel) = referenced { + e.media_id = None; + e.relative_path = Some(rel); + e.embedded_data = None; + } // else: nothing available — keep original references (reported missing on load) + + // Persist this entry's waveform pyramid (keyed by pool index, independent + // of the audio storage above). Reuse the row in place on re-save. + let wf_id = waveform_media_id(entry.pool_index); + if let Some(blob) = entry.waveform_blob.as_ref() { + txn.put_media_packed(wf_id, MediaKind::Waveform, "lbwf", blob, MediaMeta::default())?; + live_media.insert(wf_id); + } else if txn.media_exists(wf_id)? { + // Unchanged this save — keep the stored waveform row. + live_media.insert(wf_id); + } + + modified_entries.push(e); } - // 4b. Write raster layer PNG buffers to ZIP (media/raster/.png) - let step4b_start = std::time::Instant::now(); - let raster_file_options = FileOptions::default() - .compression_method(CompressionMethod::Stored); // PNG is already compressed + // --- raster keyframes -> media rows (PNG), keyed by keyframe id --- + // (Phase 0 writes all resident frames each save; a disk-dirty flag to skip + // unchanged frames in place is deferred to Phase 3.) let mut raster_count = 0usize; for layer in &document.root.children { if let crate::layer::AnyLayer::Raster(rl) = layer { for kf in &rl.keyframes { if !kf.raw_pixels.is_empty() { - // Encode raw RGBA to PNG for storage - let img = crate::brush_engine::image_from_raw( - kf.raw_pixels.clone(), kf.width, kf.height, - ); + let img = + crate::brush_engine::image_from_raw(kf.raw_pixels.clone(), kf.width, kf.height); match crate::brush_engine::encode_png(&img) { Ok(png_bytes) => { - let zip_path = kf.media_path.clone(); - zip.start_file(&zip_path, raster_file_options) - .map_err(|e| format!("Failed to create {} in ZIP: {}", zip_path, e))?; - zip.write_all(&png_bytes) - .map_err(|e| format!("Failed to write {}: {}", zip_path, e))?; + txn.put_media_packed( + kf.id, + MediaKind::Raster, + "png", + &png_bytes, + MediaMeta { + width: Some(kf.width), + height: Some(kf.height), + ..Default::default() + }, + )?; + live_media.insert(kf.id); raster_count += 1; } - Err(e) => eprintln!("⚠️ [SAVE_BEAM] Failed to encode raster PNG {}: {}", kf.media_path, e), + Err(e) => eprintln!("⚠️ [SAVE_BEAM] Failed to encode raster {}: {}", kf.id, e), } + } else if txn.media_exists(kf.id)? { + // Pixels not resident but already stored — keep the row. + live_media.insert(kf.id); } } } } - eprintln!("📊 [SAVE_BEAM] Step 4b: Write {} raster PNG buffers took {:.2}ms", - raster_count, step4b_start.elapsed().as_secs_f64() * 1000.0); - // 5. Build BeamProject structure with modified entries - let step5_start = std::time::Instant::now(); - let now = chrono::Utc::now().to_rfc3339(); + // --- orphan cleanup: drop media for removed clips/keyframes --- + let removed = txn.retain_media(&live_media)?; + + // --- project.json + meta --- let beam_project = BeamProject { version: BEAM_VERSION.to_string(), - created: now.clone(), - modified: now, + created: created.clone(), + modified: now.clone(), ui_state: document.clone(), audio_backend: SerializedAudioBackend { sample_rate: 48000, // TODO: Get from audio engine @@ -409,52 +416,187 @@ pub fn save_beam( layer_to_track_map: layer_to_track_map.clone(), }, }; - eprintln!("📊 [SAVE_BEAM] Step 5: Build BeamProject structure took {:.2}ms", step5_start.elapsed().as_secs_f64() * 1000.0); - - // 6. Write project.json (compressed with DEFLATE) - let step6_start = std::time::Instant::now(); - let json_options = FileOptions::default() - .compression_method(CompressionMethod::Deflated) - .compression_level(Some(6)); - - zip.start_file("project.json", json_options) - .map_err(|e| format!("Failed to create project.json in ZIP: {}", e))?; - - let json = serde_json::to_string_pretty(&beam_project) + let json = serde_json::to_string(&beam_project) .map_err(|e| format!("JSON serialization failed: {}", e))?; + txn.set_project_json(&json)?; + txn.set_meta("version", BEAM_VERSION)?; + txn.set_meta("created", &created)?; + txn.set_meta("modified", &now)?; + txn.commit()?; - zip.write_all(json.as_bytes()) - .map_err(|e| format!("Failed to write project.json: {}", e))?; - eprintln!("📊 [SAVE_BEAM] Step 6: Write project.json ({} bytes) took {:.2}ms", json.len(), step6_start.elapsed().as_secs_f64() * 1000.0); - - // 7. Finalize ZIP - let step7_start = std::time::Instant::now(); - zip.finish() - .map_err(|e| format!("Failed to finalize ZIP: {}", e))?; - eprintln!("📊 [SAVE_BEAM] Step 7: Finalize ZIP took {:.2}ms", step7_start.elapsed().as_secs_f64() * 1000.0); - - eprintln!("📊 [SAVE_BEAM] ✅ Total save_beam() time: {:.2}ms", fn_start.elapsed().as_secs_f64() * 1000.0); + // Close the connection before renaming (required on Windows; harmless elsewhere). + drop(archive); + if !in_place { + std::fs::rename(&tmp_path, path) + .map_err(|e| format!("Failed to finalize {:?}: {}", path, e))?; + } + eprintln!( + "📊 [SAVE_BEAM] ✅ Saved {} audio + {} raster media, {} orphans removed, in {:.2}ms", + audio_pool_entries.len(), + raster_count, + removed, + fn_start.elapsed().as_secs_f64() * 1000.0 + ); Ok(()) } -/// Load a project from a .beam file +/// Load a project from a `.beam` file. /// -/// This function: -/// 1. Opens ZIP archive and reads project.json -/// 2. Deserializes project data -/// 3. Loads embedded media files from archive -/// 4. Attempts to load external media files -/// 5. Rebuilds AudioGraphs from presets with correct sample_rate -/// -/// # Arguments -/// * `path` - Path to the .beam file -/// -/// # Returns -/// LoadedProject on success (with missing_files list), or error message +/// Detects the container format: SQLite (current) or legacy ZIP, and dispatches +/// accordingly. Both produce an identical [`LoadedProject`]. pub fn load_beam(path: &Path) -> Result { + if BeamArchive::is_sqlite(path) { + load_beam_sqlite(path) + } else { + load_beam_zip_legacy(path) + } +} + +/// Load a project from a SQLite `.beam` container. +/// +/// Phase 0 reconstitutes packed audio into each entry's `embedded_data` so the +/// existing (full-decode) audio pool loader keeps working unchanged; Phase 1b +/// replaces this with streaming reads via `BlobReader`. +fn load_beam_sqlite(path: &Path) -> Result { let fn_start = std::time::Instant::now(); - eprintln!("📊 [LOAD_BEAM] Starting load_beam()..."); + eprintln!("📊 [LOAD_BEAM] Starting load_beam() (SQLite container)..."); + + let archive = BeamArchive::open(path)?; + let json = archive.get_project_json()?; + let beam_project: BeamProject = serde_json::from_str(&json) + .map_err(|e| format!("Failed to deserialize project.json: {}", e))?; + + if beam_project.version != BEAM_VERSION { + return Err(format!( + "Unsupported file version: {} (expected {})", + beam_project.version, BEAM_VERSION + )); + } + + let mut document = beam_project.ui_state; + document.tempo_map_mut().rebuild_seconds(); + let mut audio_project = beam_project.audio_backend.project; + audio_project + .rebuild_audio_graphs(DEFAULT_BUFFER_SIZE) + .map_err(|e| format!("Failed to rebuild audio graphs: {}", e))?; + let layer_to_track_map = beam_project.audio_backend.layer_to_track_map; + + // For each packed audio item: stream it (leave `embedded_data` empty so the + // pool builds a Compressed placeholder backed by the blob factory) when it's a + // recognized audio codec; otherwise fall back to the legacy reconstitution + // (whole bytes → base64 → decode), which still covers video-container audio + // tracks symphonia can't stream and any unknown formats. + let mut restored_entries = Vec::with_capacity(beam_project.audio_backend.audio_pool_entries.len()); + for entry in &beam_project.audio_backend.audio_pool_entries { + let mut e = entry.clone(); + if let Some(id) = entry.media_id.as_ref().and_then(|s| Uuid::parse_str(s).ok()) { + match archive.media_info(id) { + Ok(Some(info)) => { + if is_streamable_audio_codec(&info.codec) { + // Stream: keep media_id, no embedded bytes. The engine opens + // the packed blob via the factory at activation time. + e.embedded_data = None; + e.relative_path = None; + } else { + match archive.read_media_full(id) { + Ok(bytes) => { + e.embedded_data = Some(daw_backend::audio::pool::EmbeddedAudioData { + data_base64: BASE64_STANDARD.encode(&bytes), + format: info.codec, + }); + e.relative_path = None; + } + Err(err) => eprintln!("⚠️ [LOAD_BEAM] Failed to read audio media {}: {}", id, err), + } + } + } + Ok(None) => eprintln!("⚠️ [LOAD_BEAM] Audio media {} missing from archive", id), + Err(err) => eprintln!("⚠️ [LOAD_BEAM] media_info({}) failed: {}", id, err), + } + } + + // Restore this entry's persisted waveform pyramid, if present — avoids + // re-decoding the source media just to redraw the overview. + let wf_id = waveform_media_id(entry.pool_index); + if let Ok(Some(_)) = archive.media_info(wf_id) { + match archive.read_media_full(wf_id) { + Ok(bytes) => e.waveform_blob = Some(bytes), + Err(err) => eprintln!("⚠️ [LOAD_BEAM] Failed to read waveform {}: {}", wf_id, err), + } + } + + restored_entries.push(e); + } + + // Raster keyframes: load PNG bytes from media rows into raw_pixels. + let mut raster_load_count = 0usize; + for layer in document.root.children.iter_mut() { + if let crate::layer::AnyLayer::Raster(rl) = layer { + for kf in &mut rl.keyframes { + if let Ok(Some(_)) = archive.media_info(kf.id) { + match archive.read_media_full(kf.id) { + Ok(png_bytes) => match crate::brush_engine::decode_png(&png_bytes) { + Ok(rgba) => { + kf.raw_pixels = rgba.into_raw(); + raster_load_count += 1; + } + Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to decode raster {}: {}", kf.id, e), + }, + Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to read raster {}: {}", kf.id, e), + } + } + } + } + } + + // Missing external files (referenced entries whose file no longer exists). + let project_dir = path.parent().unwrap_or_else(|| Path::new(".")); + let missing_files: Vec = restored_entries + .iter() + .enumerate() + .filter_map(|(idx, entry)| { + if entry.embedded_data.is_none() && entry.media_id.is_none() { + if let Some(rel) = entry.relative_path.as_ref() { + let full = if Path::new(rel).is_absolute() { + PathBuf::from(rel) + } else { + project_dir.join(rel) + }; + if !full.exists() { + return Some(MissingFileInfo { + pool_index: idx, + original_path: full, + file_type: MediaFileType::Audio, + }); + } + } + } + None + }) + .collect(); + + eprintln!( + "📊 [LOAD_BEAM] ✅ Loaded {} audio entries, {} raster frames in {:.2}ms", + restored_entries.len(), + raster_load_count, + fn_start.elapsed().as_secs_f64() * 1000.0 + ); + + Ok(LoadedProject { + document, + audio_project, + layer_to_track_map, + audio_pool_entries: restored_entries, + missing_files, + }) +} + +/// Load a project from a legacy ZIP `.beam` archive (pre-SQLite format). +/// Retained for backward compatibility; saving converts to SQLite. +fn load_beam_zip_legacy(path: &Path) -> Result { + let fn_start = std::time::Instant::now(); + eprintln!("📊 [LOAD_BEAM] Starting load_beam() (legacy ZIP)..."); // 1. Open ZIP archive let step1_start = std::time::Instant::now(); diff --git a/lightningbeam-ui/lightningbeam-core/src/lib.rs b/lightningbeam-ui/lightningbeam-core/src/lib.rs index 40ce30f..8422eed 100644 --- a/lightningbeam-ui/lightningbeam-core/src/lib.rs +++ b/lightningbeam-ui/lightningbeam-core/src/lib.rs @@ -42,6 +42,7 @@ pub mod segment_builder; pub mod planar_graph; pub mod file_types; pub mod file_io; +pub mod beam_archive; pub mod export; pub mod clipboard; pub(crate) mod clipboard_platform; diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 444bd88..b2ad7cc 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -97,7 +97,7 @@ impl VideoDecoder { // Optionally build keyframe index for fast seeking let keyframe_positions = if build_keyframes { eprintln!("[Video Decoder] Building keyframe index for {}", path); - let positions = Self::build_keyframe_index(&path, stream_index)?; + let positions = Self::scan_keyframes(&path, stream_index)?; eprintln!("[Video Decoder] Found {} keyframes", positions.len()); positions } else { @@ -125,14 +125,19 @@ impl VideoDecoder { }) } - /// Build keyframe index for this decoder - /// This can be called asynchronously after decoder creation - fn build_and_set_keyframe_index(&mut self) -> Result<(), String> { - eprintln!("[Video Decoder] Building keyframe index for {}", self.path); - let positions = Self::build_keyframe_index(&self.path, self.stream_index)?; - eprintln!("[Video Decoder] Found {} keyframes", positions.len()); + /// Source file path this decoder reads from. + pub fn path(&self) -> &str { + &self.path + } + + /// Parameters needed to scan keyframes off-thread (path + video stream index). + pub fn keyframe_scan_params(&self) -> (String, usize) { + (self.path.clone(), self.stream_index) + } + + /// Replace the keyframe index (built off-thread via [`VideoDecoder::scan_keyframes`]). + pub fn set_keyframe_index(&mut self, positions: Vec) { self.keyframe_positions = positions; - Ok(()) } /// Get the output width (scaled dimensions) @@ -150,9 +155,10 @@ impl VideoDecoder { self.get_frame(timestamp) } - /// Build an index of all keyframe positions in the video - /// This enables fast seeking by knowing exactly where keyframes are - fn build_keyframe_index(path: &str, stream_index: usize) -> Result, String> { + /// Build an index of all keyframe positions in the video by scanning packets + /// from a fresh input. Does not touch `self` — call it off-thread (it is slow + /// on long videos) and hand the result to [`VideoDecoder::set_keyframe_index`]. + pub fn scan_keyframes(path: &str, stream_index: usize) -> Result, String> { let mut input = ffmpeg::format::input(path) .map_err(|e| format!("Failed to open video for indexing: {}", e))?; @@ -340,6 +346,51 @@ impl VideoDecoder { } } +/// Generate timeline thumbnails for a video using a **dedicated** decoder that +/// is independent of any shared playback decoder — so thumbnail work never holds +/// a lock the UI/playback needs. +/// +/// Thumbnails are sampled at keyframes ~`interval_secs` apart. Decoding at a +/// keyframe is cheap (≈one frame) versus decoding forward to an arbitrary +/// timestamp (the whole GOP). Frames are decoded directly at `thumb_width` (so +/// `get_thumbnail_at`'s 128-wide assumption holds) and tightly packed RGBA is +/// handed to `on_thumb` as `(timestamp_secs, data)`. +pub fn generate_keyframe_thumbnails( + path: &str, + interval_secs: f64, + thumb_width: u32, + mut on_thumb: impl FnMut(f64, Arc>), +) -> Result<(), String> { + // Own decoder at thumbnail resolution; builds its own keyframe index. The + // large max-height lets width be the constraining dimension, so output width + // is exactly `thumb_width`. + let mut decoder = VideoDecoder::new( + path.to_string(), + 4, + Some(thumb_width), + Some(100_000), + true, // build keyframe index (needed to sample at keyframes) + )?; + + let keyframe_secs: Vec = decoder + .keyframe_positions + .iter() + .map(|&ts| ts as f64 * decoder.time_base) + .collect(); + + let mut last_emitted = f64::NEG_INFINITY; + for ks in keyframe_secs { + if ks - last_emitted < interval_secs { + continue; + } + if let Ok(rgba) = decoder.get_frame(ks) { + last_emitted = ks; + on_thumb(ks, Arc::new(rgba)); + } + } + Ok(()) +} + /// Probe video file for metadata without creating a full decoder pub fn probe_video(path: &str) -> Result { ffmpeg::init().map_err(|e| e.to_string())?; @@ -440,8 +491,9 @@ impl VideoManager { /// `target_width` and `target_height` specify the maximum dimensions /// for decoded frames. Video will be scaled down if larger. /// - /// The keyframe index is NOT built during this call - use `build_keyframe_index_async` - /// in a background thread to build it asynchronously. + /// The keyframe index is NOT built during this call — scan it off-thread via + /// [`VideoDecoder::scan_keyframes`] and store it with + /// [`VideoDecoder::set_keyframe_index`] so the slow scan never blocks playback. pub fn load_video( &mut self, clip_id: Uuid, @@ -467,20 +519,6 @@ impl VideoManager { Ok(metadata) } - /// Build keyframe index for a loaded video asynchronously - /// - /// This should be called from a background thread after load_video() - /// to avoid blocking the UI during import. - pub fn build_keyframe_index(&self, clip_id: &Uuid) -> Result<(), String> { - let decoder_arc = self.decoders.get(clip_id) - .ok_or_else(|| format!("Video clip {} not found", clip_id))?; - - let mut decoder = decoder_arc.lock() - .map_err(|e| format!("Failed to lock decoder: {}", e))?; - - decoder.build_and_set_keyframe_index() - } - /// Get a decoded frame for a specific clip at a specific timestamp /// /// Returns None if the clip is not loaded or decoding fails. @@ -517,62 +555,6 @@ impl VideoManager { Some(frame) } - /// Generate thumbnails for a video clip (single batch version - use generate_thumbnails_progressive instead) - /// - /// Thumbnails are generated every 5 seconds at 128px width. - /// This should be called in a background thread to avoid blocking. - /// Thumbnails are inserted into the cache progressively as they're generated, - /// allowing the UI to display them immediately. - /// - /// DEPRECATED: Use generate_thumbnails_progressive which releases the lock between thumbnails. - pub fn generate_thumbnails(&mut self, clip_id: &Uuid, duration: f64) -> Result<(), String> { - let decoder_arc = self.decoders.get(clip_id) - .ok_or("Clip not loaded")? - .clone(); - - let mut decoder = decoder_arc.lock() - .map_err(|e| format!("Failed to lock decoder: {}", e))?; - - // Initialize thumbnail cache entry with empty vec - self.thumbnail_cache.insert(*clip_id, Vec::new()); - - let interval = 5.0; // Generate thumbnail every 5 seconds - let mut t = 0.0; - - while t < duration { - // Decode frame at this timestamp - if let Ok(rgba_data) = decoder.get_frame(t) { - // Decode already scaled to output dimensions, but we want 128px width for thumbnails - // We need to scale down further - let current_width = decoder.output_width; - let current_height = decoder.output_height; - - // Calculate thumbnail dimensions (128px width, maintain aspect ratio) - let thumb_width = 128u32; - let aspect_ratio = current_height as f32 / current_width as f32; - let thumb_height = (thumb_width as f32 * aspect_ratio) as u32; - - // Simple nearest-neighbor downsampling for thumbnails - let thumb_data = downsample_rgba( - &rgba_data, - current_width, - current_height, - thumb_width, - thumb_height, - ); - - // Insert thumbnail into cache immediately so UI can display it - if let Some(thumbnails) = self.thumbnail_cache.get_mut(clip_id) { - thumbnails.push((t, Arc::new(thumb_data))); - } - } - - t += interval; - } - - Ok(()) - } - /// Get the decoder Arc for a clip (for external thumbnail generation) /// This allows external code to decode frames without holding the VideoManager lock pub fn get_decoder(&self, clip_id: &Uuid) -> Option>> { @@ -650,211 +632,3 @@ impl Default for VideoManager { Self::new() } } - -/// Simple nearest-neighbor downsampling for RGBA images -pub fn downsample_rgba_public( - src: &[u8], - src_width: u32, - src_height: u32, - dst_width: u32, - dst_height: u32, -) -> Vec { - downsample_rgba(src, src_width, src_height, dst_width, dst_height) -} - -/// Simple nearest-neighbor downsampling for RGBA images (internal) -fn downsample_rgba( - src: &[u8], - src_width: u32, - src_height: u32, - dst_width: u32, - dst_height: u32, -) -> Vec { - let mut dst = Vec::with_capacity((dst_width * dst_height * 4) as usize); - - let x_ratio = src_width as f32 / dst_width as f32; - let y_ratio = src_height as f32 / dst_height as f32; - - for y in 0..dst_height { - for x in 0..dst_width { - let src_x = (x as f32 * x_ratio) as u32; - let src_y = (y as f32 * y_ratio) as u32; - - let src_idx = ((src_y * src_width + src_x) * 4) as usize; - - // Copy RGBA bytes - dst.push(src[src_idx]); // R - dst.push(src[src_idx + 1]); // G - dst.push(src[src_idx + 2]); // B - dst.push(src[src_idx + 3]); // A - } - } - - dst -} - -/// Extracted audio data from a video file -#[derive(Debug, Clone)] -pub struct ExtractedAudio { - pub samples: Vec, - pub channels: u32, - pub sample_rate: u32, - pub duration: f64, -} - -/// Extract audio from a video file -/// -/// This function performs the slow FFmpeg decoding without holding any locks. -/// The caller can then quickly add the audio to the DAW backend in a background thread. -/// -/// Returns None if the video has no audio stream. -pub fn extract_audio_from_video(path: &str) -> Result, String> { - ffmpeg::init().map_err(|e| e.to_string())?; - - // Open video file - let mut input = ffmpeg::format::input(path) - .map_err(|e| format!("Failed to open video: {}", e))?; - - // Find audio stream - let audio_stream_opt = input.streams() - .best(ffmpeg::media::Type::Audio); - - // Return None if no audio stream - if audio_stream_opt.is_none() { - return Ok(None); - } - - let audio_stream = audio_stream_opt.unwrap(); - let audio_index = audio_stream.index(); - - // Get audio properties - let context_decoder = ffmpeg::codec::context::Context::from_parameters( - audio_stream.parameters() - ).map_err(|e| e.to_string())?; - - let mut audio_decoder = context_decoder.decoder().audio() - .map_err(|e| e.to_string())?; - - let sample_rate = audio_decoder.rate(); - let channels = audio_decoder.channels() as u32; - - // Decode all audio frames - let mut audio_samples: Vec = Vec::new(); - - for (stream, packet) in input.packets() { - if stream.index() == audio_index { - audio_decoder.send_packet(&packet) - .map_err(|e| e.to_string())?; - - let mut audio_frame = ffmpeg::util::frame::Audio::empty(); - while audio_decoder.receive_frame(&mut audio_frame).is_ok() { - // Convert audio to f32 packed format - let format = audio_frame.format(); - let frame_channels = audio_frame.channels() as usize; - - // Create resampler to convert to f32 packed - let mut resampler = ffmpeg::software::resampling::context::Context::get( - format, - audio_frame.channel_layout(), - sample_rate, - ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), - audio_frame.channel_layout(), - sample_rate, - ).map_err(|e| e.to_string())?; - - let mut resampled_frame = ffmpeg::util::frame::Audio::empty(); - resampler.run(&audio_frame, &mut resampled_frame) - .map_err(|e| e.to_string())?; - - // Extract f32 samples (interleaved format) - let data_ptr = resampled_frame.data(0).as_ptr() as *const f32; - let total_samples = resampled_frame.samples() * frame_channels; - - // Safety checks before creating slice from FFmpeg data - // 1. Verify f32 alignment (required: 4 bytes) - if data_ptr.align_offset(std::mem::align_of::()) != 0 { - return Err("FFmpeg audio data is not properly aligned for f32".to_string()); - } - - // 2. Verify the frame actually has enough data - let byte_size = resampled_frame.data(0).len(); - let expected_bytes = total_samples * std::mem::size_of::(); - if byte_size < expected_bytes { - return Err(format!( - "FFmpeg frame buffer too small: {} bytes, need {} bytes", - byte_size, expected_bytes - )); - } - - // SAFETY: We verified alignment and bounds above. - // The slice lifetime is tied to resampled_frame which lives until - // after extend_from_slice completes. - let samples_slice = unsafe { - std::slice::from_raw_parts(data_ptr, total_samples) - }; - - audio_samples.extend_from_slice(samples_slice); - } - } - } - - // Flush audio decoder - audio_decoder.send_eof().map_err(|e| e.to_string())?; - let mut audio_frame = ffmpeg::util::frame::Audio::empty(); - while audio_decoder.receive_frame(&mut audio_frame).is_ok() { - let format = audio_frame.format(); - let frame_channels = audio_frame.channels() as usize; - - let mut resampler = ffmpeg::software::resampling::context::Context::get( - format, - audio_frame.channel_layout(), - sample_rate, - ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), - audio_frame.channel_layout(), - sample_rate, - ).map_err(|e| e.to_string())?; - - let mut resampled_frame = ffmpeg::util::frame::Audio::empty(); - resampler.run(&audio_frame, &mut resampled_frame) - .map_err(|e| e.to_string())?; - - let data_ptr = resampled_frame.data(0).as_ptr() as *const f32; - let total_samples = resampled_frame.samples() * frame_channels; - - // Safety checks before creating slice from FFmpeg data - // 1. Verify f32 alignment (required: 4 bytes) - if data_ptr.align_offset(std::mem::align_of::()) != 0 { - return Err("FFmpeg audio data is not properly aligned for f32".to_string()); - } - - // 2. Verify the frame actually has enough data - let byte_size = resampled_frame.data(0).len(); - let expected_bytes = total_samples * std::mem::size_of::(); - if byte_size < expected_bytes { - return Err(format!( - "FFmpeg frame buffer too small: {} bytes, need {} bytes", - byte_size, expected_bytes - )); - } - - // SAFETY: We verified alignment and bounds above. - // The slice lifetime is tied to resampled_frame which lives until - // after extend_from_slice completes. - let samples_slice = unsafe { - std::slice::from_raw_parts(data_ptr, total_samples) - }; - - audio_samples.extend_from_slice(samples_slice); - } - - // Calculate duration - let total_samples_per_channel = audio_samples.len() / channels as usize; - let duration = total_samples_per_channel as f64 / sample_rate as f64; - - Ok(Some(ExtractedAudio { - samples: audio_samples, - channels, - sample_rate, - duration, - })) -} diff --git a/lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs b/lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs new file mode 100644 index 0000000..a0d4b04 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs @@ -0,0 +1,207 @@ +//! Integration tests for the SQLite-backed `.beam` container. +//! +//! These are integration tests (not `#[cfg(test)]` unit tests) so they build the +//! library in normal mode and exercise only the public API — independent of any +//! pre-existing breakage in the crate's internal test modules. + +use lightningbeam_core::beam_archive::{BeamArchive, MediaKind, MediaMeta, MediaStorage}; +use std::io::{Read, Seek, SeekFrom}; +use std::sync::atomic::{AtomicU64, Ordering}; +use uuid::Uuid; + +fn temp_db_path(tag: &str) -> std::path::PathBuf { + static N: AtomicU64 = AtomicU64::new(0); + let n = N.fetch_add(1, Ordering::Relaxed); + let mut p = std::env::temp_dir(); + p.push(format!("beam_archive_it_{}_{}_{}.beam", std::process::id(), tag, n)); + let _ = std::fs::remove_file(&p); + p +} + +#[test] +fn project_json_roundtrip() { + let path = temp_db_path("json"); + let archive = BeamArchive::create(&path).unwrap(); + archive.set_project_json("{\"hello\":\"world\"}").unwrap(); + assert_eq!(archive.get_project_json().unwrap(), "{\"hello\":\"world\"}"); + drop(archive); + let archive = BeamArchive::open(&path).unwrap(); + assert_eq!(archive.get_project_json().unwrap(), "{\"hello\":\"world\"}"); + assert!(BeamArchive::is_sqlite(&path)); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn packed_media_roundtrip_full() { + let path = temp_db_path("full"); + let mut archive = BeamArchive::create(&path).unwrap(); + let id = Uuid::from_u128(0x1234); + archive.set_chunk_size(1000); + let data: Vec = (0..3500u32).map(|i| (i % 251) as u8).collect(); + archive + .put_media_packed( + id, + MediaKind::Audio, + "flac", + &data, + MediaMeta { channels: Some(2), sample_rate: Some(44100), ..Default::default() }, + ) + .unwrap(); + + let info = archive.media_info(id).unwrap().unwrap(); + assert_eq!(info.kind, MediaKind::Audio); + assert_eq!(info.codec, "flac"); + assert_eq!(info.storage, MediaStorage::Packed); + assert_eq!(info.total_len, 3500); + assert_eq!(info.channels, Some(2)); + assert_eq!(info.sample_rate, Some(44100)); + + assert_eq!(archive.read_media_full(id).unwrap(), data); + assert_eq!(archive.media_ids_of_kind(MediaKind::Audio).unwrap(), vec![id]); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn blob_reader_streams_and_seeks() { + let path = temp_db_path("stream"); + let mut archive = BeamArchive::create(&path).unwrap(); + archive.set_chunk_size(1000); + let id = Uuid::from_u128(0xBEEF); + let data: Vec = (0..3500u32).map(|i| (i % 251) as u8).collect(); + archive + .put_media_packed(id, MediaKind::Audio, "mp3", &data, MediaMeta::default()) + .unwrap(); + + let mut reader = archive.open_blob_reader(&path, id).unwrap(); + assert_eq!(reader.len(), 3500); + + // Sequential read in odd-sized buffers crosses chunk boundaries. + let mut got = Vec::new(); + let mut buf = [0u8; 333]; + loop { + let n = reader.read(&mut buf).unwrap(); + if n == 0 { + break; + } + got.extend_from_slice(&buf[..n]); + } + assert_eq!(got, data); + + // Seek to a mid-chunk position and read across a boundary. + reader.seek(SeekFrom::Start(990)).unwrap(); + let mut window = [0u8; 20]; + let mut filled = 0; + while filled < window.len() { + let n = reader.read(&mut window[filled..]).unwrap(); + assert!(n > 0); + filled += n; + } + assert_eq!(&window[..], &data[990..1010]); + + // Seek from end and read the tail. + reader.seek(SeekFrom::End(-10)).unwrap(); + let mut tail = Vec::new(); + reader.read_to_end(&mut tail).unwrap(); + assert_eq!(tail, &data[3490..]); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn referenced_media_records_path() { + let path = temp_db_path("ref"); + let mut archive = BeamArchive::create(&path).unwrap(); + let id = Uuid::from_u128(0xCAFE); + archive + .put_media_referenced( + id, + MediaKind::Video, + "mp4", + "/mnt/share/big.mp4", + MediaMeta { width: Some(3840), height: Some(2160), ..Default::default() }, + ) + .unwrap(); + let info = archive.media_info(id).unwrap().unwrap(); + assert_eq!(info.storage, MediaStorage::Referenced); + assert_eq!(info.ext_path.as_deref(), Some("/mnt/share/big.mp4")); + assert_eq!(info.width, Some(3840)); + // Streaming a referenced item is an error (caller opens the path directly). + assert!(archive.open_blob_reader(&path, id).is_err()); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn transaction_groups_writes_and_orphan_cleanup() { + let path = temp_db_path("txn"); + let keep = Uuid::from_u128(1); + let orphan = Uuid::from_u128(2); + + // First save: two media items committed in one transaction. + { + let mut archive = BeamArchive::create(&path).unwrap(); + let txn = archive.transaction().unwrap(); + txn.put_media_packed(keep, MediaKind::Audio, "flac", &vec![9u8; 10], MediaMeta::default()) + .unwrap(); + txn.put_media_packed(orphan, MediaKind::Audio, "mp3", &vec![8u8; 10], MediaMeta::default()) + .unwrap(); + txn.set_project_json("{}").unwrap(); + txn.commit().unwrap(); + } + { + let archive = BeamArchive::open(&path).unwrap(); + assert!(archive.media_info(keep).unwrap().is_some()); + assert!(archive.media_info(orphan).unwrap().is_some()); + } + + // Second save (in place): keep only `keep`; `orphan` should be retained-out. + { + let mut archive = BeamArchive::open(&path).unwrap(); + let txn = archive.transaction().unwrap(); + // `keep` already present → in-place save leaves it untouched. + assert!(txn.media_exists(keep).unwrap()); + let mut live = std::collections::HashSet::new(); + live.insert(keep); + let removed = txn.retain_media(&live).unwrap(); + assert_eq!(removed, 1); + txn.commit().unwrap(); + } + { + let archive = BeamArchive::open(&path).unwrap(); + assert!(archive.media_info(keep).unwrap().is_some()); + assert!(archive.media_info(orphan).unwrap().is_none()); + // `keep`'s bytes survived untouched. + assert_eq!(archive.read_media_full(keep).unwrap(), vec![9u8; 10]); + } + let _ = std::fs::remove_file(&path); +} + +#[test] +fn rolled_back_transaction_writes_nothing() { + let path = temp_db_path("rollback"); + let id = Uuid::from_u128(42); + let mut archive = BeamArchive::create(&path).unwrap(); + { + let txn = archive.transaction().unwrap(); + txn.put_media_packed(id, MediaKind::Audio, "flac", &vec![1u8; 5], MediaMeta::default()) + .unwrap(); + // Drop without commit → rollback. + } + assert!(archive.media_info(id).unwrap().is_none()); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn overwrite_media_replaces_chunks() { + let path = temp_db_path("overwrite"); + let mut archive = BeamArchive::create(&path).unwrap(); + archive.set_chunk_size(100); + let id = Uuid::from_u128(7); + archive + .put_media_packed(id, MediaKind::Raster, "png", &vec![1u8; 250], MediaMeta::default()) + .unwrap(); + // Overwrite with shorter data — stale chunks must be gone. + archive + .put_media_packed(id, MediaKind::Raster, "png", &vec![2u8; 50], MediaMeta::default()) + .unwrap(); + assert_eq!(archive.read_media_full(id).unwrap(), vec![2u8; 50]); + let _ = std::fs::remove_file(&path); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/config.rs b/lightningbeam-ui/lightningbeam-editor/src/config.rs index a1efa3b..24c976e 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/config.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/config.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; use crate::keymap::KeybindingConfig; +use lightningbeam_core::file_io::LargeMediaMode; /// Application configuration (persistent) #[derive(Debug, Clone, Serialize, Deserialize)] @@ -57,6 +58,18 @@ pub struct AppConfig { /// Custom keyboard shortcut overrides (sparse — only non-default bindings stored) #[serde(default)] pub keybindings: KeybindingConfig, + + /// How to store media files at/above the large-media threshold (~2GB). + /// `Ask` (default) prompts the first time such a file is imported, then the + /// chosen mode is persisted here. Reset to `Ask` to be prompted again. + #[serde(default)] + pub large_media_default: LargeMediaMode, + + /// Finest-level resolution of the waveform LOD pyramid: source frames per + /// floor texel (`B`). Smaller = larger on-disk pyramid but zoom-in re-decodes + /// sooner; larger = smaller pyramid, wider re-decode span. Default 256. + #[serde(default = "defaults::waveform_floor_samples_per_texel")] + pub waveform_floor_samples_per_texel: u32, } impl Default for AppConfig { @@ -75,6 +88,8 @@ impl Default for AppConfig { waveform_stereo: defaults::waveform_stereo(), theme_mode: defaults::theme_mode(), keybindings: KeybindingConfig::default(), + large_media_default: LargeMediaMode::default(), + waveform_floor_samples_per_texel: defaults::waveform_floor_samples_per_texel(), } } } @@ -276,4 +291,5 @@ mod defaults { pub fn debug() -> bool { false } pub fn waveform_stereo() -> bool { false } pub fn theme_mode() -> String { "system".to_string() } + pub fn waveform_floor_samples_per_texel() -> u32 { 256 } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index e02bc60..1f56046 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -492,6 +492,11 @@ enum FileCommand { path: std::path::PathBuf, document: lightningbeam_core::document::Document, layer_to_track_map: std::collections::HashMap, + /// How to store large media on this save (from the user's preference). + large_media_mode: lightningbeam_core::file_io::LargeMediaMode, + /// Serialized waveform-pyramid blobs per audio pool index, persisted into + /// the container so a later load needn't re-decode the source media. + waveform_blobs: std::collections::HashMap>, progress_tx: std::sync::mpsc::Sender, }, Load { @@ -566,8 +571,8 @@ impl FileOperationsWorker { fn run(self) { while let Ok(command) = self.command_rx.recv() { match command { - FileCommand::Save { path, document, layer_to_track_map, progress_tx } => { - self.handle_save(path, document, &layer_to_track_map, progress_tx); + FileCommand::Save { path, document, layer_to_track_map, large_media_mode, waveform_blobs, progress_tx } => { + self.handle_save(path, document, &layer_to_track_map, large_media_mode, waveform_blobs, progress_tx); } FileCommand::Load { path, progress_tx } => { self.handle_load(path, progress_tx); @@ -582,6 +587,8 @@ impl FileOperationsWorker { path: std::path::PathBuf, document: lightningbeam_core::document::Document, layer_to_track_map: &std::collections::HashMap, + large_media_mode: lightningbeam_core::file_io::LargeMediaMode, + waveform_blobs: std::collections::HashMap>, progress_tx: std::sync::mpsc::Sender, ) { use lightningbeam_core::file_io::{save_beam, SaveSettings}; @@ -593,7 +600,7 @@ impl FileOperationsWorker { let _ = progress_tx.send(FileProgress::SerializingAudioPool); let step1_start = std::time::Instant::now(); - let audio_pool_entries = { + let mut audio_pool_entries = { let mut controller = self.audio_controller.lock().unwrap(); match controller.serialize_audio_pool(&path) { Ok(entries) => entries, @@ -603,6 +610,13 @@ impl FileOperationsWorker { } } }; + // Attach precomputed waveform pyramids so save_beam can persist them + // (keyed by pool index inside the container). + for entry in &mut audio_pool_entries { + if let Some(blob) = waveform_blobs.get(&entry.pool_index) { + entry.waveform_blob = Some(blob.clone()); + } + } eprintln!("📊 [SAVE] Step 1: Serialize audio pool took {:.2}ms", step1_start.elapsed().as_secs_f64() * 1000.0); // Step 2: Get project @@ -623,7 +637,10 @@ impl FileOperationsWorker { let _ = progress_tx.send(FileProgress::WritingZip); let step3_start = std::time::Instant::now(); - let settings = SaveSettings::default(); + let settings = SaveSettings { + large_media_mode, + ..SaveSettings::default() + }; match save_beam(&path, &document, &mut audio_project, audio_pool_entries, layer_to_track_map, &settings) { Ok(()) => { eprintln!("📊 [SAVE] Step 3: save_beam() took {:.2}ms", step3_start.elapsed().as_secs_f64() * 1000.0); @@ -687,9 +704,6 @@ enum AudioExtractionResult { channels: u32, sample_rate: u32, }, - NoAudio { - video_clip_id: Uuid, - }, Error { video_clip_id: Uuid, error: String, @@ -901,6 +915,21 @@ struct EditorApp { raw_audio_cache: HashMap>, u32, u32)>, /// Pool indices that need GPU texture upload (set when raw audio arrives, cleared after upload) waveform_gpu_dirty: HashSet, + /// Pool indices whose `raw_audio_cache` entry holds a packed min/max floor + /// (4 f32 per texel: Lmin,Lmax,Rmin,Rmax) rather than raw per-sample data. + /// Maps pool_index -> `B` (floor frames-per-texel), so the timeline render can + /// derive the floor's effective rate `sr/B` with full float precision. Populated + /// for streamed video-audio, which has no in-RAM samples to draw per-sample. + waveform_minmax_pools: HashMap, + /// Serialized waveform-pyramid blobs (LBWF bytes) per pool, kept so a save can + /// persist them into the `.beam` container without re-decoding. Populated on + /// generation and on load. See `daw_backend::audio::waveform_pyramid::to_bytes`. + waveform_pyramid_blobs: HashMap>>, + /// Receives generated waveforms from a background thread: + /// (pool_index, packed_floor_texels, source_sample_rate, channels, B, pyramid_blob). + waveform_result_rx: std::sync::mpsc::Receiver<(usize, Vec, u32, u32, u32, Vec)>, + /// Sender handed to background waveform-pyramid generation threads. + waveform_result_tx: std::sync::mpsc::Sender<(usize, Vec, u32, u32, u32, Vec)>, /// Consumer for recording audio mirror (streams recorded samples to UI for live waveform) recording_mirror_rx: Option>, /// Current file path (None if not yet saved) @@ -923,6 +952,10 @@ struct EditorApp { export_dialog: export::dialog::ExportDialog, /// Export progress dialog export_progress_dialog: export::dialog::ExportProgressDialog, + /// When `Some(name)`, show the "large media: pack or reference?" prompt for the + /// named just-imported file. Set on the first large import while the + /// `large_media_default` preference is still `Ask`. + large_media_prompt: Option, /// Preferences dialog preferences_dialog: preferences::dialog::PreferencesDialog, /// Export orchestrator for background exports @@ -1069,6 +1102,8 @@ impl EditorApp { .map(|rs| rs.target_format) .unwrap_or(wgpu::TextureFormat::Rgba8Unorm); + let (waveform_result_tx, waveform_result_rx) = std::sync::mpsc::channel(); + Self { layouts, current_layout_index: 0, @@ -1156,6 +1191,10 @@ impl EditorApp { audio_pools_with_new_waveforms: HashSet::new(), // Track pool indices with new raw audio raw_audio_cache: HashMap::new(), waveform_gpu_dirty: HashSet::new(), + waveform_minmax_pools: HashMap::new(), + waveform_pyramid_blobs: HashMap::new(), + waveform_result_rx, + waveform_result_tx, recording_mirror_rx, current_file_path: None, // No file loaded initially keymap: KeymapManager::new(&config.keybindings), @@ -1166,6 +1205,7 @@ impl EditorApp { audio_extraction_rx, export_dialog: export::dialog::ExportDialog::default(), export_progress_dialog: export::dialog::ExportProgressDialog::default(), + large_media_prompt: None, preferences_dialog: preferences::dialog::PreferencesDialog::default(), export_orchestrator: None, effect_thumbnail_generator: None, // Initialized when GPU available @@ -2882,6 +2922,8 @@ impl EditorApp { self.audio_duration_cache.clear(); self.raw_audio_cache.clear(); self.waveform_gpu_dirty.clear(); + self.waveform_minmax_pools.clear(); + self.waveform_pyramid_blobs.clear(); self.pane_instances.clear(); self.project_generation += 1; self.app_mode = AppMode::StartScreen; @@ -3666,11 +3708,21 @@ impl EditorApp { // Clone document for background thread let document = self.action_executor.document().clone(); + // Snapshot the generated waveform pyramids (by pool index) so the worker + // can persist them into the container alongside the audio. + let waveform_blobs: std::collections::HashMap> = self + .waveform_pyramid_blobs + .iter() + .map(|(&idx, blob)| (idx, blob.as_ref().clone())) + .collect(); + // Send save command to worker thread let command = FileCommand::Save { path: path.clone(), document, layer_to_track_map: self.layer_to_track_map.clone(), + large_media_mode: self.config.large_media_default, + waveform_blobs, progress_tx, }; @@ -3781,6 +3833,34 @@ impl EditorApp { self.restore_layout_from_document(); eprintln!("📊 [APPLY] Step 2: Restore UI layout took {:.2}ms", step2_start.elapsed().as_secs_f64() * 1000.0); + // Snapshot persisted waveform pyramids (with each entry's source sample + // rate) before the backend consumes the entries below. Restored into the + // GPU caches after the controller borrow ends, so a load needn't re-decode. + let loaded_waveforms: Vec<(usize, u32, Vec)> = loaded_project + .audio_pool_entries + .iter() + .filter_map(|e| e.waveform_blob.as_ref().map(|b| (e.pool_index, e.sample_rate, b.clone()))) + .collect(); + + // Streamed (packed) audio entries that have NO persisted pyramid yet + // (e.g. saved before waveform persistence): generate one in the background + // from the packed blob so the overview appears without a full decode. + // (pool_index, media_id, codec/ext, sample_rate) + let streamed_needing_waveform: Vec<(usize, String, Option, u32)> = loaded_project + .audio_pool_entries + .iter() + .filter(|e| e.waveform_blob.is_none() && e.embedded_data.is_none()) + .filter_map(|e| { + e.media_id.clone().map(|id| { + let ext = std::path::Path::new(&e.name) + .extension() + .and_then(|x| x.to_str()) + .map(|s| s.to_lowercase()); + (e.pool_index, id, ext, e.sample_rate) + }) + }) + .collect(); + // Load audio pool FIRST (before setting project, so clips can reference pool entries) let step3_start = std::time::Instant::now(); if let Some(ref controller_arc) = self.audio_controller { @@ -3794,6 +3874,15 @@ impl EditorApp { } eprintln!("📊 [APPLY] Step 3: Load audio pool took {:.2}ms", step3_start.elapsed().as_secs_f64() * 1000.0); + // Install the packed-media byte-source factory for this container before + // SetProject, so bulk activation can stream container-packed audio. + if lightningbeam_core::beam_archive::BeamArchive::is_sqlite(&path) { + let factory = lightningbeam_core::file_io::blob_source_factory(&path); + if let Err(e) = controller.set_blob_source_factory(factory) { + eprintln!("⚠️ [APPLY] Failed to install blob source factory: {}", e); + } + } + // Now set project (clips can now reference the loaded pool entries) let step4_start = std::time::Instant::now(); if let Err(e) = controller.set_project(loaded_project.audio_project) { @@ -3810,6 +3899,86 @@ impl EditorApp { ); } + // Restore waveform overviews from the persisted pyramids (no re-decode). + // Clear first so stale pools from the previous project don't linger; + // on-demand audio repopulates raw_audio_cache lazily at render time. + self.raw_audio_cache.clear(); + self.waveform_gpu_dirty.clear(); + self.waveform_minmax_pools.clear(); + self.waveform_pyramid_blobs.clear(); + for (pool_index, sample_rate, blob) in loaded_waveforms { + match daw_backend::audio::waveform_pyramid::WaveformPyramid::from_bytes(&blob) { + Ok(pyramid) => { + let b = pyramid.floor_samples_per_texel.max(1); + let channels = pyramid.channels; + let floor = pyramid.floor(); + let mut packed = Vec::with_capacity(floor.len() * 4); + for t in floor { + packed.push(t.l_min); + packed.push(t.l_max); + packed.push(t.r_min); + packed.push(t.r_max); + } + self.raw_audio_cache.insert(pool_index, (Arc::new(packed), sample_rate, channels)); + self.waveform_minmax_pools.insert(pool_index, b); + self.waveform_pyramid_blobs.insert(pool_index, Arc::new(blob)); + self.waveform_gpu_dirty.insert(pool_index); + } + Err(e) => eprintln!( + "⚠️ [APPLY] Failed to parse persisted waveform for pool {}: {}", + pool_index, e + ), + } + } + + // Background-generate overviews for streamed audio that has no persisted + // pyramid yet (older saves). Streams the packed blob via the factory and + // sends the floor through the same channel `update()` already drains; the + // next save then persists it. + if !streamed_needing_waveform.is_empty() { + let wf_tx = self.waveform_result_tx.clone(); + let floor_b = self.config.waveform_floor_samples_per_texel.max(1); + let beam_path = path.clone(); + std::thread::spawn(move || { + let factory = lightningbeam_core::file_io::blob_source_factory(&beam_path); + for (pool_index, media_id, ext, sample_rate) in streamed_needing_waveform { + let src = match factory.open(&media_id) { + Ok(s) => s, + Err(e) => { + eprintln!("[APPLY] waveform gen: open packed {} failed: {}", media_id, e); + continue; + } + }; + match daw_backend::audio::disk_reader::build_waveform_pyramid_from_source( + src, + ext.as_deref(), + floor_b, + ) { + Ok(pyramid) => { + let floor = pyramid.floor(); + let mut packed = Vec::with_capacity(floor.len() * 4); + for t in floor { + packed.push(t.l_min); + packed.push(t.l_max); + packed.push(t.r_min); + packed.push(t.r_max); + } + let blob = pyramid.to_bytes(); + let _ = wf_tx.send(( + pool_index, + packed, + sample_rate, + pyramid.channels, + pyramid.floor_samples_per_texel, + blob, + )); + } + Err(e) => eprintln!("[APPLY] waveform gen failed for pool {}: {}", pool_index, e), + } + } + }); + } + // Reset state and restore track mappings let step5_start = std::time::Instant::now(); self.layer_to_track_map.clear(); @@ -3890,6 +4059,13 @@ impl EditorApp { } eprintln!("📊 [APPLY] Step 8: Rebuilt MIDI event cache for {} clips in {:.2}ms", midi_fetched, step8_start.elapsed().as_secs_f64() * 1000.0); + // Re-register video clips with the VideoManager (decoder + keyframe index + + // thumbnails). Restored video_clips carry only metadata + file_path; without + // this they render black with no thumbnails. + let step9_start = std::time::Instant::now(); + self.register_loaded_videos(); + eprintln!("📊 [APPLY] Step 9: Registered loaded videos in {:.2}ms", step9_start.elapsed().as_secs_f64() * 1000.0); + // Reset playback state self.playback_time = 0.0; self.is_playing = false; @@ -3908,9 +4084,178 @@ impl EditorApp { println!("✅ Loaded from: {}", path.display()); } + /// Register every video clip in the loaded document with the `VideoManager` + /// so it can decode and display. Mirrors the import path's setup (decoder + + /// background keyframe index + thumbnails) but does NOT re-extract audio — + /// the extracted audio is already restored via the audio pool. + fn register_loaded_videos(&mut self) { + let doc_width = self.action_executor.document().width as u32; + let doc_height = self.action_executor.document().height as u32; + + // Snapshot (id, path) so we don't hold the document borrow while locking + // the VideoManager / spawning threads. + let videos: Vec<_> = self + .action_executor + .document() + .video_clips + .values() + .map(|c| (c.id, c.file_path.clone())) + .collect(); + + for (clip_id, file_path) in videos { + if !std::path::Path::new(&file_path).exists() { + eprintln!("⚠️ [APPLY] Video file not found, skipping: {}", file_path); + continue; + } + + { + let mut video_mgr = self.video_manager.lock().unwrap(); + if let Err(e) = video_mgr.load_video(clip_id, file_path.clone(), doc_width, doc_height) { + eprintln!("⚠️ [APPLY] Failed to load video {}: {}", file_path, e); + continue; + } + } + + self.spawn_keyframe_index(clip_id); + self.spawn_thumbnail_generation(clip_id); + } + } + + /// Spawn a background thread that builds the keyframe (seek) index for a + /// video clip. The clip must already be registered via `load_video`. + /// + /// The slow packet scan runs holding **no** lock — only brief locks bracket it + /// (to grab the decoder handle and to store the result) — so it never blocks + /// playback or the UI (which both need the VideoManager / decoder locks). + fn spawn_keyframe_index(&self, clip_id: uuid::Uuid) { + let video_manager = Arc::clone(&self.video_manager); + std::thread::spawn(move || { + let decoder_arc = { + let vm = video_manager.lock().unwrap(); + vm.get_decoder(&clip_id) + }; + let Some(decoder_arc) = decoder_arc else { return; }; + + let (path, stream_index) = { + let decoder = decoder_arc.lock().unwrap(); + decoder.keyframe_scan_params() + }; + + // Slow scan, no locks held. + match lightningbeam_core::video::VideoDecoder::scan_keyframes(&path, stream_index) { + Ok(positions) => { + let count = positions.len(); + decoder_arc.lock().unwrap().set_keyframe_index(positions); + println!(" Built keyframe index ({} keyframes) for video clip {}", count, clip_id); + } + Err(e) => eprintln!("Failed to build keyframe index for {}: {}", clip_id, e), + } + }); + } + + /// Spawn a background thread that (re)generates timeline thumbnails for a + /// video clip. Uses a **dedicated** decoder (independent of the playback + /// decoder) and samples at keyframes, so it neither blocks playback nor pays + /// the cost of decoding whole GOPs. Safe to call on its own to refresh a + /// single clip's thumbnails. + fn spawn_thumbnail_generation(&self, clip_id: uuid::Uuid) { + let video_manager = Arc::clone(&self.video_manager); + std::thread::spawn(move || { + // Grab the source path from the playback decoder (brief lock), then do + // all decoding on an independent decoder. + let path = { + let decoder_arc = { + let vm = video_manager.lock().unwrap(); + vm.get_decoder(&clip_id) + }; + let Some(decoder_arc) = decoder_arc else { return; }; + let decoder = decoder_arc.lock().unwrap(); + decoder.path().to_string() + }; + + let vm_insert = Arc::clone(&video_manager); + let result = lightningbeam_core::video::generate_keyframe_thumbnails( + &path, + 5.0, + 128, + |ts, rgba| { + let mut vm = vm_insert.lock().unwrap(); + vm.insert_thumbnail(&clip_id, ts, rgba); + }, + ); + if let Err(e) = result { + eprintln!("Thumbnail generation failed for {}: {}", clip_id, e); + } + }); + } + + /// If `path` is a large media file (≥ the large-media threshold) and the user + /// hasn't yet chosen a default pack-vs-reference policy, queue the one-time + /// prompt. The actual storage decision happens at save time from the chosen + /// preference; this only establishes that preference. + fn note_possible_large_media(&mut self, path: &std::path::Path) { + use lightningbeam_core::file_io::LargeMediaMode; + if self.config.large_media_default != LargeMediaMode::Ask || self.large_media_prompt.is_some() { + return; + } + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + if size >= lightningbeam_core::beam_archive::LARGE_MEDIA_THRESHOLD { + let name = path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| path.display().to_string()); + self.large_media_prompt = Some(name); + } + } + + /// Render the one-time "pack vs reference large media" prompt, if pending. + /// The choice is persisted to config as the future default. + fn render_large_media_prompt(&mut self, ctx: &egui::Context) { + use lightningbeam_core::file_io::LargeMediaMode; + let Some(name) = self.large_media_prompt.clone() else { + return; + }; + let threshold_gb = lightningbeam_core::beam_archive::LARGE_MEDIA_THRESHOLD as f64 + / (1024.0 * 1024.0 * 1024.0); + let mut choice: Option = None; + egui::Window::new("Large media file") + .collapsible(false) + .resizable(false) + .anchor(egui::Align2::CENTER_CENTER, egui::vec2(0.0, 0.0)) + .show(ctx, |ui| { + ui.label(format!("\"{}\" is larger than {:.0} GB.", name, threshold_gb)); + ui.add_space(6.0); + ui.label("How should large media be stored in projects from now on?"); + ui.add_space(6.0); + ui.label("• Pack — copy the bytes into the .beam file (self-contained, larger project)."); + ui.label("• Reference — keep the file on disk and store only its path (smaller project; the file must stay put)."); + ui.add_space(10.0); + ui.horizontal(|ui| { + if ui.button("Pack into project").clicked() { + choice = Some(LargeMediaMode::Pack); + } + if ui.button("Reference external file").clicked() { + choice = Some(LargeMediaMode::Reference); + } + }); + ui.add_space(4.0); + ui.label( + egui::RichText::new("You can change this later in Preferences.") + .weak() + .small(), + ); + }); + if let Some(mode) = choice { + self.config.large_media_default = mode; + self.config.save(); + self.large_media_prompt = None; + } + } + /// Import an image file as an ImageAsset fn import_image(&mut self, path: &std::path::Path) -> Option { use lightningbeam_core::clip::ImageAsset; + self.note_possible_large_media(path); // Get filename for asset name let name = path.file_stem() @@ -3963,6 +4308,7 @@ impl EditorApp { /// GPU waveform cache. fn import_audio(&mut self, path: &std::path::Path) -> Option { use lightningbeam_core::clip::AudioClip; + self.note_possible_large_media(path); let name = path.file_stem() .and_then(|s| s.to_str()) @@ -4088,6 +4434,7 @@ impl EditorApp { fn import_video(&mut self, path: &std::path::Path) -> Option { use lightningbeam_core::clip::VideoClip; use lightningbeam_core::video::probe_video; + self.note_possible_large_media(path); let name = path.file_stem() .and_then(|s| s.to_str()) @@ -4129,82 +4476,95 @@ impl EditorApp { drop(video_mgr); // Spawn background thread to build keyframe index asynchronously - let video_manager_clone = Arc::clone(&self.video_manager); - let keyframe_clip_id = clip_id; - std::thread::spawn(move || { - let video_mgr = video_manager_clone.lock().unwrap(); - if let Err(e) = video_mgr.build_keyframe_index(&keyframe_clip_id) { - eprintln!("Failed to build keyframe index: {}", e); - } else { - println!(" Built keyframe index for video clip {}", keyframe_clip_id); - } - }); + self.spawn_keyframe_index(clip_id); - // Spawn background thread for audio extraction if video has audio + // Register the video's audio track as a streaming pool entry if present. if metadata.has_audio { if let Some(ref audio_controller) = self.audio_controller { let path_clone = path_str.clone(); let video_clip_id = clip_id; let video_name = name.clone(); + let video_duration = metadata.duration; let audio_controller_clone = Arc::clone(audio_controller); let tx = self.audio_extraction_tx.clone(); + let wf_tx = self.waveform_result_tx.clone(); + let floor_b = self.config.waveform_floor_samples_per_texel.max(1); std::thread::spawn(move || { - use lightningbeam_core::video::extract_audio_from_video; use lightningbeam_core::clip::AudioClip; - // Extract audio from video (slow FFmpeg operation) - match extract_audio_from_video(&path_clone) { - Ok(Some(extracted)) => { - // Add audio to daw-backend pool synchronously to get pool index - let pool_index = { - let mut controller = audio_controller_clone.lock().unwrap(); - match controller.add_audio_file_sync( - path_clone.clone(), - extracted.samples, - extracted.channels, - extracted.sample_rate, - ) { - Ok(index) => index, - Err(e) => { - eprintln!("Failed to add audio file to backend: {}", e); - let _ = tx.send(AudioExtractionResult::Error { - video_clip_id, - error: format!("Failed to add audio to backend: {}", e), - }); - return; - } - } - }; + // Add the video's audio track as a streaming pool entry — it is + // decoded on demand from the video file by the disk reader's + // FFmpeg backend. No extraction to disk or RAM. + let mut controller = audio_controller_clone.lock().unwrap(); + let pool_index = + match controller.add_video_audio_sync(std::path::PathBuf::from(&path_clone)) { + Ok(index) => index, + Err(e) => { + drop(controller); + eprintln!("Failed to add video audio stream: {}", e); + let _ = tx.send(AudioExtractionResult::Error { + video_clip_id, + error: format!("Failed to add video audio: {}", e), + }); + return; + } + }; + // Pull the audio track's channels/sample_rate for reporting + // (duration comes from the video itself, so the clip length + // matches the video clip exactly). + let (_dur, sample_rate, channels) = controller + .get_pool_file_info(pool_index) + .unwrap_or((video_duration, 0, 0)); + drop(controller); - // Create AudioClip - let audio_clip_name = format!("{} (Audio)", video_name); - let audio_clip = AudioClip::new_sampled( - &audio_clip_name, - pool_index, - extracted.duration, - ); + let audio_clip_name = format!("{} (Audio)", video_name); + let audio_clip = + AudioClip::new_sampled(&audio_clip_name, pool_index, video_duration); - // Send success result - let _ = tx.send(AudioExtractionResult::Success { - video_clip_id, - audio_clip, + let _ = tx.send(AudioExtractionResult::Success { + video_clip_id, + audio_clip, + pool_index, + video_name, + channels, + sample_rate, + }); + + // Build the min/max waveform overview pyramid by streaming the + // video's audio through FFmpeg once. Video-audio has no in-RAM + // samples, so without this the clip would draw no waveform. + use daw_backend::audio::disk_reader::{build_waveform_pyramid, SourceKind}; + match build_waveform_pyramid( + std::path::Path::new(&path_clone), + SourceKind::VideoAudio, + floor_b, + ) { + Ok(pyramid) => { + // Pack the floor level interleaved as (Lmin,Lmax,Rmin,Rmax) + // f32 per texel — the layout the GPU min/max upload expects. + let floor = pyramid.floor(); + let mut packed = Vec::with_capacity(floor.len() * 4); + for t in floor { + packed.push(t.l_min); + packed.push(t.l_max); + packed.push(t.r_min); + packed.push(t.r_max); + } + // Serialize the whole pyramid for persistence in the .beam + // container (so a later load is a disk read, not a re-decode). + let blob = pyramid.to_bytes(); + let _ = wf_tx.send(( pool_index, - video_name, - channels: extracted.channels, - sample_rate: extracted.sample_rate, - }); - } - Ok(None) => { - // Video has no audio stream - let _ = tx.send(AudioExtractionResult::NoAudio { video_clip_id }); + packed, + sample_rate, + channels, + pyramid.floor_samples_per_texel, + blob, + )); } Err(e) => { - // Audio extraction failed - let _ = tx.send(AudioExtractionResult::Error { - video_clip_id, - error: e, - }); + eprintln!("Failed to build waveform pyramid for video audio: {}", e); } } }); @@ -4213,63 +4573,10 @@ impl EditorApp { } } - // Spawn background thread for thumbnail generation - // Get decoder once, then generate thumbnails without holding VideoManager lock - let video_manager_clone = Arc::clone(&self.video_manager); - let duration = metadata.duration; - let thumb_clip_id = clip_id; - std::thread::spawn(move || { - // Get decoder Arc with brief lock - let decoder_arc = { - let video_mgr = video_manager_clone.lock().unwrap(); - match video_mgr.get_decoder(&thumb_clip_id) { - Some(arc) => arc, - None => { - eprintln!("Failed to get decoder for thumbnail generation"); - return; - } - } - }; - // VideoManager lock released - video can now be displayed! - - let interval = 5.0; - let mut t = 0.0; - let mut thumbnail_count = 0; - - while t < duration { - // Decode frame WITHOUT holding VideoManager lock - let thumb_opt = { - let mut decoder = decoder_arc.lock().unwrap(); - match decoder.decode_frame(t) { - Ok(rgba_data) => { - let w = decoder.get_output_width(); - let h = decoder.get_output_height(); - Some((rgba_data, w, h)) - } - Err(_) => None, - } - }; - - // Downsample without any locks - if let Some((rgba_data, w, h)) = thumb_opt { - use lightningbeam_core::video::downsample_rgba_public; - let thumb_w = 128u32; - let thumb_h = (h as f32 / w as f32 * thumb_w as f32) as u32; - let thumb_data = downsample_rgba_public(&rgba_data, w, h, thumb_w, thumb_h); - - // Brief lock just to insert - { - let mut video_mgr = video_manager_clone.lock().unwrap(); - video_mgr.insert_thumbnail(&thumb_clip_id, t, Arc::new(thumb_data)); - } - thumbnail_count += 1; - } - - t += interval; - } - - println!(" Generated {} thumbnails for video clip {}", thumbnail_count, thumb_clip_id); - }); + // Spawn background thread for thumbnail generation. The video becomes + // displayable as soon as the decoder is registered (above); thumbnails + // fill in without holding the VideoManager lock during decode. + self.spawn_thumbnail_generation(clip_id); // Add clip to document let clip_id = self.action_executor.document_mut().add_video_clip(clip); @@ -4665,9 +4972,6 @@ impl EditorApp { eprintln!("⚠️ Audio extracted but VideoClip {} not found (may have been deleted)", video_clip_id); } } - AudioExtractionResult::NoAudio { video_clip_id } => { - println!("ℹ️ Video {} has no audio stream", video_clip_id); - } AudioExtractionResult::Error { video_clip_id, error } => { eprintln!("❌ Failed to extract audio from video {}: {}", video_clip_id, error); } @@ -4702,6 +5006,17 @@ impl eframe::App for EditorApp { self.handle_audio_extraction_result(result); } + // Poll completed waveform-overview pyramids (packed min/max floors). + // Stored in the same cache the GPU upload reads from, but flagged as + // min/max so the renderer uploads via the (Lmin,Lmax,Rmin,Rmax) path. + while let Ok((pool_index, packed, sample_rate, channels, b, blob)) = self.waveform_result_rx.try_recv() { + self.raw_audio_cache.insert(pool_index, (Arc::new(packed), sample_rate, channels)); + self.waveform_minmax_pools.insert(pool_index, b.max(1)); + self.waveform_pyramid_blobs.insert(pool_index, Arc::new(blob)); + self.waveform_gpu_dirty.insert(pool_index); + self.audio_pools_with_new_waveforms.insert(pool_index); + } + // Webcam management: open/close based on camera_enabled layers, poll frames { let any_camera_enabled = self.action_executor.document().all_layers().iter().any(|layer| { @@ -5509,6 +5824,9 @@ impl eframe::App for EditorApp { } } + // One-time prompt: how to store large media (pack vs reference). + self.render_large_media_prompt(ctx); + // Render video frames incrementally (if video export in progress) if let Some(orchestrator) = &mut self.export_orchestrator { if orchestrator.is_exporting() { @@ -5825,6 +6143,7 @@ impl eframe::App for EditorApp { audio_pools_with_new_waveforms: &self.audio_pools_with_new_waveforms, raw_audio_cache: &self.raw_audio_cache, waveform_gpu_dirty: &mut self.waveform_gpu_dirty, + waveform_minmax_pools: &self.waveform_minmax_pools, effect_to_load: &mut self.effect_to_load, effect_thumbnail_requests: &mut effect_thumbnail_requests, effect_thumbnail_cache: self.effect_thumbnail_generator.as_ref() @@ -6066,24 +6385,9 @@ impl eframe::App for EditorApp { } } - // Generate thumbnails in background - let vm_clone = Arc::clone(&self.video_manager); - std::thread::spawn(move || { - // Build keyframe index first - { - let vm = vm_clone.lock().unwrap(); - if let Err(e) = vm.build_keyframe_index(&clip_id) { - eprintln!("[WEBCAM] Failed to build keyframe index: {e}"); - } - } - // Generate thumbnails - { - let mut vm = vm_clone.lock().unwrap(); - if let Err(e) = vm.generate_thumbnails(&clip_id, duration) { - eprintln!("[WEBCAM] Failed to generate thumbnails: {e}"); - } - } - }); + // Build keyframe index + thumbnails in background. + self.spawn_keyframe_index(clip_id); + self.spawn_thumbnail_generation(clip_id); eprintln!( "[WEBCAM] probe_video: duration={:.4}s, fps={:.1}, {}x{}. Using probe duration for clip.", diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs index 027a839..6ddcc8f 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs @@ -242,6 +242,10 @@ pub struct SharedPaneState<'a> { pub raw_audio_cache: &'a std::collections::HashMap>, u32, u32)>, /// Pool indices needing GPU waveform texture upload pub waveform_gpu_dirty: &'a mut std::collections::HashSet, + /// Pools whose `raw_audio_cache` entry is a packed min/max floor rather than + /// raw samples (pool_index -> `B`, floor frames-per-texel). Drives the GPU + /// min/max upload path and the floor's effective rate `sr/B` in the renderer. + pub waveform_minmax_pools: &'a std::collections::HashMap, /// Effect ID to load into shader editor (set by asset library, consumed by shader editor) pub effect_to_load: &'a mut Option, /// Queue for effect thumbnail requests (effect IDs to generate thumbnails for) diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl index 9a857ba..abeadac 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/waveform.wgsl @@ -77,27 +77,29 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { let frames_per_pixel = params.sample_rate / params.pixels_per_second; // Each mip level reduces by 4x in sample count (2x in each texture dimension) let mip_f = max(0.0, log2(frames_per_pixel) / 2.0); - let max_mip = f32(textureNumLevels(peak_tex) - 1u); - let mip = min(mip_f, max_mip); - // Frame index at the chosen mip level - let mip_floor = u32(mip); - let reduction = pow(4.0, f32(mip_floor)); + // Pick the NEAREST INTEGER LOD and read its exact texel. Sampling at a + // fractional mip (trilinear) blends level N and N+1, but each level has its + // own 1D→2D row-major linearization (width halves per level), so the two + // levels disagree on which audio frame a given screen column maps to. The + // blend then reads horizontally-offset neighbours, and because a 2x zoom step + // shifts mip_f by exactly 0.5, alternate zoom levels land on a clean integer + // (correct) vs a 50/50 blend (offset) — the "every other zoom level" artifact. + // textureLoad at one integer level keeps the frame→texel mapping exact. + let max_mip = i32(textureNumLevels(peak_tex)) - 1; + let mip_i = clamp(i32(mip_f + 0.5), 0, max_mip); + let reduction = pow(4.0, f32(mip_i)); let mip_frame = frame_f / reduction; - // Convert 1D mip-space index to 2D UV coordinates - // Use actual texture dimensions (not computed from total_frames) because the - // texture may be pre-allocated larger for live recording. - let mip_dims = textureDimensions(peak_tex, mip_floor); + // Convert 1D mip-space index to 2D texel coords using this level's actual + // dimensions (texture may be pre-allocated larger, e.g. for live recording). + let mip_dims = textureDimensions(peak_tex, mip_i); let mip_tex_width = f32(mip_dims.x); - let mip_tex_height = f32(mip_dims.y); - let texel_x = mip_frame % mip_tex_width; - let texel_y = floor(mip_frame / mip_tex_width); - let uv = vec2((texel_x + 0.5) / mip_tex_width, (texel_y + 0.5) / mip_tex_height); + let texel_x = i32(mip_frame % mip_tex_width); + let texel_y = i32(floor(mip_frame / mip_tex_width)); - // Sample the peak texture at computed mip level // R = left_min, G = left_max, B = right_min, A = right_max - let peak = textureSampleLevel(peak_tex, peak_sampler, uv, mip); + let peak = textureLoad(peak_tex, vec2(texel_x, texel_y), mip_i); let clip_height = params.clip_rect.w - params.clip_rect.y; let clip_top = params.clip_rect.y; diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index fbfa360..aae69ae 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -2632,6 +2632,7 @@ impl TimelinePane { midi_event_cache: &std::collections::HashMap>, raw_audio_cache: &std::collections::HashMap>, u32, u32)>, waveform_gpu_dirty: &mut std::collections::HashSet, + waveform_minmax_pools: &std::collections::HashMap, target_format: wgpu::TextureFormat, waveform_stereo: bool, context_layers: &[&lightningbeam_core::layer::AnyLayer], @@ -2899,8 +2900,9 @@ impl TimelinePane { theme.text_color(&["#timeline", ".group-bar"], ui.ctx(), egui::Color32::from_rgb(100, 220, 220)) }; for (s, e) in &merged { - let sx = self.time_to_x(*s); - let ex = self.time_to_x(*e).max(sx + MIN_CLIP_WIDTH_PX); + // `merged` ranges are in beats; convert to seconds for time_to_x. + let sx = self.time_to_x(document.tempo_map().transform(*s)); + let ex = self.time_to_x(document.tempo_map().transform(*e)).max(sx + MIN_CLIP_WIDTH_PX); if ex >= 0.0 && sx <= rect.width() { let vsx = sx.max(0.0); let vex = ex.min(rect.width()); @@ -2940,8 +2942,9 @@ impl TimelinePane { let ci_duration = ci.total_duration(clip_dur, document.tempo_map()); let ci_end = ci_start + ci_duration; - let sx = self.time_to_x(ci_start); - let ex = self.time_to_x(ci_end); + // ci_start/ci_end are in beats; convert to seconds for time_to_x. + let sx = self.time_to_x(document.tempo_map().transform(ci_start)); + let ex = self.time_to_x(document.tempo_map().transform(ci_end)); if ex < 0.0 || sx > rect.width() { continue; } let ci_rect = egui::Rect::from_min_max( @@ -3048,8 +3051,17 @@ impl TimelinePane { None => continue, }; - let total_frames = samples.len() / (*ch).max(1) as usize; - let audio_file_duration = total_frames as f64 / *sr as f64; + // Min/max overview pools store 4 f32 per texel at the + // floor rate sr/B; raw pools store interleaved samples. + let minmax_b = waveform_minmax_pools.get(&audio_pool_index).copied(); + let is_minmax = minmax_b.is_some(); + let frame_stride = if is_minmax { 4 } else { (*ch).max(1) as usize }; + let total_frames = samples.len() / frame_stride; + let eff_sr: f32 = match minmax_b { + Some(b) => *sr as f32 / b.max(1) as f32, + None => *sr as f32, + }; + let audio_file_duration = total_frames as f64 / eff_sr as f64; let clip_dur = audio_clip.duration; let mut ci_start = ci.effective_start(); @@ -3058,8 +3070,9 @@ impl TimelinePane { } let ci_duration = ci.total_duration(clip_dur, document.tempo_map()); - let ci_screen_start = rect.min.x + self.time_to_x(ci_start); - let ci_screen_end = ci_screen_start + (ci_duration * self.pixels_per_second as f64) as f32; + // ci_start/ci_duration are in beats; convert to seconds for time_to_x. + let ci_screen_start = rect.min.x + self.time_to_x(document.tempo_map().transform(ci_start)); + let ci_screen_end = rect.min.x + self.time_to_x(document.tempo_map().transform(ci_start + ci_duration)); let waveform_rect = egui::Rect::from_min_max( egui::pos2(ci_screen_start.max(rect.min.x), wave_y_min), @@ -3081,9 +3094,10 @@ impl TimelinePane { } Some(crate::waveform_gpu::PendingUpload { samples: samples.clone(), - sample_rate: *sr, + sample_rate: if is_minmax { eff_sr.round().max(1.0) as u32 } else { *sr }, channels: *ch, frame_limit, + minmax: is_minmax, }) } else { None @@ -3098,7 +3112,7 @@ impl TimelinePane { viewport_start_time: self.viewport_start_time as f32, pixels_per_second: self.pixels_per_second as f32, audio_duration: audio_file_duration as f32, - sample_rate: *sr as f32, + sample_rate: eff_sr, clip_start_time: ci_screen_start, trim_start: ci.trim_start as f32, tex_width: crate::waveform_gpu::tex_width() as f32, @@ -3597,8 +3611,16 @@ impl TimelinePane { // Sampled Audio: Draw waveform via GPU lightningbeam_core::clip::AudioClipType::Sampled { audio_pool_index } => { if let Some((samples, sr, ch)) = raw_audio_cache.get(audio_pool_index) { - let total_frames = samples.len() / (*ch).max(1) as usize; - let audio_file_duration = total_frames as f64 / *sr as f64; + // Min/max overview pools: 4 f32/texel at rate sr/B. + let minmax_b = waveform_minmax_pools.get(audio_pool_index).copied(); + let is_minmax = minmax_b.is_some(); + let frame_stride = if is_minmax { 4 } else { (*ch).max(1) as usize }; + let total_frames = samples.len() / frame_stride; + let eff_sr: f32 = match minmax_b { + Some(b) => *sr as f32 / b.max(1) as f32, + None => *sr as f32, + }; + let audio_file_duration = total_frames as f64 / eff_sr as f64; let screen_size = ui.ctx().content_rect().size(); let pending_upload = if waveform_gpu_dirty.contains(audio_pool_index) { @@ -3620,9 +3642,10 @@ impl TimelinePane { Some(crate::waveform_gpu::PendingUpload { samples: samples.clone(), - sample_rate: *sr, + sample_rate: if is_minmax { eff_sr.round().max(1.0) as u32 } else { *sr }, channels: *ch, frame_limit, + minmax: is_minmax, }) } else { None @@ -3684,7 +3707,7 @@ impl TimelinePane { viewport_start_time: self.viewport_start_time as f32, pixels_per_second: self.pixels_per_second as f32, audio_duration: audio_file_duration as f32, - sample_rate: *sr as f32, + sample_rate: eff_sr, clip_start_time: iter_screen_start, trim_start: preview_trim_start as f32, tex_width: crate::waveform_gpu::tex_width() as f32, @@ -3730,6 +3753,7 @@ impl TimelinePane { sample_rate: *sr, channels: *ch, frame_limit: None, // recording uses incremental path + minmax: false, }) } else { None @@ -5418,7 +5442,7 @@ impl PaneRenderer for TimelinePane { // Render layer rows with clipping ui.set_clip_rect(content_rect.intersect(original_clip_rect)); - let (video_clip_hovers, pending_lane_renders) = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.focus, shared.selection, shared.midi_event_cache, shared.raw_audio_cache, shared.waveform_gpu_dirty, shared.target_format, shared.waveform_stereo, &context_layers, shared.video_manager, &audio_cache, *shared.playback_time); + let (video_clip_hovers, pending_lane_renders) = self.render_layers(ui, content_rect, shared.theme, document, shared.active_layer_id, shared.focus, shared.selection, shared.midi_event_cache, shared.raw_audio_cache, shared.waveform_gpu_dirty, shared.waveform_minmax_pools, shared.target_format, shared.waveform_stereo, &context_layers, shared.video_manager, &audio_cache, *shared.playback_time); // Render playhead on top (clip to timeline area) ui.set_clip_rect(timeline_rect.intersect(original_clip_rect)); diff --git a/lightningbeam-ui/lightningbeam-editor/src/preferences/dialog.rs b/lightningbeam-ui/lightningbeam-editor/src/preferences/dialog.rs index 970cd0e..e0f0989 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/preferences/dialog.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/preferences/dialog.rs @@ -9,6 +9,7 @@ use crate::config::AppConfig; use crate::keymap::{self, AppAction, KeymapManager}; use crate::menu::{MenuSystem, Shortcut, ShortcutKey}; use crate::theme::{Theme, ThemeMode}; +use lightningbeam_core::file_io::LargeMediaMode; /// Which tab is selected in the preferences dialog #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -59,6 +60,7 @@ struct PreferencesState { debug: bool, waveform_stereo: bool, theme_mode: ThemeMode, + large_media_default: LargeMediaMode, } impl From<(&AppConfig, &Theme)> for PreferencesState { @@ -75,6 +77,7 @@ impl From<(&AppConfig, &Theme)> for PreferencesState { debug: config.debug, waveform_stereo: config.waveform_stereo, theme_mode: theme.mode(), + large_media_default: config.large_media_default, } } } @@ -93,6 +96,7 @@ impl Default for PreferencesState { debug: false, waveform_stereo: false, theme_mode: ThemeMode::System, + large_media_default: LargeMediaMode::default(), } } } @@ -567,6 +571,24 @@ impl PreferencesDialog { &mut self.working_prefs.waveform_stereo, "Show waveforms as stacked stereo", ); + ui.horizontal(|ui| { + let threshold_gb = lightningbeam_core::beam_archive::LARGE_MEDIA_THRESHOLD + as f64 + / (1024.0 * 1024.0 * 1024.0); + ui.label(format!("Large media (>{:.0} GB):", threshold_gb)); + let label = |m: LargeMediaMode| match m { + LargeMediaMode::Ask => "Ask each time", + LargeMediaMode::Pack => "Pack into project", + LargeMediaMode::Reference => "Reference external file", + }; + egui::ComboBox::from_id_salt("large_media_default") + .selected_text(label(self.working_prefs.large_media_default)) + .show_ui(ui, |ui| { + for mode in [LargeMediaMode::Ask, LargeMediaMode::Pack, LargeMediaMode::Reference] { + ui.selectable_value(&mut self.working_prefs.large_media_default, mode, label(mode)); + } + }); + }); }); } @@ -629,6 +651,7 @@ impl PreferencesDialog { config.debug = self.working_prefs.debug; config.waveform_stereo = self.working_prefs.waveform_stereo; config.theme_mode = self.working_prefs.theme_mode.to_string_lower(); + config.large_media_default = self.working_prefs.large_media_default; config.keybindings = keybinding_config; // Apply theme immediately diff --git a/lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs b/lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs index 8195727..1301342 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/waveform_gpu.rs @@ -112,6 +112,37 @@ pub struct PendingUpload { /// The texture is allocated at full size, but total_frames is set to /// the limited count so subsequent calls use the incremental path. pub frame_limit: Option, + /// When true, `samples` is interpreted as **pre-packed min/max texels**: + /// 4 floats per "frame" = `[l_min, l_max, r_min, r_max]` (a waveform-pyramid + /// floor level). The caller passes the *effective* `sample_rate` (`sr / B`) + /// so the shader's time→texel mapping covers `B` source samples per texel. + /// When false, `samples` is raw interleaved audio (min = max per texel). + pub minmax: bool, +} + +/// Pack one source "frame" into an `Rgba16Float` texel `(Lmin,Lmax,Rmin,Rmax)`. +/// Raw audio sets min = max per channel; min/max input copies the 4 values. +#[inline] +fn pack_texel(samples: &[f32], global_frame: usize, channels: usize, minmax: bool) -> [half::f16; 4] { + if minmax { + let o = global_frame * 4; + [ + half::f16::from_f32(samples.get(o).copied().unwrap_or(0.0)), + half::f16::from_f32(samples.get(o + 1).copied().unwrap_or(0.0)), + half::f16::from_f32(samples.get(o + 2).copied().unwrap_or(0.0)), + half::f16::from_f32(samples.get(o + 3).copied().unwrap_or(0.0)), + ] + } else { + let so = global_frame * channels; + let left = samples.get(so).copied().unwrap_or(0.0); + let right = if channels >= 2 { + samples.get(so + 1).copied().unwrap_or(left) + } else { + left + }; + let (l, r) = (half::f16::from_f32(left), half::f16::from_f32(right)); + [l, l, r, r] + } } /// Maximum frames to convert and upload per frame (~250K frames ≈ 5.6s at 44.1kHz). @@ -291,8 +322,11 @@ impl WaveformGpuResources { sample_rate: u32, channels: u32, frame_limit: Option, + minmax: bool, ) -> Vec { - let new_total_frames = samples.len() / channels.max(1) as usize; + // For min/max input each "frame" is 4 floats; for raw it's `channels`. + let frame_stride = if minmax { 4 } else { channels.max(1) as usize }; + let new_total_frames = samples.len() / frame_stride; if new_total_frames == 0 { return Vec::new(); } @@ -329,22 +363,9 @@ impl WaveformGpuResources { if global_frame >= effective_frames { break; } - let sample_offset = global_frame * channels as usize; - let left = if sample_offset < samples.len() { - samples[sample_offset] - } else { - 0.0 - }; - let right = if channels >= 2 && sample_offset + 1 < samples.len() { - samples[sample_offset + 1] - } else { - left - }; let texel_offset = frame * 4; - row_data[texel_offset] = half::f16::from_f32(left); - row_data[texel_offset + 1] = half::f16::from_f32(left); - row_data[texel_offset + 2] = half::f16::from_f32(right); - row_data[texel_offset + 3] = half::f16::from_f32(right); + let t = pack_texel(samples, global_frame, channels as usize, minmax); + row_data[texel_offset..texel_offset + 4].copy_from_slice(&t); } let entry = self.entries.get(&pool_index).unwrap(); @@ -466,24 +487,9 @@ impl WaveformGpuResources { for frame in 0..seg_upload_count as usize { let global_frame = seg_start_frame as usize + frame; - let sample_offset = global_frame * channels as usize; - - let left = if sample_offset < samples.len() { - samples[sample_offset] - } else { - 0.0 - }; - let right = if channels >= 2 && sample_offset + 1 < samples.len() { - samples[sample_offset + 1] - } else { - left - }; - let texel_offset = frame * 4; - mip0_data[texel_offset] = half::f16::from_f32(left); - mip0_data[texel_offset + 1] = half::f16::from_f32(left); - mip0_data[texel_offset + 2] = half::f16::from_f32(right); - mip0_data[texel_offset + 3] = half::f16::from_f32(right); + let t = pack_texel(samples, global_frame, channels as usize, minmax); + mip0_data[texel_offset..texel_offset + 4].copy_from_slice(&t); } // Upload mip 0 (only rows with actual data) @@ -701,6 +707,7 @@ impl egui_wgpu::CallbackTrait for WaveformCallback { upload.sample_rate, upload.channels, upload.frame_limit, + upload.minmax, ); cmds.extend(new_cmds); }