From 4ad95e67556e07225480ac388fa8ec736b00e803 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Wed, 17 Jun 2026 17:58:54 -0400 Subject: [PATCH] Downmix surround to stereo + reload video audio via FFmpeg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surround → stereo downmix: - render_from_file folds multichannel sources (5.1/7.1/…) down to stereo with proper coefficients (full level for the matching front channel, 1/√2 for centre + each surround, LFE dropped), normalized per row to avoid clipping (matching ffmpeg's default). Applied uniformly to both the direct-copy and sinc-resample paths and to every storage type (PCM, compressed, video audio), only when dst==2 && src>2; unknown layouts fall back to front L/R. Previously it just took FL/FR, dropping centre dialog + surrounds. Proper video-audio reload: - A video's audio track is now stored as a path reference to the video (never packed/embedded as audio media) and re-probed via FFmpeg on load into a streaming VideoAudio entry, so multichannel audio survives reload (the old Symphonia reconstitution collapsed it, breaking the downmix). Driven by a new AudioPoolEntry.is_video_audio flag across serialize / save_beam / load. Also removes the decode-whole-video-to-RAM + temp-file path on load. Fix video scaling: - Any video with dimensions larger than the stage was being scaled down into the corner incorrectly; we now bake the frame-clip scale into the instance transform. --- daw-backend/src/audio/pool.rs | 192 +++++++++++++++--- .../lightningbeam-core/src/file_io.rs | 11 +- .../lightningbeam-core/src/renderer.rs | 32 ++- 3 files changed, 202 insertions(+), 33 deletions(-) diff --git a/daw-backend/src/audio/pool.rs b/daw-backend/src/audio/pool.rs index 8748859..6e329ad 100644 --- a/daw-backend/src/audio/pool.rs +++ b/daw-backend/src/audio/pool.rs @@ -4,6 +4,46 @@ use std::f32::consts::PI; use serde::{Deserialize, Serialize}; use crate::time::Seconds; +/// Per-output-channel mix coefficients to fold a multichannel source down to +/// stereo, indexed `[out_channel(0=L,1=R)][src_channel]`. +/// +/// Assumes the conventional interleave order for each channel count (FL, FR, FC, +/// LFE, BL, BR, SL, SR …). Uses standard ITU/AC-3-style coefficients: full level +/// for the matching front channel, `1/√2` (≈ −3 dB) for centre and each surround, +/// LFE dropped. Each row is then normalized so its absolute-coefficient sum ≤ 1, +/// which prevents clipping (matching FFmpeg's default `normalize` behaviour). +/// +/// Returns `None` for layouts we don't special-case (caller falls back to taking +/// the front L/R pair). +fn stereo_downmix_matrix(src_channels: usize) -> Option<[Vec; 2]> { + const C: f32 = std::f32::consts::FRAC_1_SQRT_2; // ≈ 0.7071 + // (L row, R row); each entry is the gain applied to that source channel. + let (l, r): (Vec, Vec) = match src_channels { + 3 => (vec![1.0, 0.0, C], vec![0.0, 1.0, C]), // FL FR FC + 4 => (vec![1.0, 0.0, C, 0.0], vec![0.0, 1.0, 0.0, C]), // quad: FL FR BL BR + 5 => (vec![1.0, 0.0, C, C, 0.0], vec![0.0, 1.0, C, 0.0, C]), // FL FR FC BL BR + // 5.1: FL FR FC LFE BL BR (LFE dropped) + 6 => (vec![1.0, 0.0, C, 0.0, C, 0.0], vec![0.0, 1.0, C, 0.0, 0.0, C]), + // 6.1: FL FR FC LFE BC SL SR (BC → both) + 7 => (vec![1.0, 0.0, C, 0.0, C, C, 0.0], vec![0.0, 1.0, C, 0.0, C, 0.0, C]), + // 7.1: FL FR FC LFE BL BR SL SR + 8 => ( + vec![1.0, 0.0, C, 0.0, C, 0.0, C, 0.0], + vec![0.0, 1.0, C, 0.0, 0.0, C, 0.0, C], + ), + _ => return None, + }; + let normalize = |row: Vec| -> Vec { + let sum: f32 = row.iter().map(|c| c.abs()).sum(); + if sum > 1.0 { + row.into_iter().map(|c| c / sum).collect() + } else { + row + } + }; + Some([normalize(l), normalize(r)]) +} + /// Windowed sinc interpolation for high-quality time stretching /// This is stateless and can handle arbitrary fractional positions #[inline] @@ -580,6 +620,15 @@ impl AudioClipPool { let dst_channels = engine_channels as usize; let output_frames = output.len() / dst_channels; + // Fold a multichannel source (5.1, 7.1, …) down to stereo with proper + // coefficients (centre + surrounds mixed in, LFE dropped) instead of just + // taking the front L/R pair. `None` ⇒ no downmix needed / unknown layout. + let downmix = if dst_channels == 2 && src_channels > 2 { + stereo_downmix_matrix(src_channels) + } else { + None + }; + let src_start_position = start_time_seconds * audio_file.sample_rate as f64; // Tell the disk reader where we're reading so it buffers the right region. @@ -625,6 +674,15 @@ impl AudioClipPool { sum += get_sample!(sf, src_ch); } sum / src_channels as f32 + } else if let Some(ref mat) = downmix { + // Surround → stereo with proper coefficients. + let mut s = 0.0f32; + for (src_ch, &c) in mat[dst_ch].iter().enumerate() { + if c != 0.0 { + s += c * get_sample!(sf, src_ch); + } + } + s } else { get_sample!(sf, dst_ch % src_channels) }; @@ -649,39 +707,45 @@ impl AudioClipPool { break; } - for dst_ch in 0..dst_channels { - let src_ch = if src_channels == dst_channels { - dst_ch - } else if src_channels == 1 { - 0 - } else if dst_channels == 1 { - usize::MAX // sentinel: average all channels below - } else { - dst_ch % src_channels - }; - - let sample = if src_ch == usize::MAX { - let mut sum = 0.0; - for ch in 0..src_channels { - let mut channel_samples = [0.0f32; KERNEL_SIZE]; - for (j, i) in (-(HALF_KERNEL as i32)..(HALF_KERNEL as i32)).enumerate() { - let idx = src_frame + i; - if idx >= 0 && (idx as usize) < audio_file.frames as usize { - channel_samples[j] = get_sample!(idx as usize, ch); - } - } - sum += windowed_sinc_interpolate(&channel_samples, frac); - } - sum / src_channels as f32 - } else { + // Sinc-interpolate a single source channel at the current position. + macro_rules! sinc_ch { + ($ch:expr) => {{ let mut channel_samples = [0.0f32; KERNEL_SIZE]; for (j, i) in (-(HALF_KERNEL as i32)..(HALF_KERNEL as i32)).enumerate() { let idx = src_frame + i; if idx >= 0 && (idx as usize) < audio_file.frames as usize { - channel_samples[j] = get_sample!(idx as usize, src_ch); + channel_samples[j] = get_sample!(idx as usize, $ch); } } windowed_sinc_interpolate(&channel_samples, frac) + }}; + } + + for dst_ch in 0..dst_channels { + let sample = if let Some(ref mat) = downmix { + // Surround → stereo: interpolate each contributing channel. + let mut s = 0.0f32; + for (ch, &c) in mat[dst_ch].iter().enumerate() { + if c != 0.0 { + s += c * sinc_ch!(ch); + } + } + s + } else if dst_channels == 1 { + let mut sum = 0.0; + for ch in 0..src_channels { + sum += sinc_ch!(ch); + } + sum / src_channels as f32 + } else { + let src_ch = if src_channels == dst_channels { + dst_ch + } else if src_channels == 1 { + 0 + } else { + dst_ch % src_channels + }; + sinc_ch!(src_ch) }; output[output_frame * dst_channels + dst_ch] += sample * gain; @@ -841,6 +905,13 @@ pub struct AudioPoolEntry { /// (in) and load layer (out); `None` everywhere else. #[serde(skip)] pub waveform_blob: Option>, + /// This entry is a video container's audio track (`relative_path` points at the + /// video file). It is always stored as a path reference (never packed/embedded + /// — the `VideoClip` already references the file) and reloaded by re-probing + /// the video via FFmpeg, so multichannel (5.1/7.1) audio survives the round-trip + /// (Symphonia reconstitution would otherwise collapse it). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub is_video_audio: bool, } impl AudioClipPool { @@ -855,12 +926,40 @@ impl AudioClipPool { let mut entries = Vec::new(); for (index, file) in self.files.iter().enumerate() { + // Video's audio track: reference the video file (it's also referenced + // by the VideoClip) and re-probe it via FFmpeg on load. Never pack or + // embed it as audio media — that both wastes space and loses the 5.1+ + // layout when Symphonia later decodes it. + if matches!(file.storage, AudioStorage::VideoAudio { .. }) { + let relative_path = pathdiff::diff_paths(&file.path, project_dir) + .map(|r| r.to_string_lossy().to_string()) + .or_else(|| Some(file.path.to_string_lossy().to_string())); + entries.push(AudioPoolEntry { + pool_index: index, + is_video_audio: true, + waveform_blob: None, + name: file + .path + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| format!("file_{}", index)), + relative_path, + duration: file.duration_seconds(), + sample_rate: file.sample_rate, + channels: file.channels, + embedded_data: None, + media_id: None, + }); + continue; + } + // Packed-in-container streaming entry: its bytes already live in the // `.beam` media table (kept in place across re-saves). Emit just the // media id — no path, no embedded bytes, nothing to decode. if let Some(media_id) = &file.packed_media_id { entries.push(AudioPoolEntry { pool_index: index, + is_video_audio: false, waveform_blob: None, name: file .path @@ -907,6 +1006,7 @@ impl AudioClipPool { let entry = AudioPoolEntry { pool_index: index, + is_video_audio: false, waveform_blob: None, name: file_path .file_name() @@ -1056,7 +1156,45 @@ impl AudioClipPool { let entry_start = std::time::Instant::now(); eprintln!("📊 [LOAD_SERIALIZED] Processing entry {}/{}: '{}'", i + 1, entries.len(), entry.name); - let success = if entry.media_id.is_some() && entry.embedded_data.is_none() { + let success = if entry.is_video_audio { + // Re-probe the video's audio track via FFmpeg → a streaming + // VideoAudio entry (keeps full 5.1/7.1; no decode-to-RAM). + match entry.relative_path.as_ref() { + Some(rel) => { + let full = if std::path::Path::new(rel).is_absolute() { + PathBuf::from(rel) + } else { + project_dir.join(rel) + }; + if full.exists() { + match crate::audio::disk_reader::VideoAudioReader::open(&full) { + Ok(reader) => { + let file = AudioFile::from_video_audio( + full, + reader.channels(), + reader.sample_rate(), + reader.total_frames(), + ); + if entry.pool_index < self.files.len() { + self.files[entry.pool_index] = file; + true + } else { + false + } + } + Err(e) => { + eprintln!("[AudioPool] Failed to reopen video audio {:?}: {}", full, e); + false + } + } + } else { + eprintln!("[AudioPool] Video file not found for audio: {:?}", full); + false + } + } + None => false, + } + } else if entry.media_id.is_some() && entry.embedded_data.is_none() { // Packed-in-container streaming entry: build a Compressed placeholder // backed by the host blob factory (opened at clip-activation time). // No decode here — playback streams through the disk reader. diff --git a/lightningbeam-ui/lightningbeam-core/src/file_io.rs b/lightningbeam-ui/lightningbeam-core/src/file_io.rs index 6d08d93..10ce29d 100644 --- a/lightningbeam-ui/lightningbeam-core/src/file_io.rs +++ b/lightningbeam-ui/lightningbeam-core/src/file_io.rs @@ -331,10 +331,13 @@ pub fn save_beam( .and_then(|x| x.to_str()) .unwrap_or("bin") .to_lowercase(); - // Large files honor the user's pack-vs-reference choice (`Ask` == - // reference); smaller files are always packed. - let reference_it = size >= LARGE_MEDIA_THRESHOLD - && _settings.large_media_mode != LargeMediaMode::Pack; + // Video-audio entries are always referenced (the video is already + // referenced by its VideoClip; reloaded by re-probing via FFmpeg). + // Otherwise large files honor the user's pack-vs-reference choice + // (`Ask` == reference); smaller files are always packed. + let reference_it = entry.is_video_audio + || (size >= LARGE_MEDIA_THRESHOLD + && _settings.large_media_mode != LargeMediaMode::Pack); if reference_it { referenced = Some(rel.clone()); } else { diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index 674495a..56a44a3 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -423,11 +423,25 @@ pub fn render_layer_isolated( * Affine::scale_non_uniform(scale_x, scale_y) * skew_transform; + // The decoded frame is scaled down to fit the document (decoder caps + // at the canvas size), so its pixel size is smaller than the clip's + // native dimensions. The instance is blitted treating the texture as + // `frame.width × frame.height`, while `clip_transform` is expressed in + // the clip's native space — so scale frame-px → clip-native-px first, + // else the frame renders small in a corner with its edges streaked. + let frame_to_clip = if frame.width > 0 && frame.height > 0 { + Affine::scale_non_uniform( + video_clip.width / frame.width as f64, + video_clip.height / frame.height as f64, + ) + } else { + Affine::IDENTITY + }; instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), width: frame.width, height: frame.height, - transform: base_transform * clip_transform, + transform: base_transform * clip_transform * frame_to_clip, opacity: (layer_opacity * inst_opacity) as f32, }); } @@ -1028,12 +1042,26 @@ fn render_video_layer( // Create rectangle path for the video frame let video_rect = Rect::new(0.0, 0.0, video_clip.width, video_clip.height); + // The decoded frame is scaled down to fit the document (the decoder caps at + // the canvas size to bound memory), so its pixel dimensions are smaller than + // the clip's native display size. Scale the image brush from frame-pixel + // space to the clip rect; without this the image is drawn 1:1 in a corner + // and its edge pixels pad the rest (small frame with "stretched corners"). + let brush_transform = if frame.width > 0 && frame.height > 0 { + Affine::scale_non_uniform( + video_clip.width / frame.width as f64, + video_clip.height / frame.height as f64, + ) + } else { + Affine::IDENTITY + }; + // Render video frame as image fill scene.fill( Fill::NonZero, instance_transform, &image_with_alpha, - None, + Some(brush_transform), &video_rect, ); clip_rendered = true;