Downmix surround to stereo + reload video audio via FFmpeg

Surround → stereo downmix:
- render_from_file folds multichannel sources (5.1/7.1/…) down to stereo with
  proper coefficients (full level for the matching front channel, 1/√2 for centre
  + each surround, LFE dropped), normalized per row to avoid clipping (matching
  ffmpeg's default). Applied uniformly to both the direct-copy and sinc-resample
  paths and to every storage type (PCM, compressed, video audio), only when
  dst==2 && src>2; unknown layouts fall back to front L/R. Previously it just took
  FL/FR, dropping centre dialog + surrounds.

Proper video-audio reload:
- A video's audio track is now stored as a path reference to the video (never
  packed/embedded as audio media) and re-probed via FFmpeg on load into a
  streaming VideoAudio entry, so multichannel audio survives reload (the old
  Symphonia reconstitution collapsed it, breaking the downmix). Driven by a new
  AudioPoolEntry.is_video_audio flag across serialize / save_beam / load. Also
  removes the decode-whole-video-to-RAM + temp-file path on load.

Fix video scaling:
- Any video with dimensions larger than the stage was being scaled down into the corner incorrectly; we now bake the frame-clip scale into the instance transform.

daw-backend/src/audio/pool.rs

@@ -4,6 +4,46 @@ use std::f32::consts::PI;
 use serde::{Deserialize, Serialize};
 use crate::time::Seconds;
 
+/// Per-output-channel mix coefficients to fold a multichannel source down to
+/// stereo, indexed `[out_channel(0=L,1=R)][src_channel]`.
+///
+/// Assumes the conventional interleave order for each channel count (FL, FR, FC,
+/// LFE, BL, BR, SL, SR …). Uses standard ITU/AC-3-style coefficients: full level
+/// for the matching front channel, `1/√2` (≈ −3 dB) for centre and each surround,
+/// LFE dropped. Each row is then normalized so its absolute-coefficient sum ≤ 1,
+/// which prevents clipping (matching FFmpeg's default `normalize` behaviour).
+///
+/// Returns `None` for layouts we don't special-case (caller falls back to taking
+/// the front L/R pair).
+fn stereo_downmix_matrix(src_channels: usize) -> Option<[Vec<f32>; 2]> {
+    const C: f32 = std::f32::consts::FRAC_1_SQRT_2; // ≈ 0.7071
+    // (L row, R row); each entry is the gain applied to that source channel.
+    let (l, r): (Vec<f32>, Vec<f32>) = match src_channels {
+        3 => (vec![1.0, 0.0, C], vec![0.0, 1.0, C]), // FL FR FC
+        4 => (vec![1.0, 0.0, C, 0.0], vec![0.0, 1.0, 0.0, C]), // quad: FL FR BL BR
+        5 => (vec![1.0, 0.0, C, C, 0.0], vec![0.0, 1.0, C, 0.0, C]), // FL FR FC BL BR
+        // 5.1: FL FR FC LFE BL BR (LFE dropped)
+        6 => (vec![1.0, 0.0, C, 0.0, C, 0.0], vec![0.0, 1.0, C, 0.0, 0.0, C]),
+        // 6.1: FL FR FC LFE BC SL SR (BC → both)
+        7 => (vec![1.0, 0.0, C, 0.0, C, C, 0.0], vec![0.0, 1.0, C, 0.0, C, 0.0, C]),
+        // 7.1: FL FR FC LFE BL BR SL SR
+        8 => (
+            vec![1.0, 0.0, C, 0.0, C, 0.0, C, 0.0],
+            vec![0.0, 1.0, C, 0.0, 0.0, C, 0.0, C],
+        ),
+        _ => return None,
+    };
+    let normalize = |row: Vec<f32>| -> Vec<f32> {
+        let sum: f32 = row.iter().map(|c| c.abs()).sum();
+        if sum > 1.0 {
+            row.into_iter().map(|c| c / sum).collect()
+        } else {
+            row
+        }
+    };
+    Some([normalize(l), normalize(r)])
+}
+
 /// Windowed sinc interpolation for high-quality time stretching
 /// This is stateless and can handle arbitrary fractional positions
 #[inline]
@@ -580,6 +620,15 @@ impl AudioClipPool {
         let dst_channels = engine_channels as usize;
         let output_frames = output.len() / dst_channels;
 
+        // Fold a multichannel source (5.1, 7.1, …) down to stereo with proper
+        // coefficients (centre + surrounds mixed in, LFE dropped) instead of just
+        // taking the front L/R pair. `None` ⇒ no downmix needed / unknown layout.
+        let downmix = if dst_channels == 2 && src_channels > 2 {
+            stereo_downmix_matrix(src_channels)
+        } else {
+            None
+        };
+
         let src_start_position = start_time_seconds * audio_file.sample_rate as f64;
 
         // Tell the disk reader where we're reading so it buffers the right region.
@@ -625,6 +674,15 @@ impl AudioClipPool {
                             sum += get_sample!(sf, src_ch);
                         }
                         sum / src_channels as f32
+                    } else if let Some(ref mat) = downmix {
+                        // Surround → stereo with proper coefficients.
+                        let mut s = 0.0f32;
+                        for (src_ch, &c) in mat[dst_ch].iter().enumerate() {
+                            if c != 0.0 {
+                                s += c * get_sample!(sf, src_ch);
+                            }
+                        }
+                        s
                     } else {
                         get_sample!(sf, dst_ch % src_channels)
                     };
@@ -649,39 +707,45 @@ impl AudioClipPool {
                     break;
                 }
 
+                // Sinc-interpolate a single source channel at the current position.
+                macro_rules! sinc_ch {
+                    ($ch:expr) => {{
+                        let mut channel_samples = [0.0f32; KERNEL_SIZE];
+                        for (j, i) in (-(HALF_KERNEL as i32)..(HALF_KERNEL as i32)).enumerate() {
+                            let idx = src_frame + i;
+                            if idx >= 0 && (idx as usize) < audio_file.frames as usize {
+                                channel_samples[j] = get_sample!(idx as usize, $ch);
+                            }
+                        }
+                        windowed_sinc_interpolate(&channel_samples, frac)
+                    }};
+                }
+
                 for dst_ch in 0..dst_channels {
+                    let sample = if let Some(ref mat) = downmix {
+                        // Surround → stereo: interpolate each contributing channel.
+                        let mut s = 0.0f32;
+                        for (ch, &c) in mat[dst_ch].iter().enumerate() {
+                            if c != 0.0 {
+                                s += c * sinc_ch!(ch);
+                            }
+                        }
+                        s
+                    } else if dst_channels == 1 {
+                        let mut sum = 0.0;
+                        for ch in 0..src_channels {
+                            sum += sinc_ch!(ch);
+                        }
+                        sum / src_channels as f32
+                    } else {
                         let src_ch = if src_channels == dst_channels {
                             dst_ch
                         } else if src_channels == 1 {
                             0
-                    } else if dst_channels == 1 {
-                        usize::MAX // sentinel: average all channels below
                         } else {
                             dst_ch % src_channels
                         };
-
-                    let sample = if src_ch == usize::MAX {
-                        let mut sum = 0.0;
-                        for ch in 0..src_channels {
-                            let mut channel_samples = [0.0f32; KERNEL_SIZE];
-                            for (j, i) in (-(HALF_KERNEL as i32)..(HALF_KERNEL as i32)).enumerate() {
-                                let idx = src_frame + i;
-                                if idx >= 0 && (idx as usize) < audio_file.frames as usize {
-                                    channel_samples[j] = get_sample!(idx as usize, ch);
-                                }
-                            }
-                            sum += windowed_sinc_interpolate(&channel_samples, frac);
-                        }
-                        sum / src_channels as f32
-                    } else {
-                        let mut channel_samples = [0.0f32; KERNEL_SIZE];
-                        for (j, i) in (-(HALF_KERNEL as i32)..(HALF_KERNEL as i32)).enumerate() {
-                            let idx = src_frame + i;
-                            if idx >= 0 && (idx as usize) < audio_file.frames as usize {
-                                channel_samples[j] = get_sample!(idx as usize, src_ch);
-                            }
-                        }
-                        windowed_sinc_interpolate(&channel_samples, frac)
+                        sinc_ch!(src_ch)
                     };
 
                     output[output_frame * dst_channels + dst_ch] += sample * gain;
@@ -841,6 +905,13 @@ pub struct AudioPoolEntry {
     /// (in) and load layer (out); `None` everywhere else.
     #[serde(skip)]
     pub waveform_blob: Option<Vec<u8>>,
+    /// This entry is a video container's audio track (`relative_path` points at the
+    /// video file). It is always stored as a path reference (never packed/embedded
+    /// — the `VideoClip` already references the file) and reloaded by re-probing
+    /// the video via FFmpeg, so multichannel (5.1/7.1) audio survives the round-trip
+    /// (Symphonia reconstitution would otherwise collapse it).
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub is_video_audio: bool,
 }
 
 impl AudioClipPool {
@@ -855,12 +926,40 @@ impl AudioClipPool {
         let mut entries = Vec::new();
 
         for (index, file) in self.files.iter().enumerate() {
+            // Video's audio track: reference the video file (it's also referenced
+            // by the VideoClip) and re-probe it via FFmpeg on load. Never pack or
+            // embed it as audio media — that both wastes space and loses the 5.1+
+            // layout when Symphonia later decodes it.
+            if matches!(file.storage, AudioStorage::VideoAudio { .. }) {
+                let relative_path = pathdiff::diff_paths(&file.path, project_dir)
+                    .map(|r| r.to_string_lossy().to_string())
+                    .or_else(|| Some(file.path.to_string_lossy().to_string()));
+                entries.push(AudioPoolEntry {
+                    pool_index: index,
+                    is_video_audio: true,
+                    waveform_blob: None,
+                    name: file
+                        .path
+                        .file_name()
+                        .map(|n| n.to_string_lossy().to_string())
+                        .unwrap_or_else(|| format!("file_{}", index)),
+                    relative_path,
+                    duration: file.duration_seconds(),
+                    sample_rate: file.sample_rate,
+                    channels: file.channels,
+                    embedded_data: None,
+                    media_id: None,
+                });
+                continue;
+            }
+
             // Packed-in-container streaming entry: its bytes already live in the
             // `.beam` media table (kept in place across re-saves). Emit just the
             // media id — no path, no embedded bytes, nothing to decode.
             if let Some(media_id) = &file.packed_media_id {
                 entries.push(AudioPoolEntry {
                     pool_index: index,
+                    is_video_audio: false,
                     waveform_blob: None,
                     name: file
                         .path
@@ -907,6 +1006,7 @@ impl AudioClipPool {
 
             let entry = AudioPoolEntry {
                 pool_index: index,
+                is_video_audio: false,
                 waveform_blob: None,
                 name: file_path
                     .file_name()
@@ -1056,7 +1156,45 @@ impl AudioClipPool {
             let entry_start = std::time::Instant::now();
             eprintln!("📊 [LOAD_SERIALIZED] Processing entry {}/{}: '{}'", i + 1, entries.len(), entry.name);
 
-            let success = if entry.media_id.is_some() && entry.embedded_data.is_none() {
+            let success = if entry.is_video_audio {
+                // Re-probe the video's audio track via FFmpeg → a streaming
+                // VideoAudio entry (keeps full 5.1/7.1; no decode-to-RAM).
+                match entry.relative_path.as_ref() {
+                    Some(rel) => {
+                        let full = if std::path::Path::new(rel).is_absolute() {
+                            PathBuf::from(rel)
+                        } else {
+                            project_dir.join(rel)
+                        };
+                        if full.exists() {
+                            match crate::audio::disk_reader::VideoAudioReader::open(&full) {
+                                Ok(reader) => {
+                                    let file = AudioFile::from_video_audio(
+                                        full,
+                                        reader.channels(),
+                                        reader.sample_rate(),
+                                        reader.total_frames(),
+                                    );
+                                    if entry.pool_index < self.files.len() {
+                                        self.files[entry.pool_index] = file;
+                                        true
+                                    } else {
+                                        false
+                                    }
+                                }
+                                Err(e) => {
+                                    eprintln!("[AudioPool] Failed to reopen video audio {:?}: {}", full, e);
+                                    false
+                                }
+                            }
+                        } else {
+                            eprintln!("[AudioPool] Video file not found for audio: {:?}", full);
+                            false
+                        }
+                    }
+                    None => false,
+                }
+            } else if entry.media_id.is_some() && entry.embedded_data.is_none() {
                 // Packed-in-container streaming entry: build a Compressed placeholder
                 // backed by the host blob factory (opened at clip-activation time).
                 // No decode here — playback streams through the disk reader.

lightningbeam-ui/lightningbeam-core/src/file_io.rs

@@ -331,10 +331,13 @@ pub fn save_beam(
                     .and_then(|x| x.to_str())
                     .unwrap_or("bin")
                     .to_lowercase();
-                // Large files honor the user's pack-vs-reference choice (`Ask` ==
-                // reference); smaller files are always packed.
-                let reference_it = size >= LARGE_MEDIA_THRESHOLD
-                    && _settings.large_media_mode != LargeMediaMode::Pack;
+                // Video-audio entries are always referenced (the video is already
+                // referenced by its VideoClip; reloaded by re-probing via FFmpeg).
+                // Otherwise large files honor the user's pack-vs-reference choice
+                // (`Ask` == reference); smaller files are always packed.
+                let reference_it = entry.is_video_audio
+                    || (size >= LARGE_MEDIA_THRESHOLD
+                        && _settings.large_media_mode != LargeMediaMode::Pack);
                 if reference_it {
                     referenced = Some(rel.clone());
                 } else {

lightningbeam-ui/lightningbeam-core/src/renderer.rs

@@ -423,11 +423,25 @@ pub fn render_layer_isolated(
                     * Affine::scale_non_uniform(scale_x, scale_y)
                     * skew_transform;
 
+                // The decoded frame is scaled down to fit the document (decoder caps
+                // at the canvas size), so its pixel size is smaller than the clip's
+                // native dimensions. The instance is blitted treating the texture as
+                // `frame.width × frame.height`, while `clip_transform` is expressed in
+                // the clip's native space — so scale frame-px → clip-native-px first,
+                // else the frame renders small in a corner with its edges streaked.
+                let frame_to_clip = if frame.width > 0 && frame.height > 0 {
+                    Affine::scale_non_uniform(
+                        video_clip.width / frame.width as f64,
+                        video_clip.height / frame.height as f64,
+                    )
+                } else {
+                    Affine::IDENTITY
+                };
                 instances.push(VideoRenderInstance {
                     rgba_data: frame.rgba_data.clone(),
                     width: frame.width,
                     height: frame.height,
-                    transform: base_transform * clip_transform,
+                    transform: base_transform * clip_transform * frame_to_clip,
                     opacity: (layer_opacity * inst_opacity) as f32,
                 });
             }
@@ -1028,12 +1042,26 @@ fn render_video_layer(
         // Create rectangle path for the video frame
         let video_rect = Rect::new(0.0, 0.0, video_clip.width, video_clip.height);
 
+        // The decoded frame is scaled down to fit the document (the decoder caps at
+        // the canvas size to bound memory), so its pixel dimensions are smaller than
+        // the clip's native display size. Scale the image brush from frame-pixel
+        // space to the clip rect; without this the image is drawn 1:1 in a corner
+        // and its edge pixels pad the rest (small frame with "stretched corners").
+        let brush_transform = if frame.width > 0 && frame.height > 0 {
+            Affine::scale_non_uniform(
+                video_clip.width / frame.width as f64,
+                video_clip.height / frame.height as f64,
+            )
+        } else {
+            Affine::IDENTITY
+        };
+
         // Render video frame as image fill
         scene.fill(
             Fill::NonZero,
             instance_transform,
             &image_with_alpha,
-            None,
+            Some(brush_transform),
             &video_rect,
         );
         clip_rendered = true;