skyler
/
Lightningbeam
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Downmix surround to stereo + reload video audio via FFmpeg 

Surround → stereo downmix:
- render_from_file folds multichannel sources (5.1/7.1/…) down to stereo with
  proper coefficients (full level for the matching front channel, 1/√2 for centre
  + each surround, LFE dropped), normalized per row to avoid clipping (matching
  ffmpeg's default). Applied uniformly to both the direct-copy and sinc-resample
  paths and to every storage type (PCM, compressed, video audio), only when
  dst==2 && src>2; unknown layouts fall back to front L/R. Previously it just took
  FL/FR, dropping centre dialog + surrounds.

Proper video-audio reload:
- A video's audio track is now stored as a path reference to the video (never
  packed/embedded as audio media) and re-probed via FFmpeg on load into a
  streaming VideoAudio entry, so multichannel audio survives reload (the old
  Symphonia reconstitution collapsed it, breaking the downmix). Driven by a new
  AudioPoolEntry.is_video_audio flag across serialize / save_beam / load. Also
  removes the decode-whole-video-to-RAM + temp-file path on load.

Fix video scaling:
- Any video with dimensions larger than the stage was being scaled down into the corner incorrectly; we now bake the frame-clip scale into the instance transform.
This commit is contained in:
Skyler Lehmkuhl 2026-06-17 17:58:54 -04:00
parent 097345be76
commit 4ad95e6755
3 changed files with 202 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
daw-backend/src/audio/pool.rs
View File

@ -4,6 +4,46 @@ use std::f32::consts::PI;
use serde::{Deserialize, Serialize};
use crate::time::Seconds;
/// Per-output-channel mix coefficients to fold a multichannel source down to
/// stereo, indexed `[out_channel(0=L,1=R)][src_channel]`.
///
/// Assumes the conventional interleave order for each channel count (FL, FR, FC,
/// LFE, BL, BR, SL, SR …). Uses standard ITU/AC-3-style coefficients: full level
/// for the matching front channel, `1/√2` (≈ −3 dB) for centre and each surround,
/// LFE dropped. Each row is then normalized so its absolute-coefficient sum ≤ 1,
/// which prevents clipping (matching FFmpeg's default `normalize` behaviour).
///
/// Returns `None` for layouts we don't special-case (caller falls back to taking
/// the front L/R pair).
fn stereo_downmix_matrix(src_channels: usize) -> Option<[Vec<f32>; 2]> {
const C: f32 = std::f32::consts::FRAC_1_SQRT_2; // ≈ 0.7071
// (L row, R row); each entry is the gain applied to that source channel.
let (l, r): (Vec<f32>, Vec<f32>) = match src_channels {
3 => (vec![1.0, 0.0, C], vec![0.0, 1.0, C]), // FL FR FC
4 => (vec![1.0, 0.0, C, 0.0], vec![0.0, 1.0, 0.0, C]), // quad: FL FR BL BR
5 => (vec![1.0, 0.0, C, C, 0.0], vec![0.0, 1.0, C, 0.0, C]), // FL FR FC BL BR
// 5.1: FL FR FC LFE BL BR (LFE dropped)
6 => (vec![1.0, 0.0, C, 0.0, C, 0.0], vec![0.0, 1.0, C, 0.0, 0.0, C]),
// 6.1: FL FR FC LFE BC SL SR (BC → both)
7 => (vec![1.0, 0.0, C, 0.0, C, C, 0.0], vec![0.0, 1.0, C, 0.0, C, 0.0, C]),
// 7.1: FL FR FC LFE BL BR SL SR
8 => (
vec![1.0, 0.0, C, 0.0, C, 0.0, C, 0.0],
vec![0.0, 1.0, C, 0.0, 0.0, C, 0.0, C],
),
_ => return None,
};
let normalize = |row: Vec<f32>| -> Vec<f32> {
let sum: f32 = row.iter().map(|c| c.abs()).sum();
if sum > 1.0 {
row.into_iter().map(|c| c / sum).collect()
} else {
row
}
};
Some([normalize(l), normalize(r)])
}
/// Windowed sinc interpolation for high-quality time stretching
/// This is stateless and can handle arbitrary fractional positions
#[inline]
@ -580,6 +620,15 @@ impl AudioClipPool {
let dst_channels = engine_channels as usize;
let output_frames = output.len() / dst_channels;
// Fold a multichannel source (5.1, 7.1, …) down to stereo with proper
// coefficients (centre + surrounds mixed in, LFE dropped) instead of just
// taking the front L/R pair. `None` ⇒ no downmix needed / unknown layout.
let downmix = if dst_channels == 2 && src_channels > 2 {
stereo_downmix_matrix(src_channels)
} else {
None
};
let src_start_position = start_time_seconds * audio_file.sample_rate as f64;
// Tell the disk reader where we're reading so it buffers the right region.
@ -625,6 +674,15 @@ impl AudioClipPool {
sum += get_sample!(sf, src_ch);
}
sum / src_channels as f32
} else if let Some(ref mat) = downmix {
// Surround → stereo with proper coefficients.
let mut s = 0.0f32;
for (src_ch, &c) in mat[dst_ch].iter().enumerate() {
if c != 0.0 {
s += c * get_sample!(sf, src_ch);
}
}
s
} else {
get_sample!(sf, dst_ch % src_channels)
};
@ -649,39 +707,45 @@ impl AudioClipPool {
break;
}
for dst_ch in 0..dst_channels {
let src_ch = if src_channels == dst_channels {
dst_ch
} else if src_channels == 1 {
0
} else if dst_channels == 1 {
usize::MAX // sentinel: average all channels below
} else {
dst_ch % src_channels
};
let sample = if src_ch == usize::MAX {
let mut sum = 0.0;
for ch in 0..src_channels {
let mut channel_samples = [0.0f32; KERNEL_SIZE];
for (j, i) in (-(HALF_KERNEL as i32)..(HALF_KERNEL as i32)).enumerate() {
let idx = src_frame + i;
if idx >= 0 && (idx as usize) < audio_file.frames as usize {
channel_samples[j] = get_sample!(idx as usize, ch);
}
}
sum += windowed_sinc_interpolate(&channel_samples, frac);
}
sum / src_channels as f32
} else {
// Sinc-interpolate a single source channel at the current position.
macro_rules! sinc_ch {
($ch:expr) => {{
let mut channel_samples = [0.0f32; KERNEL_SIZE];
for (j, i) in (-(HALF_KERNEL as i32)..(HALF_KERNEL as i32)).enumerate() {
let idx = src_frame + i;
if idx >= 0 && (idx as usize) < audio_file.frames as usize {
channel_samples[j] = get_sample!(idx as usize, src_ch);
channel_samples[j] = get_sample!(idx as usize, $ch);
}
}
windowed_sinc_interpolate(&channel_samples, frac)
}};
}
for dst_ch in 0..dst_channels {
let sample = if let Some(ref mat) = downmix {
// Surround → stereo: interpolate each contributing channel.
let mut s = 0.0f32;
for (ch, &c) in mat[dst_ch].iter().enumerate() {
if c != 0.0 {
s += c * sinc_ch!(ch);
}
}
s
} else if dst_channels == 1 {
let mut sum = 0.0;
for ch in 0..src_channels {
sum += sinc_ch!(ch);
}
sum / src_channels as f32
} else {
let src_ch = if src_channels == dst_channels {
dst_ch
} else if src_channels == 1 {
0
} else {
dst_ch % src_channels
};
sinc_ch!(src_ch)
};
output[output_frame * dst_channels + dst_ch] += sample * gain;
@ -841,6 +905,13 @@ pub struct AudioPoolEntry {
/// (in) and load layer (out); `None` everywhere else.
#[serde(skip)]
pub waveform_blob: Option<Vec<u8>>,
/// This entry is a video container's audio track (`relative_path` points at the
/// video file). It is always stored as a path reference (never packed/embedded
/// — the `VideoClip` already references the file) and reloaded by re-probing
/// the video via FFmpeg, so multichannel (5.1/7.1) audio survives the round-trip
/// (Symphonia reconstitution would otherwise collapse it).
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub is_video_audio: bool,
}
impl AudioClipPool {
@ -855,12 +926,40 @@ impl AudioClipPool {
let mut entries = Vec::new();
for (index, file) in self.files.iter().enumerate() {
// Video's audio track: reference the video file (it's also referenced
// by the VideoClip) and re-probe it via FFmpeg on load. Never pack or
// embed it as audio media — that both wastes space and loses the 5.1+
// layout when Symphonia later decodes it.
if matches!(file.storage, AudioStorage::VideoAudio { .. }) {
let relative_path = pathdiff::diff_paths(&file.path, project_dir)
.map(|r| r.to_string_lossy().to_string())
.or_else(|| Some(file.path.to_string_lossy().to_string()));
entries.push(AudioPoolEntry {
pool_index: index,
is_video_audio: true,
waveform_blob: None,
name: file
.path
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| format!("file_{}", index)),
relative_path,
duration: file.duration_seconds(),
sample_rate: file.sample_rate,
channels: file.channels,
embedded_data: None,
media_id: None,
});
continue;
}
// Packed-in-container streaming entry: its bytes already live in the
// `.beam` media table (kept in place across re-saves). Emit just the
// media id — no path, no embedded bytes, nothing to decode.
if let Some(media_id) = &file.packed_media_id {
entries.push(AudioPoolEntry {
pool_index: index,
is_video_audio: false,
waveform_blob: None,
name: file
.path
@ -907,6 +1006,7 @@ impl AudioClipPool {
let entry = AudioPoolEntry {
pool_index: index,
is_video_audio: false,
waveform_blob: None,
name: file_path
.file_name()
@ -1056,7 +1156,45 @@ impl AudioClipPool {
let entry_start = std::time::Instant::now();
eprintln!("📊 [LOAD_SERIALIZED] Processing entry {}/{}: '{}'", i + 1, entries.len(), entry.name);
let success = if entry.media_id.is_some() && entry.embedded_data.is_none() {
let success = if entry.is_video_audio {
// Re-probe the video's audio track via FFmpeg → a streaming
// VideoAudio entry (keeps full 5.1/7.1; no decode-to-RAM).
match entry.relative_path.as_ref() {
Some(rel) => {
let full = if std::path::Path::new(rel).is_absolute() {
PathBuf::from(rel)
} else {
project_dir.join(rel)
};
if full.exists() {
match crate::audio::disk_reader::VideoAudioReader::open(&full) {
Ok(reader) => {
let file = AudioFile::from_video_audio(
full,
reader.channels(),
reader.sample_rate(),
reader.total_frames(),
);
if entry.pool_index < self.files.len() {
self.files[entry.pool_index] = file;
true
} else {
false
}
}
Err(e) => {
eprintln!("[AudioPool] Failed to reopen video audio {:?}: {}", full, e);
false
}
}
} else {
eprintln!("[AudioPool] Video file not found for audio: {:?}", full);
false
}
}
None => false,
}
} else if entry.media_id.is_some() && entry.embedded_data.is_none() {
// Packed-in-container streaming entry: build a Compressed placeholder
// backed by the host blob factory (opened at clip-activation time).
// No decode here — playback streams through the disk reader.

11
lightningbeam-ui/lightningbeam-core/src/file_io.rs
View File

@ -331,10 +331,13 @@ pub fn save_beam(
.and_then(|x| x.to_str())
.unwrap_or("bin")
.to_lowercase();
// Large files honor the user's pack-vs-reference choice (`Ask` ==
// reference); smaller files are always packed.
let reference_it = size >= LARGE_MEDIA_THRESHOLD
&& _settings.large_media_mode != LargeMediaMode::Pack;
// Video-audio entries are always referenced (the video is already
// referenced by its VideoClip; reloaded by re-probing via FFmpeg).
// Otherwise large files honor the user's pack-vs-reference choice
// (`Ask` == reference); smaller files are always packed.
let reference_it = entry.is_video_audio
|| (size >= LARGE_MEDIA_THRESHOLD
&& _settings.large_media_mode != LargeMediaMode::Pack);
if reference_it {
referenced = Some(rel.clone());
} else {

32
lightningbeam-ui/lightningbeam-core/src/renderer.rs
View File

@ -423,11 +423,25 @@ pub fn render_layer_isolated(
* Affine::scale_non_uniform(scale_x, scale_y)
* skew_transform;
// The decoded frame is scaled down to fit the document (decoder caps
// at the canvas size), so its pixel size is smaller than the clip's
// native dimensions. The instance is blitted treating the texture as
// `frame.width × frame.height`, while `clip_transform` is expressed in
// the clip's native space — so scale frame-px → clip-native-px first,
// else the frame renders small in a corner with its edges streaked.
let frame_to_clip = if frame.width > 0 && frame.height > 0 {
Affine::scale_non_uniform(
video_clip.width / frame.width as f64,
video_clip.height / frame.height as f64,
)
} else {
Affine::IDENTITY
};
instances.push(VideoRenderInstance {
rgba_data: frame.rgba_data.clone(),
width: frame.width,
height: frame.height,
transform: base_transform * clip_transform,
transform: base_transform * clip_transform * frame_to_clip,
opacity: (layer_opacity * inst_opacity) as f32,
});
}
@ -1028,12 +1042,26 @@ fn render_video_layer(
// Create rectangle path for the video frame
let video_rect = Rect::new(0.0, 0.0, video_clip.width, video_clip.height);
// The decoded frame is scaled down to fit the document (the decoder caps at
// the canvas size to bound memory), so its pixel dimensions are smaller than
// the clip's native display size. Scale the image brush from frame-pixel
// space to the clip rect; without this the image is drawn 1:1 in a corner
// and its edge pixels pad the rest (small frame with "stretched corners").
let brush_transform = if frame.width > 0 && frame.height > 0 {
Affine::scale_non_uniform(
video_clip.width / frame.width as f64,
video_clip.height / frame.height as f64,
)
} else {
Affine::IDENTITY
};
// Render video frame as image fill
scene.fill(
Fill::NonZero,
instance_transform,
&image_with_alpha,
None,
Some(brush_transform),
&video_rect,
);
clip_rendered = true;