From ce151ffd61b804a0c8e7d2a412ff2be49ba0f8fe Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Mon, 22 Jun 2026 09:16:53 -0400 Subject: [PATCH] Pack video into .beam and stream frames + audio from the blob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Video was the only media type always kept external (VideoClip.file_path), so a project with video wasn't self-contained. Now video packs into the SQLite container under the same large-media policy as audio (pack < 2 GB unless the user chose Reference), and both the frames and the embedded audio track decode by streaming directly from the blob — no temp files. - New crate ffmpeg-blob-io: an AVIOContext-over-Read+Seek shim (BlobInput) that lets ffmpeg demux from an arbitrary byte source. Isolates all the unsafe FFI + ffmpeg ABI coupling (version-pinned =8.0.0/=8.0.1). Manual Drop teardown order; AVSEEK_SIZE restores the read position (FFmpeg assumes a size query doesn't move it — required for MP4 moov-at-end). - Schema/save/load: VideoClip.media_id; save_beam packs/references video as MediaKind::Video (keyed by clip id); load resolves packed vs referenced and reports missing sources. A packed clip points its linked video-audio pool entry's media_id at the video row so the audio streams from the same blob. - Frames: video.rs VideoSource{Path,Packed} threaded through new/seek/scan/ probe/thumbnails (a fresh BlobReader per open); editor builds the source from current_file_path (now set before register_loaded_videos). - Audio: VideoAudioReader::open_source via BlobInput; the disk_reader StreamSource block on packed video-audio is removed; the engine's existing factory activation routes it unchanged. Tests: ffmpeg-blob-io AVIO unit tests (WAV via Cursor, seek, open/drop loop); core packed_video_stream (blob->AVIO->Input) and beam_archive video round-trip; daw-backend open_source test (compiles; links/runs only off-container). Runtime-verified: a packed video plays frames + audio after the source file is removed. --- BEAM_FILE_FORMAT.md | 27 +- STREAMING_TO_DISK_PLAN.md | 16 +- daw-backend/Cargo.toml | 2 + daw-backend/src/audio/disk_reader.rs | 57 +++- daw-backend/src/audio/engine.rs | 1 + daw-backend/src/audio/pool.rs | 30 +- daw-backend/tests/video_audio_stream.rs | 66 ++++ ffmpeg-blob-io/Cargo.lock | 287 ++++++++++++++++++ ffmpeg-blob-io/Cargo.toml | 22 ++ ffmpeg-blob-io/src/lib.rs | 240 +++++++++++++++ ffmpeg-blob-io/tests/blob_input.rs | 100 ++++++ lightningbeam-ui/Cargo.lock | 13 +- .../lightningbeam-core/Cargo.toml | 5 + .../lightningbeam-core/src/clip.rs | 8 + .../lightningbeam-core/src/file_io.rs | 99 +++++- .../lightningbeam-core/src/video.rs | 146 ++++++--- .../lightningbeam-core/tests/beam_archive.rs | 48 +++ .../tests/packed_video_stream.rs | 81 +++++ .../lightningbeam-editor/src/main.rs | 69 +++-- 19 files changed, 1224 insertions(+), 93 deletions(-) create mode 100644 ffmpeg-blob-io/Cargo.lock create mode 100644 ffmpeg-blob-io/Cargo.toml create mode 100644 ffmpeg-blob-io/src/lib.rs create mode 100644 ffmpeg-blob-io/tests/blob_input.rs create mode 100644 lightningbeam-ui/lightningbeam-core/tests/packed_video_stream.rs diff --git a/BEAM_FILE_FORMAT.md b/BEAM_FILE_FORMAT.md index 0fbdb6c..1d6b417 100644 --- a/BEAM_FILE_FORMAT.md +++ b/BEAM_FILE_FORMAT.md @@ -144,7 +144,7 @@ re-saves. | Value | Kind | `codec` examples | Notes | |-------|---------------|------------------|-------| | `0` | `Audio` | `flac`, `mp3`, `wav`, `ogg`, `opus`, `aac`, `m4a`, `alac`, `caf`, `aiff` | Source audio for a pool entry. | -| `1` | `Video` | — | **Reserved/unused.** The current writer never emits video rows; video bytes live in an external file referenced by `VideoClip.file_path`. | +| `1` | `Video` | `mp4`, `mov`, … | A video clip's container bytes, keyed by the clip id. Packed under the large-media policy (referenced above 2 GiB unless `Pack`). Frames **and** the embedded audio stream are decoded directly from this blob. | | `2` | `Raster` | `png` | Full-resolution pixels of a raster keyframe (PNG-encoded RGBA). | | `3` | `ImageAsset` | `png`, `jpg`, … | An imported image asset's original bytes. | | `4` | `Waveform` | `lbwf` | Precomputed waveform LOD pyramid for an audio item. Opaque blob owned by `daw_backend::audio::waveform_pyramid`. | @@ -245,8 +245,9 @@ Collections that carry media linkage are **bold**: - `root: GraphicsObject` — the layer tree; raster keyframes live here and inside nested group/clip layers - **`image_assets: map`** — keyed by asset UUID (= the media id) -- **`video_clips: map`** — `VideoClip.file_path` is the external - video file; thumbnails are derived media (§7) +- **`video_clips: map`** — when `VideoClip.media_id` is set the + video is packed (a `Video` media row at the clip id); otherwise `file_path` points + at the external video file. Thumbnails are derived media (§7) - `vector_clips`, `audio_clips`, `instance_groups`, `effect_definitions`, `script_definitions`, the various `*_folders` asset trees — structural, no direct media-row linkage @@ -286,7 +287,7 @@ not serialized; they are rebuilt on load. | `sample_rate` | `u32` | Authoritative sample rate. | | `channels` | `u32` | Channel count. | | `duration` | `f64` | Seconds. | -| `is_video_audio`| `bool` | If set, the entry is the audio track of a video; always stored **referenced**. `#[serde(default, skip_if=false)]`. | +| `is_video_audio`| `bool` | If set, the entry is the audio track of a video. Referenced when the video is external; when the video is **packed**, `media_id` points at the video's `Video` row and the audio streams from that same blob. `#[serde(default, skip_if=false)]`. | | `waveform_blob` | (transient) | `#[serde(skip)]`; carries waveform bytes in memory only. | ### 8.6 Media linkage summary @@ -299,7 +300,8 @@ not serialized; they are rebuilt on load. | Raster (full) | `RasterKeyframe.id` | that UUID | | Raster proxy | derived from `RasterKeyframe.id` | §7 | | Image asset | `Document.image_assets` key / `ImageAsset.id` | that UUID | -| Video bytes | `VideoClip.file_path` | — (always external) | +| Video (packed) | `VideoClip.media_id` (= clip id) | that UUID | +| Video (referenced) | `VideoClip.file_path` | — (external) | | Video thumbnails | derived from `video_clips` key | §7 | | Waveform | derived from `AudioPoolEntry.pool_index` | §7 | @@ -323,11 +325,14 @@ save inside **one** SQLite transaction. 1. Audio pool entries → `Audio` (+ `Waveform`) media rows. 2. Raster keyframes → `Raster` (+ `RasterProxy`) media rows. 3. Video thumbnail packs → `Thumbnail` media rows. -4. Image assets → `ImageAsset` media rows. -5. **Garbage-collect**: delete every `media` row (and its chunks) whose id is **not** - in the set of live ids accumulated in steps 1–4. -6. Write `project.json` and the `meta` keys (`version`, `created`, `modified`). -7. Commit; then (create path only) rename the temp file over the target. +4. Video clips → `Video` media rows (packed/referenced per the large-media policy); + when a clip is packed, its linked video-audio pool entry's `media_id` is set to + the clip's `Video` row so the audio streams from the same blob. +5. Image assets → `ImageAsset` media rows. +6. **Garbage-collect**: delete every `media` row (and its chunks) whose id is **not** + in the set of live ids accumulated in steps 1–5. +7. Write `project.json` and the `meta` keys (`version`, `created`, `modified`). +8. Commit; then (create path only) rename the temp file over the target. ### 9.3 Packed vs. referenced decision (audio) @@ -451,8 +456,6 @@ A conforming **writer** MUST: These reflect the current reference implementation and are flagged for implementers and future spec revisions: -- `MediaKind::Video` (`1`) is defined but never written; video media is always an - external file via `VideoClip.file_path`. - The project `version` check is exact-match with no compatibility window; a future revision should define semantic-version tolerance before bumping it. diff --git a/STREAMING_TO_DISK_PLAN.md b/STREAMING_TO_DISK_PLAN.md index 2a43ebb..abd9921 100644 --- a/STREAMING_TO_DISK_PLAN.md +++ b/STREAMING_TO_DISK_PLAN.md @@ -686,11 +686,17 @@ Phase 1a and Phase 2 can ship now; everything else waits on Phase 0 (the contain `load_file_into_pool`'s full decode with the `do_import_audio` branching (PCM → mmap, compressed → `from_compressed` placeholder). Higher risk (touches the working referenced path); packed covers the common <2GB case first. - - [ ] **Deferred (follow-up): packed video streaming.** Let small videos be packed into the `.beam` - (a `MediaKind::Video` blob, `VideoClip` referencing it by id) and stream **both frames and audio** - from the DB blob via FFmpeg. ffmpeg-next has no custom-I/O wrapper, so this needs an - `AVIOContext`-over-`BlobReader` shim via raw FFI. **Decision (user):** that FFI wrapper lives in - its **own crate, version-pinned to the ffmpeg version**, isolating the unsafe + the ABI coupling. + - [x] **DONE: packed video streaming.** Small videos pack into the `.beam` + (a `MediaKind::Video` blob at the clip id, `VideoClip.media_id` referencing it) and stream **both + frames and audio** from the DB blob via FFmpeg. The `AVIOContext`-over-`Read+Seek` shim lives in + the new `ffmpeg-blob-io` crate (`BlobInput`, version-pinned `=8.0.0`/`=8.0.1`), isolating the + unsafe + ABI coupling. Frames: `video.rs` `VideoSource{Path,Packed}` opens a fresh `BlobReader` + per decoder/seek/scan. Audio: `VideoAudioReader::open_source` over the same blob (the + `disk_reader.rs` `StreamSource` blocker is removed); save points the linked video-audio pool + entry's `media_id` at the video row so it streams from the same blob. Tests: ffmpeg-blob-io AVIO + unit tests (WAV via Cursor + seek + open/drop loop), core `packed_video_stream` (blob→AVIO→Input), + `beam_archive` packed-video round-trip, daw-backend `open_source` (compiles; can't link in the + container — user runtime-verifies actual A/V playback). - [~] Phase 1c — video embedded-audio track ← **stopgap shipped; proper design next** - [x] Stopgap: `extract_audio_from_video_to_wav` streams to a temp WAV → `import_audio_sync` (mmap). Fixed the ~2.8GB-`Vec` OOM. But writes the whole WAV to `/tmp` (fills diff --git a/daw-backend/Cargo.toml b/daw-backend/Cargo.toml index 1878591..8aeebd1 100644 --- a/daw-backend/Cargo.toml +++ b/daw-backend/Cargo.toml @@ -23,6 +23,8 @@ memmap2 = "0.9" # Audio export hound = "3.5" ffmpeg-next = "8.0" # For MP3/AAC encoding +# AVIO-over-Read+Seek shim: stream a packed video's audio track from the SQLite blob. +ffmpeg-blob-io = { path = "../ffmpeg-blob-io" } # Node-based audio graph dependencies dasp_graph = "0.11" diff --git a/daw-backend/src/audio/disk_reader.rs b/daw-backend/src/audio/disk_reader.rs index f810667..6b294d6 100644 --- a/daw-backend/src/audio/disk_reader.rs +++ b/daw-backend/src/audio/disk_reader.rs @@ -571,8 +571,37 @@ impl CompressedReader { /// /// Public (vs. the private `CompressedReader`) only so integration tests can /// exercise it directly; treat it as crate-internal. +/// Adapts a host `MediaByteSource` (Read+Seek+Send+Sync) to the `ffmpeg-blob-io` +/// `BlobSource` (Read+Seek+Send) so a packed video can be demuxed from a blob. +struct ByteSourceAdapter(Box); +impl std::io::Read for ByteSourceAdapter { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.read(buf) + } +} +impl std::io::Seek for ByteSourceAdapter { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.0.seek(pos) + } +} + +/// A demuxer input for video-audio: a video file path, or a container blob streamed +/// via the AVIO shim. Both expose the same ffmpeg `Input`. +enum AudioInput { + Path(ffmpeg_next::format::context::Input), + Blob(ffmpeg_blob_io::BlobInput), +} +impl AudioInput { + fn get(&mut self) -> &mut ffmpeg_next::format::context::Input { + match self { + AudioInput::Path(i) => i, + AudioInput::Blob(b) => b.input_mut(), + } + } +} + pub struct VideoAudioReader { - input: ffmpeg_next::format::context::Input, + input: AudioInput, decoder: ffmpeg_next::decoder::Audio, /// Built lazily from the first decoded frame's format/layout → interleaved f32. resampler: Option, @@ -605,15 +634,30 @@ impl VideoAudioReader { self.total_frames } + /// Open the audio track of a video **file**. pub fn open(path: &Path) -> Result { ffmpeg_next::init().map_err(|e| e.to_string())?; let input = ffmpeg_next::format::input(&path) .map_err(|e| format!("Failed to open media: {}", e))?; + Self::from_input(AudioInput::Path(input)) + } + /// Open the audio track of a video streamed from a **byte source** (a packed + /// `.beam` video blob), via the `ffmpeg-blob-io` AVIO shim. `ext` is a container + /// hint (e.g. `"mp4"`). + pub fn open_source(src: Box, ext: Option<&str>) -> Result { + ffmpeg_next::init().map_err(|e| e.to_string())?; + let blob = ffmpeg_blob_io::BlobInput::open(Box::new(ByteSourceAdapter(src)), ext) + .map_err(|e| format!("Failed to open packed video audio: {}", e))?; + Self::from_input(AudioInput::Blob(blob)) + } + + fn from_input(mut input: AudioInput) -> Result { // Pull stream scalars + build the decoder inside a scope so the stream // borrow of `input` ends before we use `input` again. let (stream_index, time_base, stream_duration, decoder) = { let stream = input + .get() .streams() .best(ffmpeg_next::media::Type::Audio) .ok_or_else(|| "No audio stream found".to_string())?; @@ -631,8 +675,8 @@ impl VideoAudioReader { let duration_secs = if stream_duration > 0 { stream_duration as f64 * time_base - } else if input.duration() > 0 { - input.duration() as f64 / f64::from(ffmpeg_next::ffi::AV_TIME_BASE) + } else if input.get().duration() > 0 { + input.get().duration() as f64 / f64::from(ffmpeg_next::ffi::AV_TIME_BASE) } else { 0.0 }; @@ -659,6 +703,7 @@ impl VideoAudioReader { // Seek to at-or-before the target (max_ts = ts_av) so we can decode // forward to it exactly. ffmpeg-next's `seek` wants a bounded range. self.input + .get() .seek(ts_av, 0..(ts_av + 1)) .map_err(|e| format!("Seek failed: {}", e))?; self.decoder.flush(); @@ -684,7 +729,7 @@ impl VideoAudioReader { } // Read one packet (owned), releasing the `input` borrow before decoding. - let packet = self.input.packets().next().map(|(_, p)| p); + let packet = self.input.get().packets().next().map(|(_, p)| p); match packet { Some(packet) => { if packet.stream() == self.stream_index { @@ -827,8 +872,8 @@ impl StreamSource { (SourceKind::VideoAudio, StreamOpen::Path(p)) => { Ok(StreamSource::Video(VideoAudioReader::open(&p)?)) } - (SourceKind::VideoAudio, StreamOpen::Source { .. }) => { - Err("VideoAudio cannot be opened from a packed byte source".to_string()) + (SourceKind::VideoAudio, StreamOpen::Source { src, ext }) => { + Ok(StreamSource::Video(VideoAudioReader::open_source(src, ext.as_deref())?)) } } } diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index 476a5d5..ac6a6c0 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -2432,6 +2432,7 @@ impl Engine { channels, sample_rate, total_frames, + None, // imported from an external video file; packing happens on save ); let pool_index = self.audio_pool.add_file(audio_file); diff --git a/daw-backend/src/audio/pool.rs b/daw-backend/src/audio/pool.rs index 7a90f50..fb29bb5 100644 --- a/daw-backend/src/audio/pool.rs +++ b/daw-backend/src/audio/pool.rs @@ -244,12 +244,15 @@ impl AudioFile { /// Create a placeholder AudioFile for a video's audio track. `path` is the /// source video file; the audio is streamed on demand by the disk reader's - /// FFmpeg-backed `VideoAudioReader`. + /// FFmpeg-backed `VideoAudioReader`. `packed_media_id` is `Some` when the video + /// bytes are packed in the `.beam` container (audio streams from the same blob + /// via the host factory); `None` when the audio comes from an external video file. pub fn from_video_audio( path: PathBuf, channels: u32, sample_rate: u32, total_frames: u64, + packed_media_id: Option, ) -> Self { Self { path, @@ -263,7 +266,7 @@ impl AudioFile { frames: total_frames, original_format: None, original_bytes: None, - packed_media_id: None, + packed_media_id, } } @@ -1168,7 +1171,27 @@ impl AudioClipPool { let entry_start = std::time::Instant::now(); eprintln!("📊 [LOAD_SERIALIZED] Processing entry {}/{}: '{}'", i + 1, entries.len(), entry.name); - let success = if entry.is_video_audio { + let success = if entry.is_video_audio && entry.media_id.is_some() { + // Packed video: the audio track lives in the same container blob as + // the video. Build a streaming VideoAudio entry from the saved + // metadata (no probe needed); playback opens the blob via the host + // factory at clip-activation time → VideoAudioReader::open_source. + let media_id = entry.media_id.clone().unwrap(); + let total_frames = (entry.duration * entry.sample_rate as f64).ceil() as u64; + let file = AudioFile::from_video_audio( + PathBuf::new(), + entry.channels, + entry.sample_rate, + total_frames, + Some(media_id), + ); + if entry.pool_index < self.files.len() { + self.files[entry.pool_index] = file; + true + } else { + false + } + } else if entry.is_video_audio { // Re-probe the video's audio track via FFmpeg → a streaming // VideoAudio entry (keeps full 5.1/7.1; no decode-to-RAM). match entry.relative_path.as_ref() { @@ -1186,6 +1209,7 @@ impl AudioClipPool { reader.channels(), reader.sample_rate(), reader.total_frames(), + None, ); if entry.pool_index < self.files.len() { self.files[entry.pool_index] = file; diff --git a/daw-backend/tests/video_audio_stream.rs b/daw-backend/tests/video_audio_stream.rs index 066ae62..9140579 100644 --- a/daw-backend/tests/video_audio_stream.rs +++ b/daw-backend/tests/video_audio_stream.rs @@ -251,3 +251,69 @@ fn seek_is_sample_accurate() { } let _ = std::fs::remove_file(&path); } + +// ── Stage 3: stream video-audio from a byte source (packed .beam blob) ────────── + +/// A `MediaByteSource` over an in-memory buffer (stands in for a SQLite BlobReader). +struct VecSource(std::io::Cursor>, u64); +impl std::io::Read for VecSource { + fn read(&mut self, b: &mut [u8]) -> std::io::Result { self.0.read(b) } +} +impl std::io::Seek for VecSource { + fn seek(&mut self, p: std::io::SeekFrom) -> std::io::Result { self.0.seek(p) } +} +impl daw_backend::audio::disk_reader::MediaByteSource for VecSource { + fn byte_len(&self) -> u64 { self.1 } +} + +fn ramp_wav_bytes(n: u32, sample_rate: u32) -> Vec { + let channels = 1u16; + let bps = 4u32; + let data_size = n * bps; + let mut buf = Vec::with_capacity(44 + data_size as usize); + buf.extend_from_slice(b"RIFF"); + buf.extend_from_slice(&(36 + data_size).to_le_bytes()); + buf.extend_from_slice(b"WAVE"); + buf.extend_from_slice(b"fmt "); + buf.extend_from_slice(&16u32.to_le_bytes()); + buf.extend_from_slice(&3u16.to_le_bytes()); // IEEE float + buf.extend_from_slice(&channels.to_le_bytes()); + buf.extend_from_slice(&sample_rate.to_le_bytes()); + buf.extend_from_slice(&(sample_rate * channels as u32 * bps).to_le_bytes()); + buf.extend_from_slice(&((channels as u32 * bps) as u16).to_le_bytes()); + buf.extend_from_slice(&32u16.to_le_bytes()); + buf.extend_from_slice(b"data"); + buf.extend_from_slice(&data_size.to_le_bytes()); + for i in 0..n { + buf.extend_from_slice(&((i as f32) / (n as f32)).to_le_bytes()); + } + buf +} + +#[test] +fn video_audio_open_source_streams_from_bytes() { + let sr = 8000; + let n = 4000; + let bytes = ramp_wav_bytes(n, sr); + let len = bytes.len() as u64; + let src = Box::new(VecSource(std::io::Cursor::new(bytes), len)); + + // Open the audio track by streaming from the byte source (no file path). + let mut reader = VideoAudioReader::open_source(src, Some("wav")).unwrap(); + assert_eq!(reader.channels(), 1); + assert_eq!(reader.sample_rate(), sr); + + let mut all: Vec = Vec::new(); + let mut buf: Vec = Vec::new(); + loop { + let frames = reader.decode_next(&mut buf).unwrap(); + if frames == 0 { + break; + } + all.extend_from_slice(&buf); + } + assert!(all.len() as u32 >= n - 4, "decoded most of the ramp: {} of {}", all.len(), n); + // The ramp rises monotonically; sample 0 ≈ 0.0 and the last is near 1.0. + assert!(all[0].abs() < 1e-3, "first sample ~0, got {}", all[0]); + assert!(*all.last().unwrap() > 0.9, "last sample ~1.0, got {}", all.last().unwrap()); +} diff --git a/ffmpeg-blob-io/Cargo.lock b/ffmpeg-blob-io/Cargo.lock new file mode 100644 index 0000000..2287196 --- /dev/null +++ b/ffmpeg-blob-io/Cargo.lock @@ -0,0 +1,287 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "ffmpeg-blob-io" +version = "0.1.0" +dependencies = [ + "ffmpeg-next", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-next" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d658424d233cbd993a972dd73a66ca733acd12a494c68995c9ac32ae1fe65b40" +dependencies = [ + "bitflags", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bca20aa4ee774fe384c2490096c122b0b23cf524a9910add0686691003d797b" +dependencies = [ + "bindgen", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/ffmpeg-blob-io/Cargo.toml b/ffmpeg-blob-io/Cargo.toml new file mode 100644 index 0000000..c9c58a3 --- /dev/null +++ b/ffmpeg-blob-io/Cargo.toml @@ -0,0 +1,22 @@ +# Standalone crate (own workspace root) — depended on by daw-backend and +# lightningbeam-core via relative path, like daw-backend itself. +[workspace] + +[package] +name = "ffmpeg-blob-io" +version = "0.1.0" +edition = "2021" +description = "Custom AVIOContext-over-Read+Seek shim for ffmpeg-next: lets a demuxer stream from an arbitrary byte source (e.g. a SQLite blob) instead of a file path. Isolates the unsafe FFI + ffmpeg ABI coupling." + +# ABI-PINNED. This crate dereferences raw AVFormatContext / AVIOContext fields +# (`pb`, `flags`, `buffer`, `opaque`) and wraps the result back into ffmpeg-next's +# `Input`, so it MUST link the exact same libavformat ABI as the rest of the +# workspace. The `=` pins match what lightningbeam-core / daw-backend resolve; +# bump all three together when upgrading ffmpeg. +[dependencies] +ffmpeg-next = "=8.0.0" +ffmpeg-sys-next = "=8.0.1" +libc = "0.2" + +[dev-dependencies] +ffmpeg-next = "=8.0.0" diff --git a/ffmpeg-blob-io/src/lib.rs b/ffmpeg-blob-io/src/lib.rs new file mode 100644 index 0000000..8c970c6 --- /dev/null +++ b/ffmpeg-blob-io/src/lib.rs @@ -0,0 +1,240 @@ +//! Stream a demuxer from an arbitrary `Read + Seek` byte source via a custom +//! FFmpeg `AVIOContext`. +//! +//! `ffmpeg-next`'s high-level API can only open inputs by filesystem path. This +//! crate builds an `AVFormatContext` whose I/O is backed by a Rust reader (e.g. a +//! SQLite `BlobReader`), then hands back a normal [`ffmpeg_next::format::context::Input`] +//! that the rest of the codebase decodes exactly as a file-backed input. +//! +//! All of the `unsafe` FFI and ffmpeg ABI coupling lives here, isolated behind the +//! safe [`BlobInput`] type. See `BEAM_FILE_FORMAT.md` / `STREAMING_TO_DISK_PLAN.md`. +//! +//! # Safety model +//! - The reader is boxed and leaked to FFmpeg as the AVIO `opaque` pointer; the +//! read/seek callbacks reconstitute it as `&mut`. It is never aliased — only the +//! owning [`BlobInput`] (on one thread) drives it. +//! - [`BlobInput`] is `Send` but **not** `Sync`: the typical reader (`BlobReader`) +//! owns a `!Sync` SQLite connection. Each decoder / seek-reopen / off-thread scan +//! must construct its **own** `BlobInput` from a **fresh** reader. +//! - `Drop` tears down in the one correct order (Input first, then the AVIO buffer + +//! context, then the reader) so there is no use-after-free, double-free, or leak. + +use std::io::{Read, Seek, SeekFrom}; +use std::os::raw::{c_char, c_int, c_void}; +use std::ptr; + +use ffmpeg_next::format::context::Input; +use ffmpeg_sys_next as sys; + +/// A byte source FFmpeg can stream from. Blanket-implemented for any +/// `Read + Seek + Send` (e.g. `std::io::Cursor`, a SQLite `BlobReader`). +pub trait BlobSource: Read + Seek + Send {} +impl BlobSource for T {} + +// Stable FFmpeg ABI constants (avoid depending on whether bindgen exported the +// corresponding `#define`s). These are fixed across the libavformat 58–61 ABIs. +const AVSEEK_SIZE: c_int = 0x10000; +const AVSEEK_FORCE: c_int = 0x20000; +const AVFMT_FLAG_CUSTOM_IO: c_int = 0x0080; + +/// Size of the AVIO read buffer handed to FFmpeg. FFmpeg may grow/replace it. +const IO_BUFFER_SIZE: usize = 32 * 1024; + +/// A demuxer input streaming from a boxed `Read + Seek` source. +/// +/// Deref to [`Input`] for the usual ffmpeg-next decode API. +pub struct BlobInput { + // Dropped FIRST in `Drop` (avformat_close_input). `Option` so we can drop it + // explicitly before freeing the AVIO resources it points at. + input: Option, + // The custom AVIOContext. With AVFMT_FLAG_CUSTOM_IO set, avformat_close_input + // does NOT free this — we own it. Its `opaque` field is the boxed reader. + avio: *mut sys::AVIOContext, +} + +// The reader is `Send` and never aliased; we manage its lifetime manually. +unsafe impl Send for BlobInput {} +// Intentionally NOT `Sync`: see the module-level safety notes. + +impl BlobInput { + /// Open a demuxer over `src`. + /// + /// `format_hint` is an optional container short-name / extension (e.g. `"mp4"`, + /// `"mov"`, `"matroska"`) passed to `av_find_input_format` to skip probe + /// ambiguity; `None` lets FFmpeg probe from the stream. + pub fn open(src: Box, format_hint: Option<&str>) -> Result { + ffmpeg_next::init().map_err(|e| format!("ffmpeg init failed: {e}"))?; + + unsafe { + // 1. IO buffer, allocated with FFmpeg's allocator (freed with av_freep). + let buffer = sys::av_malloc(IO_BUFFER_SIZE) as *mut u8; + if buffer.is_null() { + return Err("av_malloc failed for AVIO buffer".into()); + } + + // 2. Box the reader and leak it as the opaque pointer. Double-box so the + // raw pointer is thin (a `*mut dyn` would be a fat pointer). + let opaque = Box::into_raw(Box::new(src)) as *mut c_void; + + // 3. AVIOContext over our callbacks (read-only: write_flag = 0). + let avio = sys::avio_alloc_context( + buffer, + IO_BUFFER_SIZE as c_int, + 0, + opaque, + Some(read_cb), + None, + Some(seek_cb), + ); + if avio.is_null() { + sys::av_free(buffer as *mut c_void); + drop(Box::from_raw(opaque as *mut Box)); + return Err("avio_alloc_context failed".into()); + } + + // 4. Format context wired to the custom IO. + let mut fmt = sys::avformat_alloc_context(); + if fmt.is_null() { + destroy_io(avio); + return Err("avformat_alloc_context failed".into()); + } + (*fmt).pb = avio; + (*fmt).flags |= AVFMT_FLAG_CUSTOM_IO; + + // 5. Optional format hint. + let hint_cstr = format_hint.and_then(|s| std::ffi::CString::new(s).ok()); + let infmt: *const sys::AVInputFormat = match &hint_cstr { + Some(c) => sys::av_find_input_format(c.as_ptr() as *const c_char), + None => ptr::null(), + }; + + // 6. Open. On failure avformat_open_input frees `fmt` itself (and, with + // CUSTOM_IO, leaves our pb), so we still own avio+buffer+reader. + let ret = sys::avformat_open_input(&mut fmt, ptr::null(), infmt, ptr::null_mut()); + if ret < 0 { + destroy_io(avio); + return Err(format!("avformat_open_input failed (error {ret})")); + } + + // 7. Probe streams. On failure close the now-open input, then free IO. + let ret = sys::avformat_find_stream_info(fmt, ptr::null_mut()); + if ret < 0 { + sys::avformat_close_input(&mut fmt); + destroy_io(avio); + return Err(format!("avformat_find_stream_info failed (error {ret})")); + } + + // 8. Hand ownership of `fmt` to ffmpeg-next's Input (closes on drop). + let input = Input::wrap(fmt); + Ok(BlobInput { input: Some(input), avio }) + } + } + + /// The underlying demuxer input. + pub fn input(&self) -> &Input { + self.input.as_ref().expect("BlobInput input present until drop") + } + + /// The underlying demuxer input, mutably (for `seek`, `packets`, …). + pub fn input_mut(&mut self) -> &mut Input { + self.input.as_mut().expect("BlobInput input present until drop") + } +} + +impl std::ops::Deref for BlobInput { + type Target = Input; + fn deref(&self) -> &Input { + self.input() + } +} +impl std::ops::DerefMut for BlobInput { + fn deref_mut(&mut self) -> &mut Input { + self.input_mut() + } +} + +impl Drop for BlobInput { + fn drop(&mut self) { + unsafe { + // 1. Drop the Input first: avformat_close_input. With CUSTOM_IO this does + // NOT touch self.avio or its buffer. + self.input = None; + // 2..4. Free the AVIO buffer + context and reclaim the boxed reader. + if !self.avio.is_null() { + destroy_io(self.avio); + self.avio = ptr::null_mut(); + } + } + } +} + +/// Free a standalone custom AVIOContext: its (possibly reallocated) buffer, the +/// boxed reader behind `opaque`, then the context itself — in that order. Only +/// call when the owning AVFormatContext has already been closed (or never opened). +unsafe fn destroy_io(avio: *mut sys::AVIOContext) { + if avio.is_null() { + return; + } + // Reclaim the reader box *before* freeing the context (we need `opaque`). + let opaque = (*avio).opaque; + // Free the current IO buffer (FFmpeg may have replaced the original). + sys::av_freep(&mut (*avio).buffer as *mut _ as *mut c_void); + // Free the AVIOContext struct (nulls the local). + let mut avio = avio; + sys::avio_context_free(&mut avio); + // Drop the reader last — no callback can fire now. + if !opaque.is_null() { + drop(Box::from_raw(opaque as *mut Box)); + } +} + +/// FFmpeg read callback: fill `buf` from the Rust reader. Returns bytes read, +/// `AVERROR_EOF` at end of stream, or a negative error. +unsafe extern "C" fn read_cb(opaque: *mut c_void, buf: *mut u8, buf_size: c_int) -> c_int { + if opaque.is_null() || buf.is_null() || buf_size <= 0 { + return sys::AVERROR_EOF; + } + let reader = &mut *(opaque as *mut Box); + let slice = std::slice::from_raw_parts_mut(buf, buf_size as usize); + match reader.read(slice) { + Ok(0) => sys::AVERROR_EOF, + Ok(n) => n as c_int, + // AVERROR(EIO) == -EIO on Unix; a negative value signals a read error. + Err(_) => -(libc::EIO as c_int), + } +} + +/// FFmpeg seek callback. Handles `SEEK_SET/CUR/END` and `AVSEEK_SIZE` (report total +/// length). Returns the new position, the size, or `-1` on error. +unsafe extern "C" fn seek_cb(opaque: *mut c_void, offset: i64, whence: c_int) -> i64 { + if opaque.is_null() { + return -1; + } + let reader = &mut *(opaque as *mut Box); + let whence = whence & !AVSEEK_FORCE; + if whence & AVSEEK_SIZE != 0 { + // Report total length WITHOUT moving the logical position. FFmpeg does not + // restore the position after an AVSEEK_SIZE query, so we must: measure by + // seeking to the end, then seek back. (Failing to restore corrupts reads of + // containers whose index lives at the end of the file, e.g. MP4 `moov`.) + let cur = match reader.stream_position() { + Ok(p) => p, + Err(_) => return -1, + }; + let size = match reader.seek(SeekFrom::End(0)) { + Ok(s) => s, + Err(_) => return -1, + }; + if reader.seek(SeekFrom::Start(cur)).is_err() { + return -1; + } + return size as i64; + } + let from = match whence { + libc::SEEK_SET => SeekFrom::Start(offset as u64), + libc::SEEK_CUR => SeekFrom::Current(offset), + libc::SEEK_END => SeekFrom::End(offset), + _ => return -1, + }; + reader.seek(from).map(|n| n as i64).unwrap_or(-1) +} diff --git a/ffmpeg-blob-io/tests/blob_input.rs b/ffmpeg-blob-io/tests/blob_input.rs new file mode 100644 index 0000000..91cacb8 --- /dev/null +++ b/ffmpeg-blob-io/tests/blob_input.rs @@ -0,0 +1,100 @@ +//! Exercises the custom AVIO shim end to end against a hand-built WAV, fed through +//! an in-memory `Cursor` (a `Read + Seek` source). FFmpeg can only demux it by +//! calling our `read_cb`/`seek_cb`, so a successful open proves the shim works. + +use ffmpeg_blob_io::BlobInput; +use std::io::Cursor; + +/// Build a minimal 16-bit PCM WAV in memory. +fn make_wav(sample_rate: u32, channels: u16, samples: &[i16]) -> Vec { + let bits: u16 = 16; + let block_align: u16 = channels * (bits / 8); + let byte_rate: u32 = sample_rate * block_align as u32; + let data_len: u32 = (samples.len() * 2) as u32; + + let mut v = Vec::new(); + v.extend_from_slice(b"RIFF"); + v.extend_from_slice(&(36 + data_len).to_le_bytes()); + v.extend_from_slice(b"WAVE"); + v.extend_from_slice(b"fmt "); + v.extend_from_slice(&16u32.to_le_bytes()); // fmt chunk size + v.extend_from_slice(&1u16.to_le_bytes()); // PCM + v.extend_from_slice(&channels.to_le_bytes()); + v.extend_from_slice(&sample_rate.to_le_bytes()); + v.extend_from_slice(&byte_rate.to_le_bytes()); + v.extend_from_slice(&block_align.to_le_bytes()); + v.extend_from_slice(&bits.to_le_bytes()); + v.extend_from_slice(b"data"); + v.extend_from_slice(&data_len.to_le_bytes()); + for s in samples { + v.extend_from_slice(&s.to_le_bytes()); + } + v +} + +fn sample_wav() -> Vec { + let samples: Vec = (0..1600).map(|i| ((i % 100) as i16 - 50) * 200).collect(); + make_wav(8000, 1, &samples) +} + +#[test] +fn opens_from_reader_and_reports_stream() { + let input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav")) + .expect("open WAV through the blob shim"); + + let stream = input + .streams() + .best(ffmpeg_next::media::Type::Audio) + .expect("an audio stream"); + let ctx = ffmpeg_next::codec::context::Context::from_parameters(stream.parameters()) + .expect("codec context"); + let decoder = ctx.decoder().audio().expect("audio decoder"); + + assert_eq!(decoder.rate(), 8000, "sample rate read via AVIO"); + assert_eq!(decoder.channels(), 1, "channel count read via AVIO"); +} + +#[test] +fn reads_packets_through_callbacks() { + let mut input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav")) + .expect("open WAV"); + let mut packets = 0usize; + let mut bytes = 0usize; + for (_stream, packet) in input.packets() { + packets += 1; + bytes += packet.size(); + } + assert!(packets > 0, "demuxer produced packets via read_cb"); + assert!(bytes > 0, "packets carried PCM payload"); +} + +#[test] +fn seek_then_read() { + let mut input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav")) + .expect("open WAV"); + // Seek back to the start (exercises seek_cb SEEK_SET + AVSEEK_SIZE). + input.seek(0, ..).expect("seek to start"); + let got_packet = input.packets().next().is_some(); + assert!(got_packet, "can still read after seek"); +} + +#[test] +fn open_drop_loop_is_clean() { + // Repeated open+drop surfaces double-free / leak in the Drop teardown + // (run under `RUSTFLAGS=-Zsanitizer=address` on nightly for full coverage). + for _ in 0..200 { + let input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav")) + .expect("open WAV"); + assert!(input.streams().count() >= 1); + drop(input); + } +} + +#[test] +fn bad_format_hint_errors_without_leak() { + // Garbage bytes with a real hint: open should fail cleanly (error path frees + // buffer + avio + reader), not panic or leak. + let garbage = vec![0u8; 4096]; + let res = BlobInput::open(Box::new(Cursor::new(garbage)), Some("wav")); + assert!(res.is_err(), "non-WAV bytes should fail to open as WAV"); +} diff --git a/lightningbeam-ui/Cargo.lock b/lightningbeam-ui/Cargo.lock index 4505926..7fe5fcf 100644 --- a/lightningbeam-ui/Cargo.lock +++ b/lightningbeam-ui/Cargo.lock @@ -1682,6 +1682,7 @@ dependencies = [ "dasp_rms", "dasp_sample", "dasp_signal", + "ffmpeg-blob-io", "ffmpeg-next", "hound", "memmap2", @@ -2218,6 +2219,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "ffmpeg-blob-io" +version = "0.1.0" +dependencies = [ + "ffmpeg-next", + "ffmpeg-sys-next", + "libc", +] + [[package]] name = "ffmpeg-next" version = "8.0.0" @@ -3480,6 +3490,7 @@ dependencies = [ "claxon", "daw-backend", "egui", + "ffmpeg-blob-io", "ffmpeg-next", "flacenc", "image", @@ -3505,7 +3516,7 @@ dependencies = [ [[package]] name = "lightningbeam-editor" -version = "1.0.4-alpha" +version = "1.0.5-alpha" dependencies = [ "beamdsp", "bytemuck", diff --git a/lightningbeam-ui/lightningbeam-core/Cargo.toml b/lightningbeam-ui/lightningbeam-core/Cargo.toml index 99a68c9..c8a8fef 100644 --- a/lightningbeam-ui/lightningbeam-core/Cargo.toml +++ b/lightningbeam-ui/lightningbeam-core/Cargo.toml @@ -29,6 +29,8 @@ daw-backend = { path = "../../daw-backend" } # Video decoding ffmpeg-next = "8.0" +# AVIO-over-Read+Seek shim: decode packed video by streaming from the SQLite blob. +ffmpeg-blob-io = { path = "../../ffmpeg-blob-io" } lru = "0.12" # File I/O @@ -73,3 +75,6 @@ windows-sys = { version = "0.60", features = [ version = "0.11" [dev-dependencies] +# For the packed-video streaming integration test (blob -> AVIO -> ffmpeg Input). +ffmpeg-blob-io = { path = "../../ffmpeg-blob-io" } +ffmpeg-next = "8.0" diff --git a/lightningbeam-ui/lightningbeam-core/src/clip.rs b/lightningbeam-ui/lightningbeam-core/src/clip.rs index 01e24dc..f483380 100644 --- a/lightningbeam-ui/lightningbeam-core/src/clip.rs +++ b/lightningbeam-ui/lightningbeam-core/src/clip.rs @@ -343,6 +343,13 @@ pub struct VideoClip { #[serde(default, skip_serializing_if = "Option::is_none")] pub linked_audio_clip_id: Option, + /// When set, the video bytes are packed into the `.beam` container under this + /// media id (== the clip id) and decoded by streaming from the SQLite blob. + /// `None` means the video is referenced externally via [`Self::file_path`]. + /// Reconstructed from the archive on load (see `load_beam_sqlite`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub media_id: Option, + /// Folder this clip belongs to (None = root of category) #[serde(default)] pub folder_id: Option, @@ -367,6 +374,7 @@ impl VideoClip { duration, frame_rate, linked_audio_clip_id: None, + media_id: None, folder_id: None, } } diff --git a/lightningbeam-ui/lightningbeam-core/src/file_io.rs b/lightningbeam-ui/lightningbeam-core/src/file_io.rs index 335b22f..4df18b7 100644 --- a/lightningbeam-ui/lightningbeam-core/src/file_io.rs +++ b/lightningbeam-ui/lightningbeam-core/src/file_io.rs @@ -9,7 +9,7 @@ //! too (via [`load_beam_zip_legacy`]). Saving always writes the SQLite form, so //! opening a legacy file and saving migrates it. -use crate::beam_archive::{BeamArchive, MediaKind, MediaMeta, LARGE_MEDIA_THRESHOLD}; +use crate::beam_archive::{BeamArchive, MediaKind, MediaMeta, MediaStorage, LARGE_MEDIA_THRESHOLD}; use crate::document::Document; use daw_backend::audio::pool::AudioPoolEntry; use daw_backend::audio::project::Project as AudioProject; @@ -450,6 +450,65 @@ pub fn save_beam( } } + // --- video clips -> media rows (Video bytes), keyed by the clip id --- + // Mirror the audio pack/reference decision: pack unless the file is + // >= LARGE_MEDIA_THRESHOLD and the user didn't choose Pack. Packed video is + // decoded by streaming from the blob (see video.rs `VideoSource` + ffmpeg-blob-io). + for (clip_id, clip) in &document.video_clips { + // In-place re-save: an unchanged packed video keeps its row, never re-streamed. + if clip.media_id == Some(*clip_id) && txn.media_exists(*clip_id)? { + live_media.insert(*clip_id); + continue; + } + if clip.file_path.is_empty() { + continue; + } + let full = if Path::new(&clip.file_path).is_absolute() { + PathBuf::from(&clip.file_path) + } else { + project_dir.join(&clip.file_path) + }; + if !full.is_file() { + continue; // source gone — reported missing on load + } + let size = std::fs::metadata(&full).map(|m| m.len()).unwrap_or(0); + let codec = full + .extension() + .and_then(|x| x.to_str()) + .unwrap_or("mp4") + .to_lowercase(); + let meta = MediaMeta { + width: Some(clip.width as u32), + height: Some(clip.height as u32), + ..Default::default() + }; + let reference_it = + size >= LARGE_MEDIA_THRESHOLD && settings.large_media_mode != LargeMediaMode::Pack; + if reference_it { + txn.put_media_referenced(*clip_id, MediaKind::Video, &codec, &clip.file_path, meta)?; + } else { + txn.put_media_packed_from_path(*clip_id, MediaKind::Video, &codec, &full, meta)?; + // The video's audio track lives in this same blob. Point the linked + // video-audio pool entry at the video's media row so its audio streams + // from the blob too (instead of re-probing the external file on load). + if let Some(aid) = clip.linked_audio_clip_id { + if let Some(crate::clip::AudioClipType::Sampled { audio_pool_index }) = + document.audio_clips.get(&aid).map(|c| &c.clip_type) + { + if let Some(e) = modified_entries + .iter_mut() + .find(|e| e.pool_index == *audio_pool_index) + { + e.media_id = Some(clip_id.to_string()); + e.relative_path = None; + e.embedded_data = None; + } + } + } + } + live_media.insert(*clip_id); + } + // --- image assets -> media rows (original file bytes), keyed by asset id --- let mut image_count = 0usize; for (id, asset) in &document.image_assets { @@ -646,7 +705,7 @@ fn load_beam_sqlite(path: &Path) -> Result { // Missing external files (referenced entries whose file no longer exists). let project_dir = path.parent().unwrap_or_else(|| Path::new(".")); - let missing_files: Vec = restored_entries + let mut missing_files: Vec = restored_entries .iter() .enumerate() .filter_map(|(idx, entry)| { @@ -670,6 +729,42 @@ fn load_beam_sqlite(path: &Path) -> Result { }) .collect(); + // Resolve packed/referenced video bytes per clip (mirrors audio resolution). + // A `MediaKind::Video` row keyed by the clip id means the bytes are in the + // container; the decoder streams from the blob. Referenced rows update the + // external path and report a missing source. Old projects (no Video row) stay + // referenced via their existing `file_path`. + for (clip_id, clip) in document.video_clips.iter_mut() { + match archive.media_info(*clip_id) { + Ok(Some(info)) if info.kind == MediaKind::Video => match info.storage { + MediaStorage::Packed => { + clip.media_id = Some(*clip_id); + } + MediaStorage::Referenced => { + clip.media_id = None; + if let Some(p) = info.ext_path.clone() { + clip.file_path = p; + } + let full = if Path::new(&clip.file_path).is_absolute() { + PathBuf::from(&clip.file_path) + } else { + project_dir.join(&clip.file_path) + }; + if !clip.file_path.is_empty() && !full.exists() { + missing_files.push(MissingFileInfo { + pool_index: 0, + original_path: full, + file_type: MediaFileType::Video, + }); + } + } + }, + _ => { + clip.media_id = None; + } + } + } + // Persisted video thumbnail packs (opaque LBTN blobs), keyed by clip id. The // editor decodes + inserts them and skips regeneration for these clips. let mut thumbnail_blobs = std::collections::HashMap::new(); diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 6182df7..801132e 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -6,10 +6,71 @@ use std::sync::{Arc, Mutex}; use std::num::NonZeroUsize; use std::collections::HashMap; +use std::path::PathBuf; use ffmpeg_next as ffmpeg; +use ffmpeg_blob_io::BlobInput; use lru::LruCache; use uuid::Uuid; +use crate::beam_archive::BeamArchive; + +/// Where a video clip's bytes live. +/// +/// `Path` is an external file (referenced video, webcam capture, fresh import). +/// `Packed` streams from a `MediaKind::Video` blob inside the `.beam` container. +#[derive(Clone, Debug)] +pub enum VideoSource { + /// External file path. + Path(String), + /// Packed in the container: open a fresh `BlobReader` over `media_id` in `db_path`. + Packed { db_path: PathBuf, media_id: Uuid }, +} + +impl VideoSource { + /// Open a fresh demuxer input for this source. A new `BlobReader` (own SQLite + /// connection) is created per call, so this is safe to call on any thread and + /// on every seek-reopen. + fn open(&self) -> Result { + match self { + VideoSource::Path(p) => ffmpeg::format::input(p) + .map(OwnedInput::Path) + .map_err(|e| format!("Failed to open video: {}", e)), + VideoSource::Packed { db_path, media_id } => { + let archive = BeamArchive::open(db_path)?; + let hint = archive.media_info(*media_id)?.map(|i| i.codec); + let reader = archive.open_blob_reader(db_path, *media_id)?; + BlobInput::open(Box::new(reader), hint.as_deref()) + .map(OwnedInput::Blob) + .map_err(|e| format!("Failed to open packed video: {}", e)) + } + } + } + + /// Short label for logging. + fn label(&self) -> String { + match self { + VideoSource::Path(p) => p.clone(), + VideoSource::Packed { media_id, .. } => format!("packed:{}", media_id), + } + } +} + +/// An open demuxer input, either file-backed or streaming from a container blob. +/// Both expose the same `ffmpeg` `Input` for decoding. +enum OwnedInput { + Path(ffmpeg::format::context::Input), + Blob(BlobInput), +} + +impl OwnedInput { + fn get(&mut self) -> &mut ffmpeg::format::context::Input { + match self { + OwnedInput::Path(i) => i, + OwnedInput::Blob(b) => b.input_mut(), + } + } +} + /// Metadata about a video file #[derive(Debug, Clone)] pub struct VideoMetadata { @@ -22,7 +83,7 @@ pub struct VideoMetadata { /// Video decoder with LRU frame caching pub struct VideoDecoder { - path: String, + source: VideoSource, _width: u32, // Original video width _height: u32, // Original video height output_width: u32, // Scaled output width @@ -32,7 +93,7 @@ pub struct VideoDecoder { time_base: f64, stream_index: usize, frame_cache: LruCache>, // timestamp -> RGBA data - input: Option, + input: Option, decoder: Option, last_decoded_ts: i64, // Track the last decoded frame timestamp keyframe_positions: Vec, // Index of keyframe timestamps for fast seeking @@ -45,11 +106,11 @@ impl VideoDecoder { /// Video will be scaled down if larger, preserving aspect ratio. /// `build_keyframes` controls whether to build the keyframe index immediately (slow) /// or defer it for async building later. - fn new(path: String, cache_size: usize, max_width: Option, max_height: Option, build_keyframes: bool) -> Result { + fn new(source: VideoSource, cache_size: usize, max_width: Option, max_height: Option, build_keyframes: bool) -> Result { ffmpeg::init().map_err(|e| e.to_string())?; - let input = ffmpeg::format::input(&path) - .map_err(|e| format!("Failed to open video: {}", e))?; + let mut owned = source.open()?; + let input = owned.get(); let video_stream = input.streams() .best(ffmpeg::media::Type::Video) @@ -96,17 +157,17 @@ impl VideoDecoder { // Optionally build keyframe index for fast seeking let keyframe_positions = if build_keyframes { - eprintln!("[Video Decoder] Building keyframe index for {}", path); - let positions = Self::scan_keyframes(&path, stream_index)?; + eprintln!("[Video Decoder] Building keyframe index for {}", source.label()); + let positions = Self::scan_keyframes(&source, stream_index)?; eprintln!("[Video Decoder] Found {} keyframes", positions.len()); positions } else { - eprintln!("[Video Decoder] Deferring keyframe index building for {}", path); + eprintln!("[Video Decoder] Deferring keyframe index building for {}", source.label()); Vec::new() }; Ok(Self { - path, + source, _width: width, _height: height, output_width, @@ -125,14 +186,14 @@ impl VideoDecoder { }) } - /// Source file path this decoder reads from. - pub fn path(&self) -> &str { - &self.path + /// The source this decoder reads from (file path or packed container blob). + pub fn source(&self) -> VideoSource { + self.source.clone() } - /// Parameters needed to scan keyframes off-thread (path + video stream index). - pub fn keyframe_scan_params(&self) -> (String, usize) { - (self.path.clone(), self.stream_index) + /// Parameters needed to scan keyframes off-thread (source + video stream index). + pub fn keyframe_scan_params(&self) -> (VideoSource, usize) { + (self.source.clone(), self.stream_index) } /// Replace the keyframe index (built off-thread via [`VideoDecoder::scan_keyframes`]). @@ -158,9 +219,10 @@ impl VideoDecoder { /// Build an index of all keyframe positions in the video by scanning packets /// from a fresh input. Does not touch `self` — call it off-thread (it is slow /// on long videos) and hand the result to [`VideoDecoder::set_keyframe_index`]. - pub fn scan_keyframes(path: &str, stream_index: usize) -> Result, String> { - let mut input = ffmpeg::format::input(path) + pub fn scan_keyframes(source: &VideoSource, stream_index: usize) -> Result, String> { + let mut owned = source.open() .map_err(|e| format!("Failed to open video for indexing: {}", e))?; + let input = owned.get(); let mut keyframes = Vec::new(); @@ -233,32 +295,34 @@ impl VideoDecoder { eprintln!("[Video Seek] Target: {} | Keyframe(stream): {} | Keyframe(AV): {} | Index size: {}", frame_ts, keyframe_ts_stream, keyframe_ts_av, self.keyframe_positions.len()); - // Reopen input - let mut input = ffmpeg::format::input(&self.path) + // Reopen input (a fresh BlobReader for packed sources). + let mut owned = self.source.open() .map_err(|e| format!("Failed to reopen video: {}", e))?; - // Seek directly to the keyframe with a 1-unit window - // Can't use keyframe_ts..keyframe_ts (empty) or ..= (not supported) - input.seek(keyframe_ts_av, keyframe_ts_av..(keyframe_ts_av + 1)) - .map_err(|e| format!("Seek failed: {}", e))?; + { + let input = owned.get(); + // Seek directly to the keyframe with a 1-unit window + // Can't use keyframe_ts..keyframe_ts (empty) or ..= (not supported) + input.seek(keyframe_ts_av, keyframe_ts_av..(keyframe_ts_av + 1)) + .map_err(|e| format!("Seek failed: {}", e))?; - eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis()); + eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis()); - let context_decoder = ffmpeg::codec::context::Context::from_parameters( - input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() - ).map_err(|e| e.to_string())?; + let context_decoder = ffmpeg::codec::context::Context::from_parameters( + input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() + ).map_err(|e| e.to_string())?; - let decoder = context_decoder.decoder().video() - .map_err(|e| e.to_string())?; - - self.input = Some(input); - self.decoder = Some(decoder); + let decoder = context_decoder.decoder().video() + .map_err(|e| e.to_string())?; + self.decoder = Some(decoder); + } + self.input = Some(owned); // Set last_decoded_ts to just before the seek target so forward playback works // Without this, every frame would trigger a new seek self.last_decoded_ts = frame_ts - 1; } - let input = self.input.as_mut().unwrap(); + let input = self.input.as_mut().unwrap().get(); let decoder = self.decoder.as_mut().unwrap(); // Decode frames until we find the one closest to our target timestamp @@ -356,7 +420,7 @@ impl VideoDecoder { /// `get_thumbnail_at`'s 128-wide assumption holds) and tightly packed RGBA is /// handed to `on_thumb` as `(timestamp_secs, data)`. pub fn generate_keyframe_thumbnails( - path: &str, + source: VideoSource, interval_secs: f64, thumb_width: u32, mut should_skip: impl FnMut(f64) -> bool, @@ -366,7 +430,7 @@ pub fn generate_keyframe_thumbnails( // large max-height lets width be the constraining dimension, so output width // is exactly `thumb_width`. let mut decoder = VideoDecoder::new( - path.to_string(), + source, 4, Some(thumb_width), Some(100_000), @@ -399,11 +463,11 @@ pub fn generate_keyframe_thumbnails( } /// Probe video file for metadata without creating a full decoder -pub fn probe_video(path: &str) -> Result { +pub fn probe_video(source: &VideoSource) -> Result { ffmpeg::init().map_err(|e| e.to_string())?; - let input = ffmpeg::format::input(path) - .map_err(|e| format!("Failed to open video: {}", e))?; + let mut owned = source.open()?; + let input = owned.get(); let video_stream = input.streams() .best(ffmpeg::media::Type::Video) @@ -521,16 +585,16 @@ impl VideoManager { pub fn load_video( &mut self, clip_id: Uuid, - path: String, + source: VideoSource, target_width: u32, target_height: u32, ) -> Result { // First probe the video for metadata - let metadata = probe_video(&path)?; + let metadata = probe_video(&source)?; // Create decoder with target dimensions, without building keyframe index let decoder = VideoDecoder::new( - path, + source, self.cache_size, Some(target_width), Some(target_height), diff --git a/lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs b/lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs index a0d4b04..7a4f708 100644 --- a/lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs +++ b/lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs @@ -205,3 +205,51 @@ fn overwrite_media_replaces_chunks() { assert_eq!(archive.read_media_full(id).unwrap(), vec![2u8; 50]); let _ = std::fs::remove_file(&path); } + +#[test] +fn packed_video_from_path_roundtrips() { + // Simulates the save path: stream a video file into a MediaKind::Video blob, + // then read it back (frames/audio decode would open this via the AVIO shim). + let path = temp_db_path("video_pack"); + let mut src = std::env::temp_dir(); + src.push(format!("beam_video_src_{}.mp4", std::process::id())); + let bytes: Vec = (0..(9 * 1024 * 1024u32)).map(|i| (i % 251) as u8).collect(); + std::fs::write(&src, &bytes).unwrap(); + + let id = Uuid::new_v4(); + let mut archive = BeamArchive::create(&path).unwrap(); + { + let txn = archive.transaction().unwrap(); + txn.put_media_packed_from_path( + id, + MediaKind::Video, + "mp4", + &src, + MediaMeta { width: Some(1920), height: Some(1080), ..Default::default() }, + ) + .unwrap(); + txn.commit().unwrap(); + } + + let info = archive.media_info(id).unwrap().expect("video media row"); + assert_eq!(info.kind, MediaKind::Video); + assert_eq!(info.storage, MediaStorage::Packed); + assert_eq!(info.codec, "mp4"); + assert_eq!(info.total_len, bytes.len() as u64); + assert_eq!(info.width, Some(1920)); + assert_eq!(info.height, Some(1080)); + assert_eq!(archive.read_media_full(id).unwrap(), bytes); + + // Streaming read mirrors how the decoder pulls bytes via its BlobReader. + let mut reader = archive.open_blob_reader(&path, id).unwrap(); + let mut streamed = Vec::new(); + reader.read_to_end(&mut streamed).unwrap(); + assert_eq!(streamed, bytes); + reader.seek(SeekFrom::Start(5 * 1024 * 1024)).unwrap(); + let mut buf = [0u8; 4]; + reader.read_exact(&mut buf).unwrap(); + assert_eq!(buf[0], ((5 * 1024 * 1024u32) % 251) as u8); + + let _ = std::fs::remove_file(&src); + let _ = std::fs::remove_file(&path); +} diff --git a/lightningbeam-ui/lightningbeam-core/tests/packed_video_stream.rs b/lightningbeam-ui/lightningbeam-core/tests/packed_video_stream.rs new file mode 100644 index 0000000..9bf3f1a --- /dev/null +++ b/lightningbeam-ui/lightningbeam-core/tests/packed_video_stream.rs @@ -0,0 +1,81 @@ +//! Integration test for the packed-video streaming path: +//! SQLite `MediaKind::Video` blob -> `BlobReader` -> `ffmpeg-blob-io` AVIO shim -> +//! ffmpeg `Input`. This is exactly what `video.rs`'s `VideoSource::Packed` does to +//! decode frames (and what `daw-backend` does for the embedded audio), minus the +//! thin wrapper. We use a hand-built WAV as the packed container — FFmpeg must +//! demux it through our blob reader, proving the production byte path end to end. +//! +//! (Decoding real video frames is left to user runtime verification; here we prove +//! the container streams from the blob and exposes its streams.) + +use ffmpeg_blob_io::BlobInput; +use lightningbeam_core::beam_archive::{BeamArchive, MediaKind, MediaMeta, MediaStorage}; +use std::sync::atomic::{AtomicU64, Ordering}; +use uuid::Uuid; + +fn temp_db_path(tag: &str) -> std::path::PathBuf { + static N: AtomicU64 = AtomicU64::new(0); + let n = N.fetch_add(1, Ordering::Relaxed); + let mut p = std::env::temp_dir(); + p.push(format!("packed_video_it_{}_{}_{}.beam", std::process::id(), tag, n)); + let _ = std::fs::remove_file(&p); + p +} + +/// Minimal 16-bit PCM WAV (a real, demuxable container). +fn make_wav(sample_rate: u32, channels: u16, samples: &[i16]) -> Vec { + let bits: u16 = 16; + let block_align: u16 = channels * (bits / 8); + let byte_rate: u32 = sample_rate * block_align as u32; + let data_len: u32 = (samples.len() * 2) as u32; + let mut v = Vec::new(); + v.extend_from_slice(b"RIFF"); + v.extend_from_slice(&(36 + data_len).to_le_bytes()); + v.extend_from_slice(b"WAVE"); + v.extend_from_slice(b"fmt "); + v.extend_from_slice(&16u32.to_le_bytes()); + v.extend_from_slice(&1u16.to_le_bytes()); + v.extend_from_slice(&channels.to_le_bytes()); + v.extend_from_slice(&sample_rate.to_le_bytes()); + v.extend_from_slice(&byte_rate.to_le_bytes()); + v.extend_from_slice(&block_align.to_le_bytes()); + v.extend_from_slice(&bits.to_le_bytes()); + v.extend_from_slice(b"data"); + v.extend_from_slice(&data_len.to_le_bytes()); + for s in samples { + v.extend_from_slice(&s.to_le_bytes()); + } + v +} + +#[test] +fn packed_media_streams_through_avio_to_ffmpeg() { + let path = temp_db_path("stream"); + let id = Uuid::new_v4(); + let samples: Vec = (0..4000).map(|i| ((i % 200) as i16 - 100) * 100).collect(); + let container = make_wav(8000, 1, &samples); + + // Pack as a Video media row (the save path does this for real videos). + let mut archive = BeamArchive::create(&path).unwrap(); + archive + .put_media_packed(id, MediaKind::Video, "wav", &container, MediaMeta::default()) + .unwrap(); + let info = archive.media_info(id).unwrap().unwrap(); + assert_eq!(info.kind, MediaKind::Video); + assert_eq!(info.storage, MediaStorage::Packed); + drop(archive); + + // Reproduce VideoSource::Packed::open(): fresh read-only archive + blob reader. + let archive = BeamArchive::open(&path).unwrap(); + let hint = archive.media_info(id).unwrap().map(|i| i.codec); + let reader = archive.open_blob_reader(&path, id).unwrap(); + + let input = BlobInput::open(Box::new(reader), hint.as_deref()) + .expect("open the packed container by streaming from the SQLite blob"); + assert!( + input.streams().count() >= 1, + "demuxer found streams via the blob-backed AVIO shim" + ); + + let _ = std::fs::remove_file(&path); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index e4f94b4..b01ba6c 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -4225,6 +4225,12 @@ impl EditorApp { }); } + // Set the loaded container path BEFORE registering videos: a packed video + // clip resolves to `VideoSource::Packed { db_path: current_file_path, .. }`, + // so this must be in place or the clip falls back to its (possibly missing) + // external file and renders black. + self.current_file_path = Some(path.clone()); + // Re-register video clips with the VideoManager (decoder + keyframe index + // thumbnails). Restored video_clips carry only metadata + file_path; without // this they render black with no thumbnails. @@ -4235,7 +4241,6 @@ impl EditorApp { // Reset playback state self.playback_time = 0.0; self.is_playing = false; - self.current_file_path = Some(path.clone()); // Point the raster paging store at the loaded container so faulting works. self.raster_store.set_path(self.current_file_path.clone()); @@ -4260,26 +4265,37 @@ impl EditorApp { let doc_width = self.action_executor.document().width as u32; let doc_height = self.action_executor.document().height as u32; - // Snapshot (id, path) so we don't hold the document borrow while locking - // the VideoManager / spawning threads. + // Snapshot (id, media_id, path) so we don't hold the document borrow while + // locking the VideoManager / spawning threads. + let db_path = self.current_file_path.clone(); let videos: Vec<_> = self .action_executor .document() .video_clips .values() - .map(|c| (c.id, c.file_path.clone())) + .map(|c| (c.id, c.media_id, c.file_path.clone())) .collect(); - for (clip_id, file_path) in videos { - if !std::path::Path::new(&file_path).exists() { - eprintln!("⚠️ [APPLY] Video file not found, skipping: {}", file_path); - continue; - } + for (clip_id, media_id, file_path) in videos { + // Packed video streams from the container blob; referenced video opens + // its external file (skip if it's gone). + let source = match (media_id, &db_path) { + (Some(mid), Some(db)) => { + lightningbeam_core::video::VideoSource::Packed { db_path: db.clone(), media_id: mid } + } + _ => { + if !std::path::Path::new(&file_path).exists() { + eprintln!("⚠️ [APPLY] Video file not found, skipping: {}", file_path); + continue; + } + lightningbeam_core::video::VideoSource::Path(file_path.clone()) + } + }; { let mut video_mgr = self.video_manager.lock().unwrap(); - if let Err(e) = video_mgr.load_video(clip_id, file_path.clone(), doc_width, doc_height) { - eprintln!("⚠️ [APPLY] Failed to load video {}: {}", file_path, e); + if let Err(e) = video_mgr.load_video(clip_id, source, doc_width, doc_height) { + eprintln!("⚠️ [APPLY] Failed to load video {}: {}", clip_id, e); continue; } } @@ -4308,13 +4324,13 @@ impl EditorApp { }; let Some(decoder_arc) = decoder_arc else { return; }; - let (path, stream_index) = { + let (source, stream_index) = { let decoder = decoder_arc.lock().unwrap(); decoder.keyframe_scan_params() }; // Slow scan, no locks held. - match lightningbeam_core::video::VideoDecoder::scan_keyframes(&path, stream_index) { + match lightningbeam_core::video::VideoDecoder::scan_keyframes(&source, stream_index) { Ok(positions) => { let count = positions.len(); decoder_arc.lock().unwrap().set_keyframe_index(positions); @@ -4333,22 +4349,22 @@ impl EditorApp { fn spawn_thumbnail_generation(&self, clip_id: uuid::Uuid) { let video_manager = Arc::clone(&self.video_manager); std::thread::spawn(move || { - // Grab the source path from the playback decoder (brief lock), then do - // all decoding on an independent decoder. - let path = { + // Grab the source from the playback decoder (brief lock), then do all + // decoding on an independent decoder (a fresh BlobReader for packed video). + let source = { let decoder_arc = { let vm = video_manager.lock().unwrap(); vm.get_decoder(&clip_id) }; let Some(decoder_arc) = decoder_arc else { return; }; let decoder = decoder_arc.lock().unwrap(); - decoder.path().to_string() + decoder.source() }; let vm_insert = Arc::clone(&video_manager); let vm_skip = Arc::clone(&video_manager); let result = lightningbeam_core::video::generate_keyframe_thumbnails( - &path, + source, 5.0, 128, // Resume: skip keyframes already covered (e.g. restored from a @@ -4624,8 +4640,9 @@ impl EditorApp { let path_str = path.to_string_lossy().to_string(); - // Probe video for metadata - let metadata = match probe_video(&path_str) { + // Probe video for metadata (freshly imported video is referenced until the + // first save packs it). + let metadata = match probe_video(&lightningbeam_core::video::VideoSource::Path(path_str.clone())) { Ok(meta) => meta, Err(e) => { eprintln!("Failed to probe video '{}': {}", name, e); @@ -4650,7 +4667,12 @@ impl EditorApp { let doc_height = self.action_executor.document().height as u32; let mut video_mgr = self.video_manager.lock().unwrap(); - if let Err(e) = video_mgr.load_video(clip_id, path_str.clone(), doc_width, doc_height) { + if let Err(e) = video_mgr.load_video( + clip_id, + lightningbeam_core::video::VideoSource::Path(path_str.clone()), + doc_width, + doc_height, + ) { eprintln!("Failed to load video '{}': {}", name, e); return None; } @@ -6670,7 +6692,7 @@ impl eframe::App for EditorApp { eprintln!("[WEBCAM] Recording saved to: {} (recorder duration={:.4}s)", file_path_str, result.duration); // Create VideoClip + ClipInstance from recorded file if let Some(layer_id) = webcam_layer_id { - match lightningbeam_core::video::probe_video(&file_path_str) { + match lightningbeam_core::video::probe_video(&lightningbeam_core::video::VideoSource::Path(file_path_str.clone())) { Ok(info) => { use lightningbeam_core::clip::{VideoClip, ClipInstance}; let clip = VideoClip { @@ -6685,6 +6707,7 @@ impl eframe::App for EditorApp { duration: info.duration, frame_rate: info.fps, linked_audio_clip_id: None, + media_id: None, folder_id: None, }; let clip_id = clip.id; @@ -6723,7 +6746,7 @@ impl eframe::App for EditorApp { // for the display rect. { let mut vm = self.video_manager.lock().unwrap(); - if let Err(e) = vm.load_video(clip_id, file_path_str, info.width, info.height) { + if let Err(e) = vm.load_video(clip_id, lightningbeam_core::video::VideoSource::Path(file_path_str.clone()), info.width, info.height) { eprintln!("[WEBCAM] Failed to load recorded video: {}", e); } }