skyler
/
Lightningbeam
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Release 1.0.6-alpha

This commit is contained in:
Skyler Lehmkuhl 2026-06-25 18:38:13 -04:00
parent f0929e2b6d 5bed2e8adb
commit d6506c2f22
114 changed files with 6278 additions and 12251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
.github/workflows/build.yml vendored
View File

@ -71,6 +71,7 @@ jobs:
libx11-dev libxkbcommon-dev libxcb-shape0-dev libxcb-xfixes0-dev \ libx11-dev libxkbcommon-dev libxcb-shape0-dev libxcb-xfixes0-dev \
libxdo-dev libglib2.0-dev libgtk-3-dev libvulkan-dev \ libxdo-dev libglib2.0-dev libgtk-3-dev libvulkan-dev \
yasm libx264-dev libx265-dev libvpx-dev libmp3lame-dev libopus-dev \ yasm libx264-dev libx265-dev libvpx-dev libmp3lame-dev libopus-dev \
libva-dev libdrm-dev \
libpulse-dev squashfs-tools dpkg rpm libpulse-dev squashfs-tools dpkg rpm
- name: Install cargo packaging tools (Linux) - name: Install cargo packaging tools (Linux)
@ -88,7 +89,20 @@ jobs:
- name: Install dependencies (Windows) - name: Install dependencies (Windows)
if: matrix.platform == 'windows-latest' if: matrix.platform == 'windows-latest'
shell: pwsh shell: pwsh
run: choco install cmake llvm --installargs 'ADD_CMAKE_TO_PATH=System' -y run: |
choco install cmake llvm --installargs 'ADD_CMAKE_TO_PATH=System' -y
# Ninja: build NeuralAudio with it instead of a Visual Studio generator,
# so the cmake crate doesn't need to recognize the runner's VS version.
choco install ninja -y
# Activate the MSVC developer environment (runs vcvars) so the cmake crate
# building nam-ffi/NeuralAudio can determine the Visual Studio generator.
# Without this, cmake-rs panics with "couldn't determine visual studio generator".
- name: Set up MSVC developer environment (Windows)
if: matrix.platform == 'windows-latest'
uses: ilammy/msvc-dev-cmd@v1
with:
arch: x64
# ── Common build steps ── # ── Common build steps ──
- name: Extract version - name: Extract version
@ -117,14 +131,6 @@ jobs:
# LLVM/libclang for bindgen # LLVM/libclang for bindgen
echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Setup icons
shell: bash
run: |
mkdir -p lightningbeam-ui/lightningbeam-editor/assets/icons
cp -f src-tauri/icons/32x32.png lightningbeam-ui/lightningbeam-editor/assets/icons/
cp -f src-tauri/icons/128x128.png lightningbeam-ui/lightningbeam-editor/assets/icons/
cp -f src-tauri/icons/icon.png lightningbeam-ui/lightningbeam-editor/assets/icons/256x256.png
- name: Stage factory presets - name: Stage factory presets
shell: bash shell: bash
run: | run: |
@ -161,6 +167,7 @@ jobs:
echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> "$GITHUB_ENV" echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> "$GITHUB_ENV"
- name: Build release binary - name: Build release binary
if: matrix.platform != 'windows-latest'
shell: bash shell: bash
env: env:
FFMPEG_STATIC: "1" FFMPEG_STATIC: "1"
@ -172,6 +179,23 @@ jobs:
fi fi
cargo build --release --bin lightningbeam-editor $TARGET_FLAG cargo build --release --bin lightningbeam-editor $TARGET_FLAG
# Windows must build under pwsh, not Git Bash: Git for Windows puts its
# usr\bin on PATH ahead of the MSVC tools, so its coreutils `link.exe`
# shadows MSVC's linker and rustc fails with "extra operand". pwsh keeps
# the MSVC environment (set by msvc-dev-cmd) at the front of PATH.
- name: Build release binary (Windows)
if: matrix.platform == 'windows-latest'
shell: pwsh
env:
FFMPEG_STATIC: "1"
# Force Ninja for the cmake crate (nam-ffi/NeuralAudio): cmake-rs 0.1.54
# panics on "unknown VisualStudio version: 18.0" when it auto-detects the
# runner's VS generator. Ninja sidesteps that lookup entirely.
CMAKE_GENERATOR: Ninja
run: |
cd lightningbeam-ui
cargo build --release --bin lightningbeam-editor
- name: Copy cross-compiled binary to release dir - name: Copy cross-compiled binary to release dir
if: matrix.target != '' if: matrix.target != ''
shell: bash shell: bash
@ -179,6 +203,20 @@ jobs:
mkdir -p lightningbeam-ui/target/release mkdir -p lightningbeam-ui/target/release
cp lightningbeam-ui/target/${{ matrix.target }}/release/lightningbeam-editor lightningbeam-ui/target/release/ cp lightningbeam-ui/target/${{ matrix.target }}/release/lightningbeam-editor lightningbeam-ui/target/release/
# Guard: the zero-copy export needs the static ffmpeg to have h264_vaapi, which only
# happens if libva headers were present at ffmpeg configure time. ffmpeg autodetects it,
# so a missing libva-dev silently ships a software-only build. A vaapi-enabled binary
# links libva — assert that, so the dep can never regress unnoticed.
- name: Verify VAAPI is compiled in (Linux)
if: matrix.platform == 'ubuntu-24.04'
shell: bash
run: |
if ! ldd lightningbeam-ui/target/release/lightningbeam-editor | grep -q 'libva\.so'; then
echo "::error::Release binary does not link libva — ffmpeg was built without VAAPI (is libva-dev installed?)."
exit 1
fi
echo "VAAPI OK: binary links libva."
# ── Stage presets next to binary for packaging ── # ── Stage presets next to binary for packaging ──
- name: Stage presets in target dir - name: Stage presets in target dir
shell: bash shell: bash
@ -296,7 +334,7 @@ jobs:
mkdir -p "$APP/Contents/Resources/presets" mkdir -p "$APP/Contents/Resources/presets"
cp lightningbeam-ui/target/release/lightningbeam-editor "$APP/Contents/MacOS/" cp lightningbeam-ui/target/release/lightningbeam-editor "$APP/Contents/MacOS/"
cp src-tauri/icons/icon.icns "$APP/Contents/Resources/lightningbeam-editor.icns" cp lightningbeam-ui/lightningbeam-editor/assets/icons/icon.icns "$APP/Contents/Resources/lightningbeam-editor.icns"
cp -r lightningbeam-ui/lightningbeam-editor/assets/presets/* "$APP/Contents/Resources/presets/" cp -r lightningbeam-ui/lightningbeam-editor/assets/presets/* "$APP/Contents/Resources/presets/"
cat > "$APP/Contents/Info.plist" << EOF cat > "$APP/Contents/Info.plist" << EOF

2
.gitignore vendored
View File

@ -28,8 +28,6 @@ dist-ssr
.gitignore .gitignore
# Build # Build
src-tauri/gen
src-tauri/target
lightningbeam-core/target lightningbeam-core/target
daw-backend/target daw-backend/target
target/ target/

5
ARCHITECTURE.md
View File

@ -52,7 +52,7 @@ Lightningbeam is a 2D multimedia editor combining vector animation, audio produc
Lightningbeam is undergoing a rewrite from a Tauri/JavaScript prototype to pure Rust. The original architecture hit IPC bandwidth limitations when streaming decoded video frames. The new Rust UI eliminates this bottleneck by handling all rendering natively. Lightningbeam is undergoing a rewrite from a Tauri/JavaScript prototype to pure Rust. The original architecture hit IPC bandwidth limitations when streaming decoded video frames. The new Rust UI eliminates this bottleneck by handling all rendering natively.
**Current Status**: Active development on the `rust-ui` branch. Core UI, tools, and undo system are implemented. Audio integration in progress. **Current Status**: Active development on the `main` branch. Core UI, tools, undo system, and audio integration are implemented.
## Technology Stack ## Technology Stack
@ -486,8 +486,7 @@ lightningbeam-2/
│ ├── synth/ # Synthesizers │ ├── synth/ # Synthesizers
│ └── midi/ # MIDI handling │ └── midi/ # MIDI handling
│ │
├── src-tauri/ # Legacy Tauri backend ├── src/ # Legacy JavaScript frontend (browser-only)
├── src/ # Legacy JavaScript frontend
├── CONTRIBUTING.md # Contributor guide ├── CONTRIBUTING.md # Contributor guide
├── ARCHITECTURE.md # This file ├── ARCHITECTURE.md # This file
├── README.md # Project overview ├── README.md # Project overview

484
BEAM_FILE_FORMAT.md Normal file
View File

@ -0,0 +1,484 @@
# `.beam` File Format Specification
**Status:** Normative.
**Container schema version:** `1` (`meta.schema_version`).
**Project payload version:** `1.0.0` (`BeamProject.version` / `meta.version`).
**Last updated:** 2026-06-21.
This document specifies the on-disk format of Lightningbeam `.beam` project files
precisely enough that an independent implementation can read and write them. It
describes the **current** format — a SQLite database — and the **legacy** format
(a ZIP archive) that current readers still accept and migrate.
> An earlier revision of this document described the ZIP container as the current
> format. That is now historical; see [§11 Legacy ZIP format](#11-legacy-zip-format-pre-sqlite).
## 1. Notational conventions
The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are
to be interpreted as described in RFC 2119.
Byte sizes use binary units: `KiB = 1024`, `MiB = 1024²`, `GiB = 1024³`.
"UUID" means an RFC 4122 128-bit identifier. Unless stated otherwise, a UUID is
serialized in JSON as its canonical hyphenated lowercase string, and stored in
SQLite as its 16 raw bytes in **network (big-endian) byte order** — i.e. the byte
sequence returned by `Uuid::as_bytes`, most significant byte first.
## 2. Overview
A `.beam` file is a **single SQLite 3 database** (default rollback journal; no WAL,
no special pragmas) containing:
- one JSON document, `project.json`, holding all project state that is small and
structural (the document/scene tree, audio project, asset metadata); and
- zero or more **media** items (audio, raster frames, image assets, and derived
blobs such as waveforms and thumbnails), each either **packed** into the database
in chunks or **referenced** by an external file path.
The design goals are: a single beginner-friendly file (no project folder);
streaming reads of large media via SQLite blob I/O; transactional, crash-safe,
**in-place** saves that do not rewrite unchanged media; and inspectability with the
ordinary `sqlite3` CLI.
## 3. File identification
A reader MUST determine the container type from the first 16 bytes of the file:
- If the first 16 bytes equal the ASCII string `SQLite format 3\0` (i.e.
`53 51 4C 69 74 65 20 66 6F 72 6D 61 74 20 33 00`), the file is a SQLite
`.beam` and MUST be read per [§4](#4-container-the-sqlite-schema)–[§10](#10-load-semantics).
- Otherwise the file is treated as a [legacy ZIP `.beam`](#11-legacy-zip-format-pre-sqlite)
(ZIP local-file-header magic `50 4B 03 04`).
The `.beam` extension is used for both container types; the magic bytes — not the
extension — are authoritative.
## 4. Container: the SQLite schema
A conforming SQLite `.beam` MUST contain exactly these four tables (DDL as created;
column order and types are normative):
```sql
CREATE TABLE media (
id BLOB PRIMARY KEY, -- 16-byte UUID (big-endian)
kind INTEGER NOT NULL, -- MediaKind (§6.1)
codec TEXT NOT NULL, -- original codec/container, e.g. "flac","mp3","png"
storage INTEGER NOT NULL, -- MediaStorage (§6.2): 0=Packed, 1=Referenced
ext_path TEXT, -- external path, set iff storage=Referenced
total_len INTEGER NOT NULL DEFAULT 0, -- payload length in bytes (packed); 0 if referenced
channels INTEGER, -- audio: channel count (nullable)
sample_rate INTEGER, -- audio: sample rate in Hz (nullable)
width INTEGER, -- visual media: pixel width (nullable)
height INTEGER -- visual media: pixel height (nullable)
);
CREATE TABLE media_chunk (
id INTEGER PRIMARY KEY, -- rowid (used as the handle for blob streaming)
media_id BLOB NOT NULL, -- → media.id
chunk_index INTEGER NOT NULL, -- 0-based ordinal of this chunk
bytes BLOB NOT NULL, -- chunk payload
UNIQUE (media_id, chunk_index)
);
CREATE TABLE project_json (
id INTEGER PRIMARY KEY CHECK (id = 0), -- exactly one row, id 0
data TEXT NOT NULL -- the serialized BeamProject (§8)
);
CREATE TABLE meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
```
There is no foreign-key constraint between `media_chunk.media_id` and `media.id`;
referential integrity is maintained by the writer. A reader MUST NOT assume SQLite
enforces it.
### 4.1 The `meta` table
Defined keys (all values are strings):
| Key | Value | Meaning |
|------------------|--------------------|---------|
| `schema_version` | `"1"` | Container schema version. Set once at creation. |
| `version` | `"1.0.0"` | Project payload version (mirrors `project.json`'s `version`). |
| `created` | RFC 3339 timestamp | Creation time. Preserved across in-place re-saves. |
| `modified` | RFC 3339 timestamp | Last save time. Updated on every save. |
Writers MAY add additional keys; readers MUST ignore unknown keys.
### 4.2 Two independent version numbers
The format carries **two** version numbers with different compatibility rules:
- **`meta.schema_version`** (`INTEGER`, currently `1`) — versions the SQLite
container/table layout. A reader MUST reject a file whose `schema_version` is
**greater** than the highest it supports, and SHOULD accept any value less than
or equal to its maximum (forward-compatible for older files).
- **`BeamProject.version`** / `meta.version` (currently `"1.0.0"`) — versions the
`project.json` payload. The current implementation requires **exact string
equality** and rejects any other value. A future revision MAY relax this to a
semantic-version range; until then, writers MUST emit exactly `"1.0.0"` and
readers reject anything else.
These two numbers are orthogonal and MUST both be checked.
## 5. UUIDs and identity
Every media item is identified by a UUID. In `project.json` UUIDs appear as
canonical strings; in the `media`/`media_chunk` tables they appear as 16 raw
big-endian bytes (`media.id`, `media_chunk.media_id`). A reader MUST treat the two
representations as equal by their 128-bit value.
`media_chunk.id` is the SQLite rowid and is the handle a reader uses to open a blob
for streaming (`sqlite3_blob_open` on table `media_chunk`, column `bytes`). It has
no meaning beyond row identity and MUST NOT be relied upon to be stable across
re-saves.
## 6. Media model
### 6.1 `MediaKind` (`media.kind`)
| Value | Kind | `codec` examples | Notes |
|-------|---------------|------------------|-------|
| `0` | `Audio` | `flac`, `mp3`, `wav`, `ogg`, `opus`, `aac`, `m4a`, `alac`, `caf`, `aiff` | Source audio for a pool entry. |
| `1` | `Video` | `mp4`, `mov`, … | A video clip's container bytes, keyed by the clip id. Packed under the large-media policy (referenced above 2 GiB unless `Pack`). Frames **and** the embedded audio stream are decoded directly from this blob. |
| `2` | `Raster` | `png` | Full-resolution pixels of a raster keyframe (PNG-encoded RGBA). |
| `3` | `ImageAsset` | `png`, `jpg`, … | An imported image asset's original bytes. |
| `4` | `Waveform` | `lbwf` | Precomputed waveform LOD pyramid for an audio item. Opaque blob owned by `daw_backend::audio::waveform_pyramid`. |
| `5` | `Thumbnail` | `lbtn` | A pack of precomputed video thumbnails for a clip. Opaque blob owned by the editor. |
| `6` | `RasterProxy` | `png` | Low-resolution PNG proxy of a raster keyframe, shown during cold scrubs while full-res pages in. |
A reader MUST reject a `kind` value it does not recognise only if it needs that
item; unknown kinds MAY otherwise be ignored.
### 6.2 `MediaStorage` (`media.storage`)
| Value | Storage | Bytes location | `total_len` | `ext_path` |
|-------|--------------|----------------|-------------|------------|
| `0` | `Packed` | `media_chunk` rows in this DB | payload length | `NULL` |
| `1` | `Referenced` | external file at `ext_path` | `0` | path string |
### 6.3 Packed storage and chunking
Packed bytes are split into chunks of **`CHUNK_SIZE = 4 MiB`** and stored one chunk
per `media_chunk` row, ordered by `chunk_index` ascending starting at `0`. The
writer MUST:
1. set `media.total_len` to the exact total byte length of the payload;
2. write `ceil(total_len / CHUNK_SIZE)` chunk rows (zero rows iff `total_len == 0`);
3. make every chunk exactly `CHUNK_SIZE` bytes **except the last**, which holds the
remainder `total_len − (n−1)·CHUNK_SIZE`.
Because chunk lengths are fully determined by `total_len` and `CHUNK_SIZE`, a reader
performing random access MUST compute, for a byte offset `pos < total_len`:
```
chunk_index = pos / CHUNK_SIZE
offset_in_chunk = pos % CHUNK_SIZE
chunk_len = min(CHUNK_SIZE, total_len − chunk_index·CHUNK_SIZE)
```
and read from the row with that `chunk_index`. A reader MAY stream the whole payload
by concatenating chunk `bytes` in `chunk_index` order. The chunk size MAY differ in
files written by other tools; a robust reader SHOULD derive chunk boundaries from
actual row lengths rather than assuming 4 MiB. (The reference reader assumes uniform
`CHUNK_SIZE` except for the last chunk; writers targeting it MUST keep chunks
uniform.)
### 6.4 Referenced storage
A referenced item stores only `ext_path` (with `total_len = 0` and no chunk rows).
The path is resolved relative to the directory containing the `.beam` file unless it
is absolute. If the file is absent at load time, the item is reported as a *missing
file* ([§10.4](#104-missing-referenced-files)) — this is non-fatal.
## 7. Derived media IDs
Three media kinds are keyed by UUIDs **derived** from another id rather than by an
independent random UUID. A reader/writer MUST compute them exactly as follows
(`from_u128` constructs a UUID from a 128-bit big-endian integer):
| Kind | Derived from | Formula |
|---------------|---------------------|---------|
| `Waveform` | audio pool **index** `i` (`usize`) | `from_u128((0x4C42_5746 << 96) \| (i as u128))` — high 32 bits = `0x4C425746` ("LBWF"), low 96 bits = the pool index. |
| `Thumbnail` | video clip UUID `c` | `from_u128(c.as_u128() ^ 0x4C42_544E_4C42_544E_4C42_544E_4C42_544E)` — full-width XOR with "LBTN" repeated 4×. |
| `RasterProxy` | raster keyframe UUID `k` | `from_u128(k.as_u128() ^ 0x4C42_5058_4C42_5058_4C42_5058_4C42_5058)` — full-width XOR with "LBPX" repeated 4×. |
The full-resolution `Raster` row is keyed by the keyframe's **own** UUID (no
derivation); an `ImageAsset` row is keyed by the asset's own UUID; a packed `Audio`
row is keyed by the pool entry's `media_id`.
> Note the asymmetry: the waveform id ORs the pool index into the high bits, while
> thumbnail/proxy ids XOR a 128-bit sentinel with a source UUID. This is intentional
> (waveforms are keyed by *index*, not by a UUID) but is a sharp edge for
> implementers.
## 8. `project.json`
`project_json.data` is a UTF-8 JSON document: the serde serialization of a
`BeamProject`. This section specifies the top-level structure and the entities a
reader needs to resolve media; the deep UI/scene tree is defined by the
corresponding Rust types (serde field names match struct field names unless a
`#[serde(rename)]` is noted) and is intentionally **not** enumerated exhaustively
here.
### 8.1 `BeamProject` (root)
| Field | Type | Notes |
|-----------------|---------------------------|-------|
| `version` | string | MUST be `"1.0.0"`. |
| `created` | string (RFC 3339) | |
| `modified` | string (RFC 3339) | |
| `ui_state` | `Document` | The scene/document tree (§8.2). |
| `audio_backend` | `SerializedAudioBackend` | Audio engine state (§8.3). |
### 8.2 `Document` (top-level fields)
Collections that carry media linkage are **bold**:
- `id: Uuid`, `name: string`, `background_color`, `width: f64`, `height: f64`,
`framerate: f64`, `duration: f64`
- `time_signature`, `master_layer`, `timeline_mode` — all `#[serde(default)]`
- `root: GraphicsObject` — the layer tree; raster keyframes live here and inside
nested group/clip layers
- **`image_assets: map<Uuid, ImageAsset>`** — keyed by asset UUID (= the media id)
- **`video_clips: map<Uuid, VideoClip>`** — when `VideoClip.media_id` is set the
video is packed (a `Video` media row at the clip id); otherwise `file_path` points
at the external video file. Thumbnails are derived media (§7)
- `vector_clips`, `audio_clips`, `instance_groups`, `effect_definitions`,
`script_definitions`, the various `*_folders` asset trees — structural, no direct
media-row linkage
- `ui_layout`, `ui_layout_base` — `Option`, skipped when `None`
- `current_time`, `layer_to_clip_map` — `#[serde(skip)]` (not persisted)
`RasterKeyframe` (within the layer tree) has `id: Uuid` (= its full-res `Raster`
media id and the seed for its `RasterProxy` id), `time`, `width`, `height`,
`tween_after`, and `stroke_log`. Pixel buffers are `#[serde(skip)]` and faulted in
from media rows. (The legacy ZIP entry path `media/raster/<id>.png` is derived from
`id`; older files may also carry a now-ignored `media_path` field.)
### 8.3 `SerializedAudioBackend`
| Field | Type | Notes |
|----------------------|---------------------|-------|
| `sample_rate` | `u32` | The project's system sample rate (mirrors `AudioProject.sample_rate`). Per-file rates are on each `AudioPoolEntry.sample_rate`. |
| `project` | `AudioProject` | Tracks + MIDI clip pool (§8.4). |
| `audio_pool_entries` | `[AudioPoolEntry]` | Audio source registry (§8.5). |
| `layer_to_track_map` | `map<Uuid, u32>` | UI layer UUID → engine track id. `#[serde(default)]`. |
### 8.4 `AudioProject` (top-level fields)
`tracks: map<TrackId, TrackNode>`, `root_tracks: [TrackId]`, `next_track_id`,
`sample_rate: u32`, `midi_clip_pool`, `next_midi_clip_instance_id`. DSP graphs are
not serialized; they are rebuilt on load.
### 8.5 `AudioPoolEntry` (media linkage)
| Field | Type | Role |
|-----------------|----------------------------|------|
| `pool_index` | `usize` | Stable index; seeds the `Waveform` media id (§7). |
| `name` | string | |
| `media_id` | `Option<string>` (UUID) | Set ⇔ audio bytes are **packed** in this DB under that id. `#[serde(default, skip_serializing_if=None)]`. |
| `relative_path` | `Option<string>` | Set ⇔ audio is **referenced** (external file) or missing. |
| `embedded_data` | `Option<EmbeddedAudioData>`| Legacy/inline: `{ data_base64, format }`. Used when bytes are neither packed nor referenced. |
| `sample_rate` | `u32` | Authoritative sample rate. |
| `channels` | `u32` | Channel count. |
| `duration` | `f64` | Seconds. |
| `is_video_audio`| `bool` | If set, the entry is the audio track of a video. Referenced when the video is external; when the video is **packed**, `media_id` points at the video's `Video` row and the audio streams from that same blob. `#[serde(default, skip_if=false)]`. |
| `waveform_blob` | (transient) | `#[serde(skip)]`; carries waveform bytes in memory only. |
### 8.6 Media linkage summary
| Media kind | Referenced from `project.json` by | Media-row id |
|--------------------|-----------------------------------|--------------|
| Audio (packed) | `AudioPoolEntry.media_id` | that UUID |
| Audio (referenced) | `AudioPoolEntry.relative_path` | — (external) |
| Audio (embedded) | `AudioPoolEntry.embedded_data` | — (inline base64) |
| Raster (full) | `RasterKeyframe.id` | that UUID |
| Raster proxy | derived from `RasterKeyframe.id` | §7 |
| Image asset | `Document.image_assets` key / `ImageAsset.id` | that UUID |
| Video (packed) | `VideoClip.media_id` (= clip id) | that UUID |
| Video (referenced) | `VideoClip.file_path` | — (external) |
| Video thumbnails | derived from `video_clips` key | §7 |
| Waveform | derived from `AudioPoolEntry.pool_index` | §7 |
## 9. Save semantics
A conforming writer MUST perform all media, `project.json`, and `meta` writes for a
save inside **one** SQLite transaction.
### 9.1 In-place vs. create
- If the target path exists **and** is already a SQLite `.beam`, the writer opens it
and writes in place. Unchanged packed media MUST NOT be rewritten (§9.3); this is
the central performance/crash-safety property, and writers MUST NOT use a
copy-to-temp-and-rename flow for in-place saves.
- Otherwise (new file, or migrating a legacy ZIP), the writer creates a fresh DB at
`<path>.beam.tmp`, writes everything, commits, then atomically `rename`s it over
the target.
### 9.2 Order of operations within the transaction
1. Audio pool entries → `Audio` (+ `Waveform`) media rows.
2. Raster keyframes → `Raster` (+ `RasterProxy`) media rows.
3. Video thumbnail packs → `Thumbnail` media rows.
4. Video clips → `Video` media rows (packed/referenced per the large-media policy);
when a clip is packed, its linked video-audio pool entry's `media_id` is set to
the clip's `Video` row so the audio streams from the same blob.
5. Image assets → `ImageAsset` media rows.
6. **Garbage-collect**: delete every `media` row (and its chunks) whose id is **not**
in the set of live ids accumulated in steps 1–5.
7. Write `project.json` and the `meta` keys (`version`, `created`, `modified`).
8. Commit; then (create path only) rename the temp file over the target.
### 9.3 Packed vs. referenced decision (audio)
For each audio pool entry, in order:
- If a packed row for its `media_id` already exists in the archive, keep it untouched
(do not rewrite the bytes) and keep the entry packed.
- Else if its external file resolves and exists: store **referenced** if the entry is
video audio, *or* the file is `≥ LARGE_MEDIA_THRESHOLD` (2 GiB) and the large-media
mode is not `Pack`; otherwise **pack** it (streamed from disk chunk-by-chunk).
- Else if it has `embedded_data`: base64-decode and pack it.
- Else: leave its references as-is (it will be reported missing on load).
Raster, image, and thumbnail media follow the same "write if present, else keep the
existing row" rule so that paged-out content survives a save without being held in
memory.
## 10. Load semantics
### 10.1 Dispatch and version checks
Open the file, read `project.json`, parse `BeamProject`. Reject unless
`version == "1.0.0"` (§4.2). The container's `schema_version` is verified on open
(`≤` the reader's maximum).
### 10.2 Audio resolution
Per pool entry with a `media_id`: look up the media row. If its `codec` is a
**streamable** audio codec (`mp3`, `flac`, `ogg`/`oga`, `wav`/`wave`, `aiff`/`aif`,
`aac`, `m4a`, `opus`, `alac`, `caf`), the bytes are **streamed lazily** from the blob
at playback time (the entry keeps `media_id`, with `embedded_data`/`path` cleared).
Otherwise the bytes are read eagerly into `embedded_data`. A precomputed `Waveform`
blob, if present, is read into the entry.
### 10.3 Visual media paging
- **Raster** full-res pixels are **not** eagerly decoded; each keyframe is flagged
for fault-in and its pixels are paged from its `Raster` row on demand. Its
`RasterProxy` PNG **is** read and decoded eagerly so cold scrubs show a low-res
frame immediately.
- **Image assets** are **not** eagerly read; they page from their `ImageAsset` rows
on demand.
- **Thumbnail** packs are read eagerly per video clip.
### 10.4 Missing referenced files
A pool entry that is neither packed nor embedded, whose `relative_path` resolves to a
non-existent file, MUST be reported as a missing file (non-fatal). The host
application prompts the user to relocate it.
## 11. Legacy ZIP format (pre-SQLite)
Files whose magic is not `SQLite format 3\0` are read as a ZIP archive with this
internal layout:
- `project.json` — the same serialized `BeamProject` (`version` must be `"1.0.0"`).
- `media/audio/<name>.<ext>` — audio source files. The codec is taken from the
extension. **FLAC entries are decoded to PCM `f32` and re-encoded in memory as
IEEE-float WAV (WAV format tag 3)** before being base64-embedded; other formats are
embedded as-is. The entry's `relative_path` is cleared after extraction.
- `media/raster/<uuid>.png` — raster keyframe pixels, named `media/raster/<id>.png`
after the keyframe's `id`.
A reader MUST NOT modify a legacy ZIP on open; it loads into memory only. **The next
save migrates the project to SQLite** (the save sees a non-SQLite target, so it takes
the create-and-rename path and writes a fresh `.beam` database).
> Known limitation of the legacy reader: raster pixels are loaded only for
> **top-level** layers; raster keyframes nested inside groups/clips are not populated
> from a ZIP. The SQLite path recurses correctly. Re-saving to SQLite is the remedy.
## 12. Large-media policy
- `LARGE_MEDIA_THRESHOLD = 2 GiB`. Files at or above it trigger the packed-vs-
referenced decision below; smaller files are always packed (unless video audio,
which is always referenced).
- `LargeMediaMode` ∈ `{ Ask, Pack, Reference }`:
- `Pack` — pack large files into the database.
- `Reference` — store large files by external path.
- `Ask` (default) — prompt the user on the first large import, then persist their
choice; treated as `Reference` at save time until answered.
- **The chosen mode is an application preference, not part of the `.beam` file.** It
is stored in the editor's `config.json` (`AppConfig.large_media_default`) under the
platform config directory, not in the project. Readers/writers of the format need
not consult it except to drive the packed/referenced save decision.
## 13. Conformance summary
A conforming **reader** MUST:
1. dispatch on the 16-byte magic (§3);
2. reject `schema_version` greater than supported and `version != "1.0.0"` (§4.2);
3. resolve packed media via `media`/`media_chunk` (§6.3) and referenced media
relative to the file (§6.4), reporting missing referenced files non-fatally;
4. compute derived media ids exactly as in §7.
A conforming **writer** MUST:
1. produce the four tables of §4 with `schema_version="1"` and `version="1.0.0"`;
2. store UUIDs as 16 big-endian bytes and chunk packed media at a uniform size with a
remainder final chunk, setting `total_len` correctly (§6.3);
3. perform each save in a single transaction and garbage-collect orphaned media (§9),
and prefer in-place writes that do not rewrite unchanged packed media.
## 14. Security considerations
- **Path traversal:** referenced media and legacy ZIP entries carry filesystem paths.
Readers SHOULD resolve relative paths against the project directory and SHOULD
reject or sandbox paths that escape it (`..`, absolute paths) when opening
untrusted files.
- **Resource limits:** `project.json` and packed blobs are attacker-controllable in
an untrusted file. Readers SHOULD bound JSON size and avoid loading entire large
blobs into memory (stream via §6.3) to resist memory-exhaustion.
- **Decoder safety:** audio/image bytes are handed to media decoders (Symphonia,
image, FFmpeg); keep those dependencies current, as they parse untrusted input.
- **Legacy ZIP bombs:** the ZIP path SHOULD enforce sane decompression-ratio/size
limits.
## 15. Non-normative notes / known quirks
These reflect the current reference implementation and are flagged for implementers
and future spec revisions:
- The project `version` check is exact-match with no compatibility window; a future
revision should define semantic-version tolerance before bumping it.
(Earlier revisions of this section flagged a hardcoded save sample rate, a vestigial
`RasterKeyframe.media_path` field, and unused `SaveSettings` fields; these have since
been fixed/removed.)
## 16. Inspecting a `.beam` file
Because the container is plain SQLite, any `.beam` (SQLite variant) can be inspected
with standard tooling, e.g.:
```sh
sqlite3 project.beam '.tables'
sqlite3 project.beam 'SELECT key,value FROM meta;'
sqlite3 project.beam 'SELECT hex(id),kind,codec,storage,total_len FROM media;'
sqlite3 project.beam 'SELECT data FROM project_json;' | jq .
```
## 17. References
- Container: `lightningbeam-ui/lightningbeam-core/src/beam_archive.rs`
- Save/load orchestration & `BeamProject`: `lightningbeam-ui/lightningbeam-core/src/file_io.rs`
- Audio pool entry: `daw-backend/src/audio/pool.rs`
- Document / scene tree: `lightningbeam-ui/lightningbeam-core/src/document.rs`
- RFC 2119 (requirement keywords), RFC 4122 (UUID), RFC 3339 (timestamps)

16
CONTRIBUTING.md
View File

@ -122,25 +122,23 @@ lightningbeam-2/
│ │ ├── track.rs # Track management │ │ ├── track.rs # Track management
│ │ └── project.rs # Project state │ │ └── project.rs # Project state
│ └── effects/ # Audio effects │ └── effects/ # Audio effects
├── src-tauri/ # Legacy Tauri backend └── src/ # Legacy JavaScript frontend (browser-only)
└── src/ # Legacy JavaScript frontend
``` ```
## Making Changes ## Making Changes
### Branching Strategy ### Branching Strategy
- `main` - Stable branch - `main` - Main development branch
- `rust-ui` - Active development branch for Rust UI rewrite - Feature branches - Create from `main` for new features
- Feature branches - Create from `rust-ui` for new features
### Before You Start ### Before You Start
1. Check existing issues or create a new one to discuss your change 1. Check existing issues or create a new one to discuss your change
2. Make sure you're on the latest `rust-ui` branch: 2. Make sure you're on the latest `main` branch:
```bash ```bash
git checkout rust-ui git checkout main
git pull origin rust-ui git pull origin main
``` ```
3. Create a feature branch: 3. Create a feature branch:
```bash ```bash
@ -241,7 +239,7 @@ this solution.
### Pull Request Process ### Pull Request Process
1. Push your branch to GitHub or Gitea 1. Push your branch to GitHub or Gitea
2. Open a pull request against `rust-ui` branch 2. Open a pull request against `main` branch
- GitHub: https://github.com/skykooler/lightningbeam - GitHub: https://github.com/skykooler/lightningbeam
- Gitea: https://git.skyler.io/skyler/lightningbeam - Gitea: https://git.skyler.io/skyler/lightningbeam
3. Provide a clear description of: 3. Provide a clear description of:

11
Changelog.md
View File

@ -1,3 +1,14 @@
# 1.0.6-alpha:
Changes:
- Hardware-accelerated H.264 video export: each frame is rendered and encoded on the GPU (zero-copy VAAPI), roughly 2x faster, with automatic fallback to software encoding when hardware acceleration isn't available (Linux, Intel/AMD only for now)
- Video export now runs on a background thread, so the UI stays responsive during export and edits made while exporting no longer affect the output
- Grouped and nested video clips now composite on the GPU path
- Video is now packed into and streamed from the .beam project container
Bugfixes:
- Fix an export hang when a video's audio track is shorter than the video
- Fix a sample key-range overlap bug in instruments
# 1.0.5-alpha: # 1.0.5-alpha:
Changes: Changes:
- Add shape tweens (morph vector geometry between keyframes) - Add shape tweens (morph vector geometry between keyframes)

2
README.md
View File

@ -48,7 +48,7 @@ A free and open-source 2D multimedia editor combining vector animation, audio pr
## Project Status ## Project Status
Lightningbeam is under active development on the `rust-ui` branch. The project has been rewritten from a Tauri/JavaScript prototype to a pure Rust application to eliminate IPC bottlenecks and achieve better performance for real-time video and audio processing. Lightningbeam is developed on the `main` branch. The project has been rewritten from a Tauri/JavaScript prototype to a pure Rust application to eliminate IPC bottlenecks and achieve better performance for real-time video and audio processing.
**Current Status:** **Current Status:**
- ✅ Core UI panes (Stage, Timeline, Asset Library, Info Panel, Toolbar) - ✅ Core UI panes (Stage, Timeline, Asset Library, Info Panel, Toolbar)

16
STREAMING_TO_DISK_PLAN.md
View File

@ -686,11 +686,17 @@ Phase 1a and Phase 2 can ship now; everything else waits on Phase 0 (the contain
`load_file_into_pool`'s full decode with the `do_import_audio` branching (PCM → mmap, compressed `load_file_into_pool`'s full decode with the `do_import_audio` branching (PCM → mmap, compressed
→ `from_compressed` placeholder). Higher risk (touches the working referenced path); packed → `from_compressed` placeholder). Higher risk (touches the working referenced path); packed
covers the common <2GB case first. covers the common <2GB case first.
- [ ] **Deferred (follow-up): packed video streaming.** Let small videos be packed into the `.beam` - [x] **DONE: packed video streaming.** Small videos pack into the `.beam`
(a `MediaKind::Video` blob, `VideoClip` referencing it by id) and stream **both frames and audio** (a `MediaKind::Video` blob at the clip id, `VideoClip.media_id` referencing it) and stream **both
from the DB blob via FFmpeg. ffmpeg-next has no custom-I/O wrapper, so this needs an frames and audio** from the DB blob via FFmpeg. The `AVIOContext`-over-`Read+Seek` shim lives in
`AVIOContext`-over-`BlobReader` shim via raw FFI. **Decision (user):** that FFI wrapper lives in the new `ffmpeg-blob-io` crate (`BlobInput`, version-pinned `=8.0.0`/`=8.0.1`), isolating the
its **own crate, version-pinned to the ffmpeg version**, isolating the unsafe + the ABI coupling. unsafe + ABI coupling. Frames: `video.rs` `VideoSource{Path,Packed}` opens a fresh `BlobReader`
per decoder/seek/scan. Audio: `VideoAudioReader::open_source` over the same blob (the
`disk_reader.rs` `StreamSource` blocker is removed); save points the linked video-audio pool
entry's `media_id` at the video row so it streams from the same blob. Tests: ffmpeg-blob-io AVIO
unit tests (WAV via Cursor + seek + open/drop loop), core `packed_video_stream` (blob→AVIO→Input),
`beam_archive` packed-video round-trip, daw-backend `open_source` (compiles; can't link in the
container — user runtime-verifies actual A/V playback).
- [~] Phase 1c — video embedded-audio track ← **stopgap shipped; proper design next** - [~] Phase 1c — video embedded-audio track ← **stopgap shipped; proper design next**
- [x] Stopgap: `extract_audio_from_video_to_wav` streams to a temp WAV → `import_audio_sync` - [x] Stopgap: `extract_audio_from_video_to_wav` streams to a temp WAV → `import_audio_sync`
(mmap). Fixed the ~2.8GB-`Vec<f32>` OOM. But writes the whole WAV to `/tmp` (fills (mmap). Fixed the ~2.8GB-`Vec<f32>` OOM. But writes the whole WAV to `/tmp` (fills

701
beam_inspector.py Executable file
View File

@ -0,0 +1,701 @@
#!/usr/bin/env python3
"""
Lightningbeam .beam File Inspector
A command-line tool to inspect .beam project files.
"""
import argparse
import json
import sqlite3
import sys
import uuid as uuidlib
import zipfile
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
# First 16 bytes of any SQLite 3 database.
SQLITE_MAGIC = b"SQLite format 3\x00"
# media.kind values (see BEAM_FILE_FORMAT.md §6.1).
MEDIA_KIND_NAMES = {
0: "Audio", 1: "Video", 2: "Raster", 3: "ImageAsset",
4: "Waveform", 5: "Thumbnail", 6: "RasterProxy",
}
# media.storage values (§6.2).
MEDIA_STORAGE_NAMES = {0: "Packed", 1: "Referenced"}
def _is_sqlite(path: Path) -> bool:
"""True if the file begins with the SQLite 3 header magic."""
try:
with open(path, "rb") as f:
return f.read(16) == SQLITE_MAGIC
except OSError:
return False
def _human_size(n: int) -> str:
f = float(n)
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
if f < 1024 or unit == "TiB":
return f"{f:.0f} {unit}" if unit == "B" else f"{f:.1f} {unit}"
f /= 1024
return f"{n} B"
class BeamInspector:
"""Inspector for .beam project files (SQLite container, or legacy ZIP)."""
def __init__(self, beam_file: Path):
self.beam_file = beam_file
self.project_data: Optional[Dict[str, Any]] = None
# SQLite (current) vs ZIP (legacy) container.
self.is_sqlite = _is_sqlite(beam_file)
# Populated for SQLite files by load():
self.meta: Dict[str, str] = {}
self.media: List[Dict[str, Any]] = [] # rows from the media table
self.media_by_id: Dict[str, Dict[str, Any]] = {} # uuid-string -> row
def _connect(self) -> sqlite3.Connection:
"""Open the SQLite container read-only."""
uri = self.beam_file.resolve().as_uri() + "?mode=ro"
return sqlite3.connect(uri, uri=True)
def load(self) -> bool:
"""Load and parse the .beam file (container-agnostic)."""
try:
if self.is_sqlite:
return self._load_sqlite()
return self._load_zip()
except Exception as e:
print(f"Error loading .beam file: {e}", file=sys.stderr)
return False
def _load_zip(self) -> bool:
with zipfile.ZipFile(self.beam_file, 'r') as zip_ref:
with zip_ref.open('project.json') as f:
self.project_data = json.load(f)
return True
def _load_sqlite(self) -> bool:
con = self._connect()
try:
cur = con.cursor()
row = cur.execute("SELECT data FROM project_json WHERE id = 0").fetchone()
if not row:
raise ValueError("archive has no project.json row")
self.project_data = json.loads(row[0])
self.meta = {k: v for (k, v) in cur.execute("SELECT key, value FROM meta")}
for (idb, kind, codec, storage, ext_path, total_len,
channels, sample_rate, width, height) in cur.execute(
"SELECT id, kind, codec, storage, ext_path, total_len, "
"channels, sample_rate, width, height FROM media"
):
uid = str(uuidlib.UUID(bytes=bytes(idb)))
info = {
"uuid": uid,
"kind": kind,
"codec": codec,
"storage": storage,
"ext_path": ext_path,
"total_len": total_len or 0,
"channels": channels,
"sample_rate": sample_rate,
"width": width,
"height": height,
}
self.media.append(info)
self.media_by_id[uid] = info
finally:
con.close()
return True
def show_info(self):
"""Display basic project information."""
if not self.project_data:
return
print("=" * 60)
print("PROJECT INFORMATION")
print("=" * 60)
print(f"Container: {'SQLite' if self.is_sqlite else 'Legacy ZIP'}")
if self.is_sqlite:
print(f"Schema Ver: {self.meta.get('schema_version', 'Unknown')}")
print(f"Version: {self.project_data.get('version', 'Unknown')}")
print(f"Created: {self.project_data.get('created', 'Unknown')}")
print(f"Modified: {self.project_data.get('modified', 'Unknown')}")
ui_state = self.project_data.get('ui_state', {})
print(f"\nProject Name: {ui_state.get('name', 'Unnamed')}")
print(f"ID: {ui_state.get('id', 'Unknown')}")
print(f"Dimensions: {ui_state.get('width', 0):.0f} x {ui_state.get('height', 0):.0f}")
print(f"Framerate: {ui_state.get('framerate', 0):.1f} fps")
print(f"Duration: {ui_state.get('duration', 0):.2f} seconds")
bg = ui_state.get('background_color', {})
print(f"Background: rgba({bg.get('r', 0)}, {bg.get('g', 0)}, {bg.get('b', 0)}, {bg.get('a', 255)})")
audio_backend = self.project_data.get('audio_backend', {})
print(f"\nSample Rate: {audio_backend.get('sample_rate', 0)} Hz")
def show_clips(self):
"""Display clips information."""
if not self.project_data:
return
ui_state = self.project_data.get('ui_state', {})
vector_clips = ui_state.get('vector_clips', {})
video_clips = ui_state.get('video_clips', {})
audio_clips = ui_state.get('audio_clips', {})
print("\n" + "=" * 60)
print("CLIPS")
print("=" * 60)
print(f"Vector Clips: {len(vector_clips)}")
print(f"Video Clips: {len(video_clips)}")
print(f"Audio Clips: {len(audio_clips)}")
if vector_clips:
print("\nVector Clips:")
for clip_id, clip in vector_clips.items():
print(f" - {clip.get('name', 'Unnamed')} (ID: {clip_id[:8]}...)")
if video_clips:
print("\nVideo Clips:")
for clip_id, clip in video_clips.items():
print(f" - {clip.get('name', 'Unnamed')} (ID: {clip_id[:8]}...)")
if audio_clips:
print("\nAudio Clips:")
for clip_id, clip in audio_clips.items():
print(f" - {clip.get('name', 'Unnamed')} (ID: {clip_id[:8]}...)")
def show_layers(self):
"""Display layer hierarchy."""
if not self.project_data:
return
ui_state = self.project_data.get('ui_state', {})
root = ui_state.get('root', {})
print("\n" + "=" * 60)
print("LAYER HIERARCHY")
print("=" * 60)
def print_layer(layer: Dict[str, Any], indent: int = 0):
# Handle case where layer might not be a dictionary
if not isinstance(layer, dict):
prefix = " " * indent
print(f"{prefix}- ERROR: Layer is {type(layer).__name__}, not a dict: {repr(layer)[:50]}")
return
layer_type = layer.get('type', 'Unknown')
layer_name = layer.get('name', 'Unnamed')
layer_id = layer.get('id', 'Unknown')
prefix = " " * indent
# Handle layer_id that might not be a string
id_str = layer_id[:8] + "..." if isinstance(layer_id, str) and len(layer_id) > 8 else str(layer_id)
print(f"{prefix}- [{layer_type}] {layer_name} (ID: {id_str})")
# Recursively print children
children = layer.get('children', [])
for child in children:
print_layer(child, indent + 1)
print_layer(root)
def show_tracks(self):
"""Display audio tracks information."""
if not self.project_data:
return
audio_backend = self.project_data.get('audio_backend', {})
project = audio_backend.get('project', {})
tracks_dict = project.get('tracks', {})
root_tracks = project.get('root_tracks', [])
master_track = project.get('master_track', {})
print("\n" + "=" * 60)
print("AUDIO TRACKS")
print("=" * 60)
print(f"Total Tracks: {len(tracks_dict)}")
# Iterate through root_tracks in order
for track_id in root_tracks:
track_id_str = str(track_id)
if track_id_str not in tracks_dict:
print(f"\n Track {track_id}: ERROR - Track ID not found in tracks dict")
continue
track_node = tracks_dict[track_id_str]
# TrackNode is an enum with variants like {"Audio": {...}} or {"Midi": {...}}
if not isinstance(track_node, dict):
print(f"\n Track {track_id}: ERROR - Track node is {type(track_node).__name__}, not a dict")
continue
# Extract the variant (Audio, Midi, or Group)
if len(track_node) != 1:
print(f"\n Track {track_id}: ERROR - Track node has unexpected structure")
continue
track_type, track_data = list(track_node.items())[0]
if not isinstance(track_data, dict):
print(f"\n Track {track_id}: ERROR - Track data is {type(track_data).__name__}, not a dict")
continue
track_name = track_data.get('name', f'Track {track_id}')
muted = track_data.get('muted', False)
solo = track_data.get('solo', False)
volume = track_data.get('volume', 1.0)
pan = track_data.get('pan', 0.0)
status = []
if muted:
status.append('MUTED')
if solo:
status.append('SOLO')
status_str = f" [{', '.join(status)}]" if status else ""
print(f"\n Track {track_id}: {track_name}{status_str}")
print(f" Type: {track_type}")
print(f" Volume: {volume:.2f}")
print(f" Pan: {pan:.2f}")
if track_type == "Midi":
print(f" Instrument: {track_data.get('instrument', 'Unknown')}")
notes = track_data.get('notes', [])
print(f" Notes: {len(notes)}")
preset = track_data.get('instrument_graph_preset')
if preset:
nodes = preset.get('nodes', [])
conns = preset.get('connections', [])
print(f" Graph: {len(nodes)} nodes, {len(conns)} connections")
else:
print(f" Graph: (no preset saved)")
elif track_type == "Audio":
clips = track_data.get('clips', [])
print(f" Clips: {len(clips)}")
preset = track_data.get('effects_graph_preset')
if preset:
nodes = preset.get('nodes', [])
conns = preset.get('connections', [])
print(f" Graph: {len(nodes)} nodes, {len(conns)} connections")
else:
print(f" Graph: (no preset saved)")
print(f"\nMaster Track:")
print(f" Volume: {master_track.get('volume', 1.0):.2f}")
def show_graphs(self):
"""Display detailed node graph information for all tracks."""
if not self.project_data:
return
audio_backend = self.project_data.get('audio_backend', {})
project = audio_backend.get('project', {})
tracks_dict = project.get('tracks', {})
print("\n" + "=" * 60)
print("NODE GRAPHS")
print("=" * 60)
for track_id_str, track_node in tracks_dict.items():
if not isinstance(track_node, dict) or len(track_node) != 1:
continue
track_type, track_data = list(track_node.items())[0]
track_name = track_data.get('name', f'Track {track_id_str}')
# Get the appropriate preset
if track_type == "Audio":
preset = track_data.get('effects_graph_preset')
graph_label = "Effects Graph"
elif track_type == "Midi":
preset = track_data.get('instrument_graph_preset')
graph_label = "Instrument Graph"
else:
continue
print(f"\n Track {track_id_str}: {track_name} ({track_type}) - {graph_label}")
if not preset:
print(f" ** NO PRESET SAVED **")
continue
nodes = preset.get('nodes', [])
connections = preset.get('connections', [])
output_node = preset.get('output_node')
midi_targets = preset.get('midi_targets', [])
metadata = preset.get('metadata', {})
if metadata:
print(f" Preset Name: {metadata.get('name', 'Unknown')}")
print(f" Nodes ({len(nodes)}):")
for i, node in enumerate(nodes):
node_type = node.get('node_type', '?')
node_name = node.get('name', node_type)
params = node.get('parameters', {})
pos_x = node.get('position_x', 0)
pos_y = node.get('position_y', 0)
markers = []
if output_node is not None and i == output_node:
markers.append("OUTPUT")
if i in midi_targets:
markers.append("MIDI TARGET")
marker_str = f" [{', '.join(markers)}]" if markers else ""
print(f" [{i}] {node_type}{marker_str} (name={node_name}, pos={pos_x:.0f},{pos_y:.0f})")
if params:
for param_name, param_val in params.items():
print(f" {param_name} = {param_val}")
print(f" Connections ({len(connections)}):")
for conn in connections:
src = conn.get('from_node', '?')
src_port = conn.get('from_output', '?')
dst = conn.get('to_node', '?')
dst_port = conn.get('to_input', '?')
print(f" [{src}]:{src_port} -> [{dst}]:{dst_port}")
# Also show layer_to_track_map if present
track_map = audio_backend.get('layer_to_track_map', {})
if track_map:
print(f"\n Layer-to-Track Mapping ({len(track_map)} entries):")
for layer_id, track_id in track_map.items():
print(f" {layer_id[:8]}... -> Track {track_id}")
else:
print(f"\n Layer-to-Track Mapping: (none saved)")
def show_audio_pool(self):
"""Display audio pool entries."""
if not self.project_data:
return
audio_backend = self.project_data.get('audio_backend', {})
pool_entries = audio_backend.get('audio_pool_entries', [])
print("\n" + "=" * 60)
print("AUDIO POOL")
print("=" * 60)
print(f"Total Entries: {len(pool_entries)}")
for entry in pool_entries:
pool_index = entry.get('pool_index', '?')
name = entry.get('name', 'Unnamed')
relative_path = entry.get('relative_path')
media_id = entry.get('media_id')
channels = entry.get('channels', 0)
sample_rate = entry.get('sample_rate', 0)
has_embedded = entry.get('embedded_data') is not None
# Storage precedence matches the loader (§8.5/§9.3):
# packed (media_id) > external (relative_path) > embedded > unresolved.
row = self.media_by_id.get(media_id) if media_id else None
if media_id:
size = f", {_human_size(row['total_len'])}" if row else ""
codec = f", {row['codec']}" if row else ""
storage_type = f"Packed in DB{codec}{size}"
elif relative_path and not self.is_sqlite and relative_path.startswith("media/audio/"):
# Legacy ZIP: media/audio/* lives inside the archive, not external.
storage_type = "Embedded (in ZIP)"
elif relative_path:
storage_type = "External reference"
elif has_embedded:
storage_type = "Embedded (inline base64)"
else:
storage_type = "Unresolved (missing)"
print(f"\n [{pool_index}] {name}")
if media_id:
print(f" Media ID: {media_id}")
print(f" Path: {relative_path if relative_path else 'N/A'}")
print(f" Storage: {storage_type}")
print(f" Channels: {channels}")
print(f" Sample Rate: {sample_rate} Hz")
def show_media(self):
"""Display the SQLite media store (the `media` table)."""
if not self.is_sqlite:
print("\n(media table view applies to SQLite .beam files; "
"use --zip for legacy archives)", file=sys.stderr)
return
print("\n" + "=" * 60)
print("MEDIA STORE")
print("=" * 60)
print(f"Total media rows: {len(self.media)}")
# Summary by kind.
by_kind: Dict[str, List[Dict[str, Any]]] = {}
for m in self.media:
by_kind.setdefault(MEDIA_KIND_NAMES.get(m["kind"], f"Kind {m['kind']}"), []).append(m)
if by_kind:
print("\nBy kind:")
for kind_name, rows in sorted(by_kind.items()):
packed = sum(r["total_len"] for r in rows if r["storage"] == 0)
print(f" {kind_name:<12} {len(rows):>4} ({_human_size(packed)} packed)")
print(f"\n{'Kind':<12} {'Storage':<11} {'Codec':<6} {'Size':>10} UUID / details")
print("-" * 78)
for m in self.media:
kind = MEDIA_KIND_NAMES.get(m["kind"], f"Kind {m['kind']}")
storage = MEDIA_STORAGE_NAMES.get(m["storage"], f"Stor {m['storage']}")
size = _human_size(m["total_len"]) if m["storage"] == 0 else "-"
detail = m["uuid"]
extra = []
if m["channels"] is not None:
extra.append(f"{m['channels']}ch@{m['sample_rate']}Hz")
if m["width"] is not None:
extra.append(f"{m['width']}x{m['height']}")
if m["storage"] == 1 and m["ext_path"]:
extra.append(f"-> {m['ext_path']}")
if extra:
detail += " " + " ".join(extra)
print(f"{kind:<12} {storage:<11} {m['codec']:<6} {size:>10} {detail}")
def show_container(self):
"""Show whichever container structure applies to this file."""
if self.is_sqlite:
self.show_media()
else:
self.show_zip_structure()
def show_zip_structure(self):
"""Display the ZIP file structure."""
if self.is_sqlite:
print("\n(this is a SQLite .beam; use --media for the media store)",
file=sys.stderr)
return
print("\n" + "=" * 60)
print("ZIP ARCHIVE STRUCTURE")
print("=" * 60)
try:
with zipfile.ZipFile(self.beam_file, 'r') as zip_ref:
total_size = 0
compressed_size = 0
print(f"{'File':<40} {'Size':>12} {'Compressed':>12} {'Method':<10}")
print("-" * 80)
for info in zip_ref.infolist():
size = info.file_size
comp_size = info.compress_size
method = "DEFLATE" if info.compress_type == 8 else "STORED" if info.compress_type == 0 else f"Type {info.compress_type}"
total_size += size
compressed_size += comp_size
print(f"{info.filename:<40} {size:>12,} {comp_size:>12,} {method:<10}")
print("-" * 80)
print(f"{'TOTAL':<40} {total_size:>12,} {compressed_size:>12,}")
if total_size > 0:
ratio = (1 - compressed_size / total_size) * 100
print(f"\nCompression Ratio: {ratio:.1f}%")
except Exception as e:
print(f"Error reading ZIP structure: {e}", file=sys.stderr)
def extract_json(self, output_path: Optional[Path] = None):
"""Extract project.json to a file or stdout."""
try:
if self.is_sqlite:
con = self._connect()
try:
row = con.execute("SELECT data FROM project_json WHERE id = 0").fetchone()
finally:
con.close()
if not row:
raise ValueError("archive has no project.json row")
json_data = row[0].encode("utf-8") if isinstance(row[0], str) else row[0]
else:
with zipfile.ZipFile(self.beam_file, 'r') as zip_ref:
json_data = zip_ref.read('project.json')
if output_path:
output_path.write_bytes(json_data)
print(f"Extracted project.json to: {output_path}")
else:
data = json.loads(json_data)
print(json.dumps(data, indent=2))
except Exception as e:
print(f"Error extracting project.json: {e}", file=sys.stderr)
def extract_media(self, output_dir: Path):
"""Extract all media to a directory.
SQLite: each packed media row is reassembled from its chunks and written
as ``<uuid>.<codec>``; referenced rows are reported, not copied.
ZIP (legacy): the ``media/`` entries are extracted preserving paths.
"""
try:
if self.is_sqlite:
self._extract_media_sqlite(output_dir)
else:
self._extract_media_zip(output_dir)
except Exception as e:
print(f"Error extracting media: {e}", file=sys.stderr)
def _extract_media_sqlite(self, output_dir: Path):
con = self._connect()
try:
rows = con.execute(
"SELECT id, kind, codec, storage, ext_path, total_len FROM media"
).fetchall()
if not rows:
print("No media rows found in archive.")
return
output_dir.mkdir(parents=True, exist_ok=True)
written = 0
for (idb, kind, codec, storage, ext_path, total_len) in rows:
uid = str(uuidlib.UUID(bytes=bytes(idb)))
kind_name = MEDIA_KIND_NAMES.get(kind, f"kind{kind}")
if storage == 1: # Referenced
print(f"Referenced (not copied): {uid} [{kind_name}] -> {ext_path}")
continue
chunks = con.execute(
"SELECT bytes FROM media_chunk WHERE media_id = ? ORDER BY chunk_index",
(idb,),
).fetchall()
data = b"".join(bytes(c[0]) for c in chunks)
out = output_dir / f"{uid}.{codec}"
out.write_bytes(data)
print(f"Extracted: {out.name} [{kind_name}] {_human_size(len(data))}")
written += 1
print(f"\nExtracted {written} packed media item(s) to: {output_dir}")
finally:
con.close()
def _extract_media_zip(self, output_dir: Path):
with zipfile.ZipFile(self.beam_file, 'r') as zip_ref:
media_files = [f for f in zip_ref.namelist() if f.startswith('media/')]
if not media_files:
print("No media files found in archive.")
return
output_dir.mkdir(parents=True, exist_ok=True)
for media_file in media_files:
output_path = output_dir / media_file
output_path.parent.mkdir(parents=True, exist_ok=True)
with zip_ref.open(media_file) as source:
output_path.write_bytes(source.read())
print(f"Extracted: {media_file}")
print(f"\nExtracted {len(media_files)} media file(s) to: {output_dir}")
def main():
parser = argparse.ArgumentParser(
description="Inspect Lightningbeam .beam project files",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s project.beam # Show all information
%(prog)s project.beam --info # Show only basic info
%(prog)s project.beam --tracks # Show only tracks
%(prog)s project.beam --media # Show the SQLite media store
%(prog)s project.beam --zip # Show legacy ZIP structure
%(prog)s project.beam --extract-json # Print project.json to stdout
%(prog)s project.beam --extract-json out.json # Save project.json
%(prog)s project.beam --extract-media ./media # Extract media files
Handles both current SQLite .beam files and legacy ZIP .beam files
(detected automatically by the file's magic bytes).
"""
)
parser.add_argument('beam_file', type=Path, help='.beam file to inspect')
parser.add_argument('--info', action='store_true', help='Show project information')
parser.add_argument('--clips', action='store_true', help='Show clips')
parser.add_argument('--layers', action='store_true', help='Show layer hierarchy')
parser.add_argument('--tracks', action='store_true', help='Show audio tracks')
parser.add_argument('--graphs', action='store_true', help='Show node graph details for all tracks')
parser.add_argument('--pool', action='store_true', help='Show audio pool')
parser.add_argument('--media', action='store_true', help='Show the SQLite media store')
parser.add_argument('--zip', action='store_true', help='Show legacy ZIP structure')
parser.add_argument('--extract-json', nargs='?', const=True, metavar='OUTPUT',
help='Extract project.json (to file or stdout)')
parser.add_argument('--extract-media', type=Path, metavar='DIR',
help='Extract media files to directory')
args = parser.parse_args()
# Validate input file
if not args.beam_file.exists():
print(f"Error: File not found: {args.beam_file}", file=sys.stderr)
sys.exit(1)
if not args.beam_file.suffix == '.beam':
print(f"Warning: File does not have .beam extension: {args.beam_file}", file=sys.stderr)
# Create inspector
inspector = BeamInspector(args.beam_file)
# Handle extract operations (don't need to load project.json)
if args.extract_json:
if args.extract_json is True:
inspector.extract_json()
else:
inspector.extract_json(Path(args.extract_json))
return
if args.extract_media:
inspector.extract_media(args.extract_media)
return
# Load the beam file
if not inspector.load():
sys.exit(1)
# If no specific flags, show everything
show_all = not any([args.info, args.clips, args.layers, args.tracks,
args.graphs, args.pool, args.media, args.zip])
if show_all or args.info:
inspector.show_info()
if show_all or args.clips:
inspector.show_clips()
if show_all or args.layers:
inspector.show_layers()
if show_all or args.tracks:
inspector.show_tracks()
if show_all or args.graphs:
inspector.show_graphs()
if show_all or args.pool:
inspector.show_audio_pool()
if show_all:
# Show whichever container structure applies to this file.
inspector.show_container()
elif args.media:
inspector.show_media()
elif args.zip:
inspector.show_zip_structure()
if __name__ == '__main__':
main()

210
beam_inspector_README.md Normal file
View File

@ -0,0 +1,210 @@
# Lightningbeam .beam File Inspector
A Python command-line tool to inspect and analyze `.beam` project files.
Handles **both** container formats automatically (detected by the file's magic
bytes): the current **SQLite** `.beam` and the **legacy ZIP** `.beam`. See
[BEAM_FILE_FORMAT.md](./BEAM_FILE_FORMAT.md) for the format specification.
## Features
- **Project Information**: Container type, schema version, metadata, dimensions, framerate, duration, background color
- **Clips Analysis**: List all vector, video, and audio clips
- **Layer Hierarchy**: Display the complete layer tree structure
- **Audio Tracks**: Show all audio/MIDI tracks with their settings
- **Audio Pool**: List all audio files and their storage details
- **Media Store** (SQLite): List the `media` table — kinds, storage, codecs, sizes
- **ZIP Structure** (legacy): Examine the internal ZIP archive structure and compression
- **Extraction**: Extract `project.json` or media files
## Installation
The tool requires Python 3.6+ with no external dependencies (uses only the standard
library; `sqlite3` ships with Python).
```bash
chmod +x beam_inspector.py
```
## Usage
### Show All Information
```bash
./beam_inspector.py project.beam
```
### Show Specific Sections
```bash
# Basic project info only
./beam_inspector.py project.beam --info
# Audio tracks only
./beam_inspector.py project.beam --tracks
# Audio pool entries
./beam_inspector.py project.beam --pool
# Layer hierarchy
./beam_inspector.py project.beam --layers
# Clips summary
./beam_inspector.py project.beam --clips
# Media store (SQLite files): the media table — kinds, storage, codecs, sizes
./beam_inspector.py project.beam --media
# ZIP archive structure (legacy ZIP files only)
./beam_inspector.py project.beam --zip
```
When run with no section flags, the tool shows everything and automatically
picks the media-store view (SQLite) or ZIP-structure view (legacy) for the file.
### Extract Files
```bash
# Print project.json to stdout (pretty-printed)
./beam_inspector.py project.beam --extract-json
# Save project.json to a file
./beam_inspector.py project.beam --extract-json output.json
# Extract all media files to a directory
./beam_inspector.py project.beam --extract-media ./extracted_media
```
## Example Output
```
============================================================
PROJECT INFORMATION
============================================================
Container: SQLite
Schema Ver: 1
Version: 1.0.0
Created: 2025-12-01T12:00:00Z
Modified: 2025-12-01T12:30:00Z
Project Name: My Animation
ID: 550e8400-e29b-41d4-a716-446655440000
Dimensions: 1920 x 1080
Framerate: 60.0 fps
Duration: 10.00 seconds
Background: rgba(255, 255, 255, 255)
Sample Rate: 48000 Hz
============================================================
AUDIO TRACKS
============================================================
Total Tracks: 2
Track 0: Piano [SOLO]
Type: Midi
Volume: 0.80
Pan: 0.00
Instrument: Piano
Notes: 24
Track 1: Background Music
Type: Audio
Volume: 0.60
Pan: 0.00
Clips: 3
Master Track:
Volume: 1.00
============================================================
AUDIO POOL
============================================================
Total Entries: 2
[0] C2.mp3
Media ID: e7e555a6-85f1-4faa-bfd4-c6e4b790986a
Path: N/A
Storage: Packed in DB, mp3, 412.0 KiB
Channels: 2
Sample Rate: 44100 Hz
[1] Background.flac
Media ID: a18c0e22-1d3b-4c77-9f0a-2b5e6c4d8e10
Path: N/A
Storage: Packed in DB, flac, 6.4 MiB
Channels: 2
Sample Rate: 48000 Hz
```
## Understanding the Output
### Storage Types
- **Packed in DB**: Audio bytes are chunked into the SQLite `media` table (the entry's `media_id` resolves to a packed row). Current default for most audio.
- **External reference**: File is referenced from the filesystem by `relative_path` (e.g. large media or video audio).
- **Embedded (inline base64)**: Bytes stored directly in `project.json` (`embedded_data`) — legacy/fallback.
- **Embedded (in ZIP)**: Legacy ZIP files only — bytes stored inside the archive under `media/audio/`.
- **Unresolved (missing)**: No packed row, external file, or embedded data — reported as a missing file on load.
### Track Types
- **Audio**: Traditional audio track with clips from the audio pool
- **Midi**: MIDI track with note events and virtual instrument
### Layer Types
Based on the `AnyLayer` enum:
- **Group**: Container for other layers
- **Vector**: Vector graphics layer
- **Video**: Video clip layer
- **Audio**: Audio waveform layer
- **Image**: Raster image layer
- **Text**: Text layer
## Advanced Usage
### Combine with Other Tools
```bash
# Pretty-print and page through JSON
./beam_inspector.py project.beam --extract-json | less
# Search for specific content in JSON
./beam_inspector.py project.beam --extract-json | grep "sample_rate"
# Count total audio files
./beam_inspector.py project.beam --pool | grep "^\[" | wc -l
# Extract and process media files
# SQLite: writes flat <uuid>.<codec> files; ZIP: preserves media/ paths
./beam_inspector.py project.beam --extract-media /tmp/media
ls -lh /tmp/media/
```
### Scripting
```python
from beam_inspector import BeamInspector
from pathlib import Path
inspector = BeamInspector(Path("project.beam"))
if inspector.load():
# Access parsed data
version = inspector.project_data['version']
tracks = inspector.project_data['audio_backend']['project']['tracks']
print(f"Found {len(tracks)} tracks in version {version}")
```
## Troubleshooting
### "Error loading .beam file"
- Ensure the file is a valid SQLite database (current) or ZIP archive (legacy)
- Check that `project.json` exists (the `project_json` table for SQLite, or a top-level entry for ZIP)
- Verify the JSON is well-formed
### "File not found"
- Provide the full path to the .beam file
- Check file permissions
### Missing media files
- External references may point to files that don't exist
- Use `--extract-media` to see what's actually in the archive
- Check the `relative_path` values in `--pool` output
## See Also
- [BEAM_FILE_FORMAT.md](./BEAM_FILE_FORMAT.md) - Complete .beam file format specification
- [Lightningbeam Documentation](./README.md) - Main project documentation

4
create_release.sh
View File

@ -26,7 +26,9 @@ echo "Merging $MAIN_BRANCH into $RELEASE_BRANCH..."
git merge $MAIN_BRANCH --no-ff -m "Release $VERSION" git merge $MAIN_BRANCH --no-ff -m "Release $VERSION"
echo "Pushing $RELEASE_BRANCH..." echo "Pushing $RELEASE_BRANCH..."
git push origin $RELEASE_BRANCH # Push to the 'all' remote so the release branch lands on both GitHub and Gitea.
# CI (GitHub Actions) still triggers via the GitHub pushurl.
git push all $RELEASE_BRANCH
git checkout $MAIN_BRANCH git checkout $MAIN_BRANCH

2
daw-backend/Cargo.toml
View File

@ -23,6 +23,8 @@ memmap2 = "0.9"
# Audio export # Audio export
hound = "3.5" hound = "3.5"
ffmpeg-next = "8.0" # For MP3/AAC encoding ffmpeg-next = "8.0" # For MP3/AAC encoding
# AVIO-over-Read+Seek shim: stream a packed video's audio track from the SQLite blob.
ffmpeg-blob-io = { path = "../ffmpeg-blob-io" }
# Node-based audio graph dependencies # Node-based audio graph dependencies
dasp_graph = "0.11" dasp_graph = "0.11"

84
daw-backend/src/audio/disk_reader.rs
View File

@ -77,6 +77,12 @@ pub struct ReadAheadBuffer {
/// When true, `render_from_file` will block-wait for frames instead of /// When true, `render_from_file` will block-wait for frames instead of
/// returning silence on buffer miss. Used during offline export. /// returning silence on buffer miss. Used during offline export.
export_mode: AtomicBool, export_mode: AtomicBool,
/// Set by the disk reader when the source reaches EOF (no more frames will
/// ever be decoded past the current valid range). Lets export-mode block-waits
/// give up immediately for frames past the end of the audio (e.g. a video whose
/// audio track is shorter than the video) instead of timing out per chunk.
/// Cleared on `reset` (a seek may decode fresh data again).
finished: AtomicBool,
} }
// SAFETY: See the doc comment on ReadAheadBuffer for the full safety argument. // SAFETY: See the doc comment on ReadAheadBuffer for the full safety argument.
@ -112,6 +118,7 @@ impl ReadAheadBuffer {
sample_rate, sample_rate,
target_frame: AtomicU64::new(0), target_frame: AtomicU64::new(0),
export_mode: AtomicBool::new(false), export_mode: AtomicBool::new(false),
finished: AtomicBool::new(false),
} }
} }
@ -216,11 +223,24 @@ impl ReadAheadBuffer {
self.export_mode.load(Ordering::Acquire) self.export_mode.load(Ordering::Acquire)
} }
/// Mark that the source has reached EOF (set by the disk reader).
pub fn set_finished(&self, finished: bool) {
self.finished.store(finished, Ordering::Release);
}
/// True once the source hit EOF: no frames past the current valid range
/// will ever be decoded (until a `reset`/seek).
pub fn is_finished(&self) -> bool {
self.finished.load(Ordering::Acquire)
}
/// Reset the buffer to start at `new_start` with zero valid frames. /// Reset the buffer to start at `new_start` with zero valid frames.
/// Called by the **disk reader thread** (producer) after a seek. /// Called by the **disk reader thread** (producer) after a seek.
pub fn reset(&self, new_start: u64) { pub fn reset(&self, new_start: u64) {
self.valid_frames.store(0, Ordering::Release); self.valid_frames.store(0, Ordering::Release);
self.start_frame.store(new_start, Ordering::Release); self.start_frame.store(new_start, Ordering::Release);
// A seek may decode fresh data past the previous EOF position.
self.finished.store(false, Ordering::Release);
} }
/// Write interleaved samples into the buffer, extending the valid range. /// Write interleaved samples into the buffer, extending the valid range.
@ -571,8 +591,37 @@ impl CompressedReader {
/// ///
/// Public (vs. the private `CompressedReader`) only so integration tests can /// Public (vs. the private `CompressedReader`) only so integration tests can
/// exercise it directly; treat it as crate-internal. /// exercise it directly; treat it as crate-internal.
/// Adapts a host `MediaByteSource` (Read+Seek+Send+Sync) to the `ffmpeg-blob-io`
/// `BlobSource` (Read+Seek+Send) so a packed video can be demuxed from a blob.
struct ByteSourceAdapter(Box<dyn MediaByteSource>);
impl std::io::Read for ByteSourceAdapter {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.0.read(buf)
}
}
impl std::io::Seek for ByteSourceAdapter {
fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
self.0.seek(pos)
}
}
/// A demuxer input for video-audio: a video file path, or a container blob streamed
/// via the AVIO shim. Both expose the same ffmpeg `Input`.
enum AudioInput {
Path(ffmpeg_next::format::context::Input),
Blob(ffmpeg_blob_io::BlobInput),
}
impl AudioInput {
fn get(&mut self) -> &mut ffmpeg_next::format::context::Input {
match self {
AudioInput::Path(i) => i,
AudioInput::Blob(b) => b.input_mut(),
}
}
}
pub struct VideoAudioReader { pub struct VideoAudioReader {
input: ffmpeg_next::format::context::Input, input: AudioInput,
decoder: ffmpeg_next::decoder::Audio, decoder: ffmpeg_next::decoder::Audio,
/// Built lazily from the first decoded frame's format/layout → interleaved f32. /// Built lazily from the first decoded frame's format/layout → interleaved f32.
resampler: Option<ffmpeg_next::software::resampling::Context>, resampler: Option<ffmpeg_next::software::resampling::Context>,
@ -605,15 +654,30 @@ impl VideoAudioReader {
self.total_frames self.total_frames
} }
/// Open the audio track of a video **file**.
pub fn open(path: &Path) -> Result<Self, String> { pub fn open(path: &Path) -> Result<Self, String> {
ffmpeg_next::init().map_err(|e| e.to_string())?; ffmpeg_next::init().map_err(|e| e.to_string())?;
let input = ffmpeg_next::format::input(&path) let input = ffmpeg_next::format::input(&path)
.map_err(|e| format!("Failed to open media: {}", e))?; .map_err(|e| format!("Failed to open media: {}", e))?;
Self::from_input(AudioInput::Path(input))
}
/// Open the audio track of a video streamed from a **byte source** (a packed
/// `.beam` video blob), via the `ffmpeg-blob-io` AVIO shim. `ext` is a container
/// hint (e.g. `"mp4"`).
pub fn open_source(src: Box<dyn MediaByteSource>, ext: Option<&str>) -> Result<Self, String> {
ffmpeg_next::init().map_err(|e| e.to_string())?;
let blob = ffmpeg_blob_io::BlobInput::open(Box::new(ByteSourceAdapter(src)), ext)
.map_err(|e| format!("Failed to open packed video audio: {}", e))?;
Self::from_input(AudioInput::Blob(blob))
}
fn from_input(mut input: AudioInput) -> Result<Self, String> {
// Pull stream scalars + build the decoder inside a scope so the stream // Pull stream scalars + build the decoder inside a scope so the stream
// borrow of `input` ends before we use `input` again. // borrow of `input` ends before we use `input` again.
let (stream_index, time_base, stream_duration, decoder) = { let (stream_index, time_base, stream_duration, decoder) = {
let stream = input let stream = input
.get()
.streams() .streams()
.best(ffmpeg_next::media::Type::Audio) .best(ffmpeg_next::media::Type::Audio)
.ok_or_else(|| "No audio stream found".to_string())?; .ok_or_else(|| "No audio stream found".to_string())?;
@ -631,8 +695,8 @@ impl VideoAudioReader {
let duration_secs = if stream_duration > 0 { let duration_secs = if stream_duration > 0 {
stream_duration as f64 * time_base stream_duration as f64 * time_base
} else if input.duration() > 0 { } else if input.get().duration() > 0 {
input.duration() as f64 / f64::from(ffmpeg_next::ffi::AV_TIME_BASE) input.get().duration() as f64 / f64::from(ffmpeg_next::ffi::AV_TIME_BASE)
} else { } else {
0.0 0.0
}; };
@ -659,6 +723,7 @@ impl VideoAudioReader {
// Seek to at-or-before the target (max_ts = ts_av) so we can decode // Seek to at-or-before the target (max_ts = ts_av) so we can decode
// forward to it exactly. ffmpeg-next's `seek` wants a bounded range. // forward to it exactly. ffmpeg-next's `seek` wants a bounded range.
self.input self.input
.get()
.seek(ts_av, 0..(ts_av + 1)) .seek(ts_av, 0..(ts_av + 1))
.map_err(|e| format!("Seek failed: {}", e))?; .map_err(|e| format!("Seek failed: {}", e))?;
self.decoder.flush(); self.decoder.flush();
@ -684,7 +749,7 @@ impl VideoAudioReader {
} }
// Read one packet (owned), releasing the `input` borrow before decoding. // Read one packet (owned), releasing the `input` borrow before decoding.
let packet = self.input.packets().next().map(|(_, p)| p); let packet = self.input.get().packets().next().map(|(_, p)| p);
match packet { match packet {
Some(packet) => { Some(packet) => {
if packet.stream() == self.stream_index { if packet.stream() == self.stream_index {
@ -827,8 +892,8 @@ impl StreamSource {
(SourceKind::VideoAudio, StreamOpen::Path(p)) => { (SourceKind::VideoAudio, StreamOpen::Path(p)) => {
Ok(StreamSource::Video(VideoAudioReader::open(&p)?)) Ok(StreamSource::Video(VideoAudioReader::open(&p)?))
} }
(SourceKind::VideoAudio, StreamOpen::Source { .. }) => { (SourceKind::VideoAudio, StreamOpen::Source { src, ext }) => {
Err("VideoAudio cannot be opened from a packed byte source".to_string()) Ok(StreamSource::Video(VideoAudioReader::open_source(src, ext.as_deref())?))
} }
} }
} }
@ -1093,7 +1158,12 @@ impl DiskReader {
// Decode more data into the buffer. // Decode more data into the buffer.
match reader.decode_next(&mut decode_buf) { match reader.decode_next(&mut decode_buf) {
Ok(0) => {} // EOF Ok(0) => {
// EOF: no more frames will be decoded for this buffer until a
// seek. Tell export-mode waiters so they stop blocking on
// past-the-end frames (e.g. video longer than its audio).
buffer.set_finished(true);
}
Ok(frames) => { Ok(frames) => {
let was_empty = buffer.valid_frames_count() == 0; let was_empty = buffer.valid_frames_count() == 0;
buffer.write_samples(&decode_buf, frames); buffer.write_samples(&decode_buf, frames);

3
daw-backend/src/audio/engine.rs
View File

@ -116,7 +116,6 @@ pub struct Engine {
// Current tempo map — kept in sync with SetTempo/SetTempoMap commands. // Current tempo map — kept in sync with SetTempo/SetTempoMap commands.
tempo_map: crate::TempoMap, tempo_map: crate::TempoMap,
current_fps: f64,
// Current time signature — updated by SetTempo, used when SetTempoMap fires. // Current time signature — updated by SetTempo, used when SetTempoMap fires.
time_sig: (u32, u32), time_sig: (u32, u32),
} }
@ -197,7 +196,6 @@ impl Engine {
timing_sum_total_us: 0, timing_sum_total_us: 0,
timing_overrun_count: 0, timing_overrun_count: 0,
tempo_map: crate::TempoMap::constant(120.0), tempo_map: crate::TempoMap::constant(120.0),
current_fps: 30.0,
time_sig: (4, 4), time_sig: (4, 4),
} }
} }
@ -2434,6 +2432,7 @@ impl Engine {
channels, channels,
sample_rate, sample_rate,
total_frames, total_frames,
None, // imported from an external video file; packing happens on save
); );
let pool_index = self.audio_pool.add_file(audio_file); let pool_index = self.audio_pool.add_file(audio_file);

38
daw-backend/src/audio/pool.rs
View File

@ -244,12 +244,15 @@ impl AudioFile {
/// Create a placeholder AudioFile for a video's audio track. `path` is the /// Create a placeholder AudioFile for a video's audio track. `path` is the
/// source video file; the audio is streamed on demand by the disk reader's /// source video file; the audio is streamed on demand by the disk reader's
/// FFmpeg-backed `VideoAudioReader`. /// FFmpeg-backed `VideoAudioReader`. `packed_media_id` is `Some` when the video
/// bytes are packed in the `.beam` container (audio streams from the same blob
/// via the host factory); `None` when the audio comes from an external video file.
pub fn from_video_audio( pub fn from_video_audio(
path: PathBuf, path: PathBuf,
channels: u32, channels: u32,
sample_rate: u32, sample_rate: u32,
total_frames: u64, total_frames: u64,
packed_media_id: Option<String>,
) -> Self { ) -> Self {
Self { Self {
path, path,
@ -263,7 +266,7 @@ impl AudioFile {
frames: total_frames, frames: total_frames,
original_format: None, original_format: None,
original_bytes: None, original_bytes: None,
packed_media_id: None, packed_media_id,
} }
} }
@ -574,6 +577,14 @@ impl AudioClipPool {
// Spin-wait with small sleeps until the disk reader fills the buffer // Spin-wait with small sleeps until the disk reader fills the buffer
let mut wait_iters = 0u64; let mut wait_iters = 0u64;
while !ra.has_range(src_start, frames_needed) { while !ra.has_range(src_start, frames_needed) {
// The source reached EOF and these frames are past the end of the
// audio (e.g. a video whose audio track is shorter than the video,
// or a container without exact stream duration). They will never
// arrive — stop waiting and let the render fill silence, instead of
// burning the 10s safety valve on every remaining chunk.
if ra.is_finished() {
break;
}
std::thread::sleep(std::time::Duration::from_micros(100)); std::thread::sleep(std::time::Duration::from_micros(100));
wait_iters += 1; wait_iters += 1;
if wait_iters > 100_000 { if wait_iters > 100_000 {
@ -1168,7 +1179,27 @@ impl AudioClipPool {
let entry_start = std::time::Instant::now(); let entry_start = std::time::Instant::now();
eprintln!("📊 [LOAD_SERIALIZED] Processing entry {}/{}: '{}'", i + 1, entries.len(), entry.name); eprintln!("📊 [LOAD_SERIALIZED] Processing entry {}/{}: '{}'", i + 1, entries.len(), entry.name);
let success = if entry.is_video_audio { let success = if entry.is_video_audio && entry.media_id.is_some() {
// Packed video: the audio track lives in the same container blob as
// the video. Build a streaming VideoAudio entry from the saved
// metadata (no probe needed); playback opens the blob via the host
// factory at clip-activation time → VideoAudioReader::open_source.
let media_id = entry.media_id.clone().unwrap();
let total_frames = (entry.duration * entry.sample_rate as f64).ceil() as u64;
let file = AudioFile::from_video_audio(
PathBuf::new(),
entry.channels,
entry.sample_rate,
total_frames,
Some(media_id),
);
if entry.pool_index < self.files.len() {
self.files[entry.pool_index] = file;
true
} else {
false
}
} else if entry.is_video_audio {
// Re-probe the video's audio track via FFmpeg → a streaming // Re-probe the video's audio track via FFmpeg → a streaming
// VideoAudio entry (keeps full 5.1/7.1; no decode-to-RAM). // VideoAudio entry (keeps full 5.1/7.1; no decode-to-RAM).
match entry.relative_path.as_ref() { match entry.relative_path.as_ref() {
@ -1186,6 +1217,7 @@ impl AudioClipPool {
reader.channels(), reader.channels(),
reader.sample_rate(), reader.sample_rate(),
reader.total_frames(), reader.total_frames(),
None,
); );
if entry.pool_index < self.files.len() { if entry.pool_index < self.files.len() {
self.files[entry.pool_index] = file; self.files[entry.pool_index] = file;

5
daw-backend/src/audio/project.rs
View File

@ -41,6 +41,11 @@ impl Project {
} }
} }
/// The project's system sample rate in Hz.
pub fn sample_rate(&self) -> u32 {
self.sample_rate
}
/// Generate a new unique track ID /// Generate a new unique track ID
fn next_id(&mut self) -> TrackId { fn next_id(&mut self) -> TrackId {
let id = self.next_track_id; let id = self.next_track_id;

66
daw-backend/tests/video_audio_stream.rs
View File

@ -251,3 +251,69 @@ fn seek_is_sample_accurate() {
} }
let _ = std::fs::remove_file(&path); let _ = std::fs::remove_file(&path);
} }
// ── Stage 3: stream video-audio from a byte source (packed .beam blob) ──────────
/// A `MediaByteSource` over an in-memory buffer (stands in for a SQLite BlobReader).
struct VecSource(std::io::Cursor<Vec<u8>>, u64);
impl std::io::Read for VecSource {
fn read(&mut self, b: &mut [u8]) -> std::io::Result<usize> { self.0.read(b) }
}
impl std::io::Seek for VecSource {
fn seek(&mut self, p: std::io::SeekFrom) -> std::io::Result<u64> { self.0.seek(p) }
}
impl daw_backend::audio::disk_reader::MediaByteSource for VecSource {
fn byte_len(&self) -> u64 { self.1 }
}
fn ramp_wav_bytes(n: u32, sample_rate: u32) -> Vec<u8> {
let channels = 1u16;
let bps = 4u32;
let data_size = n * bps;
let mut buf = Vec::with_capacity(44 + data_size as usize);
buf.extend_from_slice(b"RIFF");
buf.extend_from_slice(&(36 + data_size).to_le_bytes());
buf.extend_from_slice(b"WAVE");
buf.extend_from_slice(b"fmt ");
buf.extend_from_slice(&16u32.to_le_bytes());
buf.extend_from_slice(&3u16.to_le_bytes()); // IEEE float
buf.extend_from_slice(&channels.to_le_bytes());
buf.extend_from_slice(&sample_rate.to_le_bytes());
buf.extend_from_slice(&(sample_rate * channels as u32 * bps).to_le_bytes());
buf.extend_from_slice(&((channels as u32 * bps) as u16).to_le_bytes());
buf.extend_from_slice(&32u16.to_le_bytes());
buf.extend_from_slice(b"data");
buf.extend_from_slice(&data_size.to_le_bytes());
for i in 0..n {
buf.extend_from_slice(&((i as f32) / (n as f32)).to_le_bytes());
}
buf
}
#[test]
fn video_audio_open_source_streams_from_bytes() {
let sr = 8000;
let n = 4000;
let bytes = ramp_wav_bytes(n, sr);
let len = bytes.len() as u64;
let src = Box::new(VecSource(std::io::Cursor::new(bytes), len));
// Open the audio track by streaming from the byte source (no file path).
let mut reader = VideoAudioReader::open_source(src, Some("wav")).unwrap();
assert_eq!(reader.channels(), 1);
assert_eq!(reader.sample_rate(), sr);
let mut all: Vec<f32> = Vec::new();
let mut buf: Vec<f32> = Vec::new();
loop {
let frames = reader.decode_next(&mut buf).unwrap();
if frames == 0 {
break;
}
all.extend_from_slice(&buf);
}
assert!(all.len() as u32 >= n - 4, "decoded most of the ramp: {} of {}", all.len(), n);
// The ramp rises monotonically; sample 0 ≈ 0.0 and the last is near 1.0.
assert!(all[0].abs() < 1e-3, "first sample ~0, got {}", all[0]);
assert!(*all.last().unwrap() > 0.9, "last sample ~1.0, got {}", all.last().unwrap());
}

287
ffmpeg-blob-io/Cargo.lock generated Normal file
View File

@ -0,0 +1,287 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "aho-corasick"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
dependencies = [
"memchr",
]
[[package]]
name = "bindgen"
version = "0.72.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
dependencies = [
"bitflags",
"cexpr",
"clang-sys",
"itertools",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex 1.3.0",
"syn",
]
[[package]]
name = "bitflags"
version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
[[package]]
name = "cc"
version = "1.2.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
dependencies = [
"find-msvc-tools",
"shlex 2.0.1",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "clang-sys"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "ffmpeg-blob-io"
version = "0.1.0"
dependencies = [
"ffmpeg-next",
"ffmpeg-sys-next",
"libc",
]
[[package]]
name = "ffmpeg-next"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d658424d233cbd993a972dd73a66ca733acd12a494c68995c9ac32ae1fe65b40"
dependencies = [
"bitflags",
"ffmpeg-sys-next",
"libc",
]
[[package]]
name = "ffmpeg-sys-next"
version = "8.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bca20aa4ee774fe384c2490096c122b0b23cf524a9910add0686691003d797b"
dependencies = [
"bindgen",
"cc",
"libc",
"num_cpus",
"pkg-config",
"vcpkg",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "glob"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
[[package]]
name = "hermit-abi"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "libloading"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
dependencies = [
"cfg-if",
"windows-link",
]
[[package]]
name = "memchr"
version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
[[package]]
name = "minimal-lexical"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "nom"
version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
dependencies = [
"memchr",
"minimal-lexical",
]
[[package]]
name = "num_cpus"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "pkg-config"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
dependencies = [
"proc-macro2",
]
[[package]]
name = "regex"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "rustc-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "shlex"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"

22
ffmpeg-blob-io/Cargo.toml Normal file
View File

@ -0,0 +1,22 @@
# Standalone crate (own workspace root) — depended on by daw-backend and
# lightningbeam-core via relative path, like daw-backend itself.
[workspace]
[package]
name = "ffmpeg-blob-io"
version = "0.1.0"
edition = "2021"
description = "Custom AVIOContext-over-Read+Seek shim for ffmpeg-next: lets a demuxer stream from an arbitrary byte source (e.g. a SQLite blob) instead of a file path. Isolates the unsafe FFI + ffmpeg ABI coupling."
# ABI-PINNED. This crate dereferences raw AVFormatContext / AVIOContext fields
# (`pb`, `flags`, `buffer`, `opaque`) and wraps the result back into ffmpeg-next's
# `Input`, so it MUST link the exact same libavformat ABI as the rest of the
# workspace. The `=` pins match what lightningbeam-core / daw-backend resolve;
# bump all three together when upgrading ffmpeg.
[dependencies]
ffmpeg-next = "=8.0.0"
ffmpeg-sys-next = "=8.0.1"
libc = "0.2"
[dev-dependencies]
ffmpeg-next = "=8.0.0"

240
ffmpeg-blob-io/src/lib.rs Normal file
View File

@ -0,0 +1,240 @@
//! Stream a demuxer from an arbitrary `Read + Seek` byte source via a custom
//! FFmpeg `AVIOContext`.
//!
//! `ffmpeg-next`'s high-level API can only open inputs by filesystem path. This
//! crate builds an `AVFormatContext` whose I/O is backed by a Rust reader (e.g. a
//! SQLite `BlobReader`), then hands back a normal [`ffmpeg_next::format::context::Input`]
//! that the rest of the codebase decodes exactly as a file-backed input.
//!
//! All of the `unsafe` FFI and ffmpeg ABI coupling lives here, isolated behind the
//! safe [`BlobInput`] type. See `BEAM_FILE_FORMAT.md` / `STREAMING_TO_DISK_PLAN.md`.
//!
//! # Safety model
//! - The reader is boxed and leaked to FFmpeg as the AVIO `opaque` pointer; the
//! read/seek callbacks reconstitute it as `&mut`. It is never aliased — only the
//! owning [`BlobInput`] (on one thread) drives it.
//! - [`BlobInput`] is `Send` but **not** `Sync`: the typical reader (`BlobReader`)
//! owns a `!Sync` SQLite connection. Each decoder / seek-reopen / off-thread scan
//! must construct its **own** `BlobInput` from a **fresh** reader.
//! - `Drop` tears down in the one correct order (Input first, then the AVIO buffer +
//! context, then the reader) so there is no use-after-free, double-free, or leak.
use std::io::{Read, Seek, SeekFrom};
use std::os::raw::{c_char, c_int, c_void};
use std::ptr;
use ffmpeg_next::format::context::Input;
use ffmpeg_sys_next as sys;
/// A byte source FFmpeg can stream from. Blanket-implemented for any
/// `Read + Seek + Send` (e.g. `std::io::Cursor`, a SQLite `BlobReader`).
pub trait BlobSource: Read + Seek + Send {}
impl<T: Read + Seek + Send> BlobSource for T {}
// Stable FFmpeg ABI constants (avoid depending on whether bindgen exported the
// corresponding `#define`s). These are fixed across the libavformat 58–61 ABIs.
const AVSEEK_SIZE: c_int = 0x10000;
const AVSEEK_FORCE: c_int = 0x20000;
const AVFMT_FLAG_CUSTOM_IO: c_int = 0x0080;
/// Size of the AVIO read buffer handed to FFmpeg. FFmpeg may grow/replace it.
const IO_BUFFER_SIZE: usize = 32 * 1024;
/// A demuxer input streaming from a boxed `Read + Seek` source.
///
/// Deref to [`Input`] for the usual ffmpeg-next decode API.
pub struct BlobInput {
// Dropped FIRST in `Drop` (avformat_close_input). `Option` so we can drop it
// explicitly before freeing the AVIO resources it points at.
input: Option<Input>,
// The custom AVIOContext. With AVFMT_FLAG_CUSTOM_IO set, avformat_close_input
// does NOT free this — we own it. Its `opaque` field is the boxed reader.
avio: *mut sys::AVIOContext,
}
// The reader is `Send` and never aliased; we manage its lifetime manually.
unsafe impl Send for BlobInput {}
// Intentionally NOT `Sync`: see the module-level safety notes.
impl BlobInput {
/// Open a demuxer over `src`.
///
/// `format_hint` is an optional container short-name / extension (e.g. `"mp4"`,
/// `"mov"`, `"matroska"`) passed to `av_find_input_format` to skip probe
/// ambiguity; `None` lets FFmpeg probe from the stream.
pub fn open(src: Box<dyn BlobSource>, format_hint: Option<&str>) -> Result<Self, String> {
ffmpeg_next::init().map_err(|e| format!("ffmpeg init failed: {e}"))?;
unsafe {
// 1. IO buffer, allocated with FFmpeg's allocator (freed with av_freep).
let buffer = sys::av_malloc(IO_BUFFER_SIZE) as *mut u8;
if buffer.is_null() {
return Err("av_malloc failed for AVIO buffer".into());
}
// 2. Box the reader and leak it as the opaque pointer. Double-box so the
// raw pointer is thin (a `*mut dyn` would be a fat pointer).
let opaque = Box::into_raw(Box::new(src)) as *mut c_void;
// 3. AVIOContext over our callbacks (read-only: write_flag = 0).
let avio = sys::avio_alloc_context(
buffer,
IO_BUFFER_SIZE as c_int,
0,
opaque,
Some(read_cb),
None,
Some(seek_cb),
);
if avio.is_null() {
sys::av_free(buffer as *mut c_void);
drop(Box::from_raw(opaque as *mut Box<dyn BlobSource>));
return Err("avio_alloc_context failed".into());
}
// 4. Format context wired to the custom IO.
let mut fmt = sys::avformat_alloc_context();
if fmt.is_null() {
destroy_io(avio);
return Err("avformat_alloc_context failed".into());
}
(*fmt).pb = avio;
(*fmt).flags |= AVFMT_FLAG_CUSTOM_IO;
// 5. Optional format hint.
let hint_cstr = format_hint.and_then(|s| std::ffi::CString::new(s).ok());
let infmt: *const sys::AVInputFormat = match &hint_cstr {
Some(c) => sys::av_find_input_format(c.as_ptr() as *const c_char),
None => ptr::null(),
};
// 6. Open. On failure avformat_open_input frees `fmt` itself (and, with
// CUSTOM_IO, leaves our pb), so we still own avio+buffer+reader.
let ret = sys::avformat_open_input(&mut fmt, ptr::null(), infmt, ptr::null_mut());
if ret < 0 {
destroy_io(avio);
return Err(format!("avformat_open_input failed (error {ret})"));
}
// 7. Probe streams. On failure close the now-open input, then free IO.
let ret = sys::avformat_find_stream_info(fmt, ptr::null_mut());
if ret < 0 {
sys::avformat_close_input(&mut fmt);
destroy_io(avio);
return Err(format!("avformat_find_stream_info failed (error {ret})"));
}
// 8. Hand ownership of `fmt` to ffmpeg-next's Input (closes on drop).
let input = Input::wrap(fmt);
Ok(BlobInput { input: Some(input), avio })
}
}
/// The underlying demuxer input.
pub fn input(&self) -> &Input {
self.input.as_ref().expect("BlobInput input present until drop")
}
/// The underlying demuxer input, mutably (for `seek`, `packets`, …).
pub fn input_mut(&mut self) -> &mut Input {
self.input.as_mut().expect("BlobInput input present until drop")
}
}
impl std::ops::Deref for BlobInput {
type Target = Input;
fn deref(&self) -> &Input {
self.input()
}
}
impl std::ops::DerefMut for BlobInput {
fn deref_mut(&mut self) -> &mut Input {
self.input_mut()
}
}
impl Drop for BlobInput {
fn drop(&mut self) {
unsafe {
// 1. Drop the Input first: avformat_close_input. With CUSTOM_IO this does
// NOT touch self.avio or its buffer.
self.input = None;
// 2..4. Free the AVIO buffer + context and reclaim the boxed reader.
if !self.avio.is_null() {
destroy_io(self.avio);
self.avio = ptr::null_mut();
}
}
}
}
/// Free a standalone custom AVIOContext: its (possibly reallocated) buffer, the
/// boxed reader behind `opaque`, then the context itself — in that order. Only
/// call when the owning AVFormatContext has already been closed (or never opened).
unsafe fn destroy_io(avio: *mut sys::AVIOContext) {
if avio.is_null() {
return;
}
// Reclaim the reader box *before* freeing the context (we need `opaque`).
let opaque = (*avio).opaque;
// Free the current IO buffer (FFmpeg may have replaced the original).
sys::av_freep(&mut (*avio).buffer as *mut _ as *mut c_void);
// Free the AVIOContext struct (nulls the local).
let mut avio = avio;
sys::avio_context_free(&mut avio);
// Drop the reader last — no callback can fire now.
if !opaque.is_null() {
drop(Box::from_raw(opaque as *mut Box<dyn BlobSource>));
}
}
/// FFmpeg read callback: fill `buf` from the Rust reader. Returns bytes read,
/// `AVERROR_EOF` at end of stream, or a negative error.
unsafe extern "C" fn read_cb(opaque: *mut c_void, buf: *mut u8, buf_size: c_int) -> c_int {
if opaque.is_null() || buf.is_null() || buf_size <= 0 {
return sys::AVERROR_EOF;
}
let reader = &mut *(opaque as *mut Box<dyn BlobSource>);
let slice = std::slice::from_raw_parts_mut(buf, buf_size as usize);
match reader.read(slice) {
Ok(0) => sys::AVERROR_EOF,
Ok(n) => n as c_int,
// AVERROR(EIO) == -EIO on Unix; a negative value signals a read error.
Err(_) => -(libc::EIO as c_int),
}
}
/// FFmpeg seek callback. Handles `SEEK_SET/CUR/END` and `AVSEEK_SIZE` (report total
/// length). Returns the new position, the size, or `-1` on error.
unsafe extern "C" fn seek_cb(opaque: *mut c_void, offset: i64, whence: c_int) -> i64 {
if opaque.is_null() {
return -1;
}
let reader = &mut *(opaque as *mut Box<dyn BlobSource>);
let whence = whence & !AVSEEK_FORCE;
if whence & AVSEEK_SIZE != 0 {
// Report total length WITHOUT moving the logical position. FFmpeg does not
// restore the position after an AVSEEK_SIZE query, so we must: measure by
// seeking to the end, then seek back. (Failing to restore corrupts reads of
// containers whose index lives at the end of the file, e.g. MP4 `moov`.)
let cur = match reader.stream_position() {
Ok(p) => p,
Err(_) => return -1,
};
let size = match reader.seek(SeekFrom::End(0)) {
Ok(s) => s,
Err(_) => return -1,
};
if reader.seek(SeekFrom::Start(cur)).is_err() {
return -1;
}
return size as i64;
}
let from = match whence {
libc::SEEK_SET => SeekFrom::Start(offset as u64),
libc::SEEK_CUR => SeekFrom::Current(offset),
libc::SEEK_END => SeekFrom::End(offset),
_ => return -1,
};
reader.seek(from).map(|n| n as i64).unwrap_or(-1)
}

100
ffmpeg-blob-io/tests/blob_input.rs Normal file
View File

@ -0,0 +1,100 @@
//! Exercises the custom AVIO shim end to end against a hand-built WAV, fed through
//! an in-memory `Cursor` (a `Read + Seek` source). FFmpeg can only demux it by
//! calling our `read_cb`/`seek_cb`, so a successful open proves the shim works.
use ffmpeg_blob_io::BlobInput;
use std::io::Cursor;
/// Build a minimal 16-bit PCM WAV in memory.
fn make_wav(sample_rate: u32, channels: u16, samples: &[i16]) -> Vec<u8> {
let bits: u16 = 16;
let block_align: u16 = channels * (bits / 8);
let byte_rate: u32 = sample_rate * block_align as u32;
let data_len: u32 = (samples.len() * 2) as u32;
let mut v = Vec::new();
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(36 + data_len).to_le_bytes());
v.extend_from_slice(b"WAVE");
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&16u32.to_le_bytes()); // fmt chunk size
v.extend_from_slice(&1u16.to_le_bytes()); // PCM
v.extend_from_slice(&channels.to_le_bytes());
v.extend_from_slice(&sample_rate.to_le_bytes());
v.extend_from_slice(&byte_rate.to_le_bytes());
v.extend_from_slice(&block_align.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
v.extend_from_slice(b"data");
v.extend_from_slice(&data_len.to_le_bytes());
for s in samples {
v.extend_from_slice(&s.to_le_bytes());
}
v
}
fn sample_wav() -> Vec<u8> {
let samples: Vec<i16> = (0..1600).map(|i| ((i % 100) as i16 - 50) * 200).collect();
make_wav(8000, 1, &samples)
}
#[test]
fn opens_from_reader_and_reports_stream() {
let input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav"))
.expect("open WAV through the blob shim");
let stream = input
.streams()
.best(ffmpeg_next::media::Type::Audio)
.expect("an audio stream");
let ctx = ffmpeg_next::codec::context::Context::from_parameters(stream.parameters())
.expect("codec context");
let decoder = ctx.decoder().audio().expect("audio decoder");
assert_eq!(decoder.rate(), 8000, "sample rate read via AVIO");
assert_eq!(decoder.channels(), 1, "channel count read via AVIO");
}
#[test]
fn reads_packets_through_callbacks() {
let mut input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav"))
.expect("open WAV");
let mut packets = 0usize;
let mut bytes = 0usize;
for (_stream, packet) in input.packets() {
packets += 1;
bytes += packet.size();
}
assert!(packets > 0, "demuxer produced packets via read_cb");
assert!(bytes > 0, "packets carried PCM payload");
}
#[test]
fn seek_then_read() {
let mut input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav"))
.expect("open WAV");
// Seek back to the start (exercises seek_cb SEEK_SET + AVSEEK_SIZE).
input.seek(0, ..).expect("seek to start");
let got_packet = input.packets().next().is_some();
assert!(got_packet, "can still read after seek");
}
#[test]
fn open_drop_loop_is_clean() {
// Repeated open+drop surfaces double-free / leak in the Drop teardown
// (run under `RUSTFLAGS=-Zsanitizer=address` on nightly for full coverage).
for _ in 0..200 {
let input = BlobInput::open(Box::new(Cursor::new(sample_wav())), Some("wav"))
.expect("open WAV");
assert!(input.streams().count() >= 1);
drop(input);
}
}
#[test]
fn bad_format_hint_errors_without_leak() {
// Garbage bytes with a real hint: open should fail cleanly (error path frees
// buffer + avio + reader), not panic or leak.
let garbage = vec![0u8; 4096];
let res = BlobInput::open(Box::new(Cursor::new(garbage)), Some("wav"));
assert!(res.is_err(), "non-WAV bytes should fail to open as WAV");
}

27
lightningbeam-ui/Cargo.lock generated
View File

@ -1682,6 +1682,7 @@ dependencies = [
"dasp_rms", "dasp_rms",
"dasp_sample", "dasp_sample",
"dasp_signal", "dasp_signal",
"ffmpeg-blob-io",
"ffmpeg-next", "ffmpeg-next",
"hound", "hound",
"memmap2", "memmap2",
@ -2218,6 +2219,15 @@ dependencies = [
"simd-adler32", "simd-adler32",
] ]
[[package]]
name = "ffmpeg-blob-io"
version = "0.1.0"
dependencies = [
"ffmpeg-next",
"ffmpeg-sys-next",
"libc",
]
[[package]] [[package]]
name = "ffmpeg-next" name = "ffmpeg-next"
version = "8.0.0" version = "8.0.0"
@ -2839,6 +2849,19 @@ dependencies = [
"bitflags 2.10.0", "bitflags 2.10.0",
] ]
[[package]]
name = "gpu-video-encoder"
version = "0.1.0"
dependencies = [
"ash",
"ffmpeg-sys-next",
"libc",
"pollster 0.4.0",
"wgpu",
"wgpu-hal",
"wgpu-types",
]
[[package]] [[package]]
name = "gtk" name = "gtk"
version = "0.18.2" version = "0.18.2"
@ -3480,6 +3503,7 @@ dependencies = [
"claxon", "claxon",
"daw-backend", "daw-backend",
"egui", "egui",
"ffmpeg-blob-io",
"ffmpeg-next", "ffmpeg-next",
"flacenc", "flacenc",
"image", "image",
@ -3505,7 +3529,7 @@ dependencies = [
[[package]] [[package]]
name = "lightningbeam-editor" name = "lightningbeam-editor"
version = "1.0.4-alpha" version = "1.0.5-alpha"
dependencies = [ dependencies = [
"beamdsp", "beamdsp",
"bytemuck", "bytemuck",
@ -3519,6 +3543,7 @@ dependencies = [
"egui_extras", "egui_extras",
"egui_node_graph2", "egui_node_graph2",
"ffmpeg-next", "ffmpeg-next",
"gpu-video-encoder",
"half", "half",
"image", "image",
"kurbo 0.12.0", "kurbo 0.12.0",

1
lightningbeam-ui/Cargo.toml
View File

@ -4,6 +4,7 @@ members = [
"lightningbeam-editor", "lightningbeam-editor",
"lightningbeam-core", "lightningbeam-core",
"beamdsp", "beamdsp",
"gpu-video-encoder",
] ]
[workspace.dependencies] [workspace.dependencies]

20
lightningbeam-ui/gpu-video-encoder/Cargo.toml Normal file
View File

@ -0,0 +1,20 @@
[package]
name = "gpu-video-encoder"
version = "0.1.0"
edition = "2021"
description = "Zero-copy GPU video encoding (RGBA->NV12 compute + hardware encoder interop). Unsafe FFI isolated here."
[dependencies]
wgpu = { workspace = true }
# Raw Vulkan access for the DMA-BUF import. Versions MUST match what wgpu links
# (wgpu-hal 27.0.4 / ash 0.38) so the hal/ash types unify across the boundary.
wgpu-hal = { version = "27", features = ["vulkan"] }
wgpu-types = "27"
ash = "0.38"
# Raw FFmpeg FFI for VAAPI hwcontext + hardware encode. Matches the editor's
# ffmpeg-next 8.0 / static link so cargo unifies to one libav* across the build.
ffmpeg-sys-next = { version = "8.0", features = ["static"] }
libc = "0.2"
[dev-dependencies]
pollster = "0.4"

193
lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs Normal file
View File

@ -0,0 +1,193 @@
//! Import a tiled VAAPI NV12 DMA-BUF as two wgpu textures (Y = R8, UV = RG8), aliasing
//! the one imported `VkDeviceMemory` at the plane offsets. Two single-format images are
//! used instead of one multi-planar image so each is an ordinary wgpu render target.
use crate::vaapi::MappedSurface;
use crate::vk_device::DrmDevice;
use ash::vk;
/// Plane layout for a single-object NV12 DMA-BUF (the common VAAPI case).
#[derive(Clone, Copy)]
pub struct Nv12DmaBuf {
pub fd: i32,
pub size: u64,
pub modifier: u64,
pub width: u32,
pub height: u32,
pub y_offset: u64,
pub y_pitch: u64,
pub uv_offset: u64,
pub uv_pitch: u64,
}
/// Frees the shared imported `VkDeviceMemory` once both plane images are gone. Held by
/// both textures' drop callbacks (via `Arc`); the last one to run frees the memory —
/// after wgpu has destroyed the images, in its wait-idle'd deferred-destruction pass.
struct MemoryGuard {
device: ash::Device,
memory: vk::DeviceMemory,
}
impl Drop for MemoryGuard {
fn drop(&mut self) {
unsafe { self.device.free_memory(self.memory, None) };
}
}
/// A VAAPI surface imported as two wgpu plane textures. The underlying Vulkan image/
/// memory are destroyed by wgpu (via drop callbacks) when these textures drop.
pub struct ImportedNv12 {
y: wgpu::Texture,
uv: wgpu::Texture,
}
impl ImportedNv12 {
pub fn y(&self) -> &wgpu::Texture {
&self.y
}
pub fn uv(&self) -> &wgpu::Texture {
&self.uv
}
}
/// Convenience: map a freshly-allocated `MappedSurface` and import it.
pub fn import(drm: &DrmDevice, surf: &MappedSurface) -> Result<ImportedNv12, String> {
import_raw(
drm,
&Nv12DmaBuf {
fd: surf.fd,
size: surf.size,
modifier: surf.modifier,
width: surf.width,
height: surf.height,
y_offset: surf.y_offset,
y_pitch: surf.y_pitch,
uv_offset: surf.uv_offset,
uv_pitch: surf.uv_pitch,
},
)
}
/// Import an NV12 DMA-BUF (described by `buf`) as two wgpu plane textures. The fd is
/// duplicated, so the caller keeps ownership of theirs.
pub fn import_raw(drm: &DrmDevice, buf: &Nv12DmaBuf) -> Result<ImportedNv12, String> {
unsafe {
let device = drm.raw_device.clone();
let instance = &drm.raw_instance;
let dup_fd = libc::dup(buf.fd);
if dup_fd < 0 {
return Err("dup(dma-buf fd) failed".into());
}
let make_image = |format: vk::Format, w: u32, h: u32, pitch: u64| -> Result<vk::Image, String> {
let mut ext = vk::ExternalMemoryImageCreateInfo::default()
.handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT);
let plane_layouts = [vk::SubresourceLayout::default().offset(0).row_pitch(pitch)];
let mut drm_info = vk::ImageDrmFormatModifierExplicitCreateInfoEXT::default()
.drm_format_modifier(buf.modifier)
.plane_layouts(&plane_layouts);
let info = vk::ImageCreateInfo::default()
.image_type(vk::ImageType::TYPE_2D)
.format(format)
.extent(vk::Extent3D { width: w, height: h, depth: 1 })
.mip_levels(1)
.array_layers(1)
.samples(vk::SampleCountFlags::TYPE_1)
.tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT)
.usage(
vk::ImageUsageFlags::COLOR_ATTACHMENT
| vk::ImageUsageFlags::TRANSFER_SRC
| vk::ImageUsageFlags::TRANSFER_DST,
)
.sharing_mode(vk::SharingMode::EXCLUSIVE)
.initial_layout(vk::ImageLayout::UNDEFINED)
.push_next(&mut ext)
.push_next(&mut drm_info);
device
.create_image(&info, None)
.map_err(|e| format!("vkCreateImage(modifier) failed: {e:?}"))
};
let img_y = make_image(vk::Format::R8_UNORM, buf.width, buf.height, buf.y_pitch)?;
let img_uv = make_image(vk::Format::R8G8_UNORM, buf.width / 2, buf.height / 2, buf.uv_pitch)?;
let fd_dev = ash::khr::external_memory_fd::Device::new(instance, &device);
let mut fd_props = vk::MemoryFdPropertiesKHR::default();
fd_dev
.get_memory_fd_properties(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT, dup_fd, &mut fd_props)
.map_err(|e| format!("vkGetMemoryFdPropertiesKHR failed: {e:?}"))?;
let req_y = device.get_image_memory_requirements(img_y);
let req_uv = device.get_image_memory_requirements(img_uv);
let type_bits = fd_props.memory_type_bits & req_y.memory_type_bits & req_uv.memory_type_bits;
if type_bits == 0 {
return Err("no memory type compatible with dma-buf + both plane images".into());
}
let mem_type = type_bits.trailing_zeros();
let mut import_info = vk::ImportMemoryFdInfoKHR::default()
.handle_type(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT)
.fd(dup_fd);
let alloc = vk::MemoryAllocateInfo::default()
.allocation_size(buf.size)
.memory_type_index(mem_type)
.push_next(&mut import_info);
let memory = device
.allocate_memory(&alloc, None)
.map_err(|e| format!("vkAllocateMemory(import dma-buf) failed: {e:?}"))?;
device
.bind_image_memory(img_y, memory, buf.y_offset)
.map_err(|e| format!("bind Y plane: {e:?}"))?;
device
.bind_image_memory(img_uv, memory, buf.uv_offset)
.map_err(|e| format!("bind UV plane: {e:?}"))?;
// Shared guard: frees `memory` once both images' drop callbacks have run.
let mem_guard = std::sync::Arc::new(MemoryGuard { device: device.clone(), memory });
let hal_device = drm
.device
.as_hal::<wgpu_hal::vulkan::Api>()
.ok_or("device is not Vulkan")?;
let wrap = |img: vk::Image, format: wgpu::TextureFormat, w: u32, h: u32| -> wgpu::Texture {
// wgpu destroys the image (after wait-idle) when the texture drops; the
// captured Arc<MemoryGuard> frees the shared memory once both have run.
let dev = device.clone();
let guard = mem_guard.clone();
let cb: wgpu_hal::DropCallback = Box::new(move || {
dev.destroy_image(img, None);
drop(guard);
});
let hal_desc = wgpu_hal::TextureDescriptor {
label: Some("vaapi-plane"),
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format,
usage: wgpu_types::TextureUses::COLOR_TARGET | wgpu_types::TextureUses::COPY_SRC,
memory_flags: wgpu_hal::MemoryFlags::empty(),
view_formats: vec![],
};
let hal_tex = hal_device.texture_from_raw(img, &hal_desc, Some(cb));
drm.device.create_texture_from_hal::<wgpu_hal::vulkan::Api>(
hal_tex,
&wgpu::TextureDescriptor {
label: Some("vaapi-plane"),
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC,
view_formats: &[],
},
)
};
let y = wrap(img_y, wgpu::TextureFormat::R8Unorm, buf.width, buf.height);
let uv = wrap(img_uv, wgpu::TextureFormat::Rg8Unorm, buf.width / 2, buf.height / 2);
drop(hal_device);
Ok(ImportedNv12 { y, uv })
}
}

292
lightningbeam-ui/gpu-video-encoder/src/encoder.rs Normal file
View File

@ -0,0 +1,292 @@
//! End-to-end zero-copy H.264 encoder: render an RGBA wgpu texture straight into a VAAPI
//! NV12 surface (no CPU copy) and encode it with `h264_vaapi`. The caller renders frames
//! on [`ZeroCopyEncoder::device`] (the custom Vulkan device with DMA-BUF import enabled).
//!
//! Imports are cached by VASurface id, so the pooled surfaces are imported once each.
use crate::dmabuf::{self, ImportedNv12, Nv12DmaBuf};
use crate::render_nv12::Rgba2Nv12;
use crate::vk_device::{self, DrmDevice};
use ffmpeg_sys_next as ff;
use std::collections::HashMap;
use std::ffi::CString;
use std::path::Path;
use std::ptr;
#[inline]
fn averror(e: i32) -> i32 {
-e
}
pub struct ZeroCopyEncoder {
drm: DrmDevice,
renderer: Rgba2Nv12,
hw_device: *mut ff::AVBufferRef,
frames_ref: *mut ff::AVBufferRef,
enc: *mut ff::AVCodecContext,
pkt: *mut ff::AVPacket,
/// Output container (e.g. `.mp4`); packets are muxed into it directly.
oc: *mut ff::AVFormatContext,
enc_tb: ff::AVRational,
stream_tb: ff::AVRational,
width: u32,
height: u32,
pts: i64,
cache: HashMap<usize, ImportedNv12>,
}
// The encoder owns its FFmpeg contexts (raw `*mut`) and Vulkan/wgpu handles exclusively; it is
// never shared, only moved. Sending it to a dedicated export thread is sound.
unsafe impl Send for ZeroCopyEncoder {}
impl ZeroCopyEncoder {
/// Build a zero-copy `h264_vaapi` encoder writing to `output_path` (container inferred
/// from the extension, e.g. `.mp4`). `Err` if VAAPI/the device is unavailable.
pub fn new(
width: u32,
height: u32,
framerate: i32,
bitrate_kbps: u32,
output_path: &Path,
) -> Result<Self, String> {
let drm = vk_device::create()?;
let renderer = Rgba2Nv12::new(&drm.device);
unsafe {
let mut hw_device = crate::vaapi::create_device()?;
let name = CString::new("h264_vaapi").unwrap();
let codec = ff::avcodec_find_encoder_by_name(name.as_ptr());
if codec.is_null() {
ff::av_buffer_unref(&mut hw_device);
return Err("h264_vaapi not found".into());
}
let enc = ff::avcodec_alloc_context3(codec);
(*enc).width = width as i32;
(*enc).height = height as i32;
(*enc).time_base = ff::AVRational { num: 1, den: framerate };
(*enc).framerate = ff::AVRational { num: framerate, den: 1 };
(*enc).pix_fmt = ff::AVPixelFormat::AV_PIX_FMT_VAAPI;
(*enc).bit_rate = (bitrate_kbps as i64) * 1000;
let frames_ref = ff::av_hwframe_ctx_alloc(hw_device);
{
let fctx = (*frames_ref).data as *mut ff::AVHWFramesContext;
(*fctx).format = ff::AVPixelFormat::AV_PIX_FMT_VAAPI;
(*fctx).sw_format = ff::AVPixelFormat::AV_PIX_FMT_NV12;
(*fctx).width = width as i32;
(*fctx).height = height as i32;
(*fctx).initial_pool_size = 16;
}
if ff::av_hwframe_ctx_init(frames_ref) < 0 {
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::avcodec_free_context(&mut (enc as *mut _));
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_ctx_init failed".into());
}
(*enc).hw_frames_ctx = ff::av_buffer_ref(frames_ref);
// Output container (format inferred from the path's extension).
let cleanup = |frames_ref: *mut ff::AVBufferRef, enc: *mut ff::AVCodecContext, hw: *mut ff::AVBufferRef| {
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::avcodec_free_context(&mut (enc as *mut _));
let mut h = hw;
ff::av_buffer_unref(&mut h);
};
let path_c = CString::new(output_path.to_string_lossy().as_ref()).unwrap();
let mut oc: *mut ff::AVFormatContext = ptr::null_mut();
if ff::avformat_alloc_output_context2(&mut oc, ptr::null(), ptr::null(), path_c.as_ptr()) < 0
|| oc.is_null()
{
cleanup(frames_ref, enc, hw_device);
return Err(format!("avformat_alloc_output_context2 for {output_path:?} failed"));
}
// mp4/mov want SPS/PPS in extradata, not inline — set before opening the encoder.
if (*(*oc).oformat).flags & ff::AVFMT_GLOBALHEADER as i32 != 0 {
(*enc).flags |= ff::AV_CODEC_FLAG_GLOBAL_HEADER as i32;
}
if ff::avcodec_open2(enc, codec, ptr::null_mut()) < 0 {
ff::avformat_free_context(oc);
cleanup(frames_ref, enc, hw_device);
return Err("avcodec_open2(h264_vaapi) failed".into());
}
let stream = ff::avformat_new_stream(oc, codec);
if stream.is_null() {
ff::avformat_free_context(oc);
cleanup(frames_ref, enc, hw_device);
return Err("avformat_new_stream failed".into());
}
if ff::avcodec_parameters_from_context((*stream).codecpar, enc) < 0 {
ff::avformat_free_context(oc);
cleanup(frames_ref, enc, hw_device);
return Err("avcodec_parameters_from_context failed".into());
}
(*stream).time_base = (*enc).time_base;
if ff::avio_open(&mut (*oc).pb, path_c.as_ptr(), ff::AVIO_FLAG_WRITE as i32) < 0 {
ff::avformat_free_context(oc);
cleanup(frames_ref, enc, hw_device);
return Err(format!("avio_open {output_path:?} failed"));
}
if ff::avformat_write_header(oc, ptr::null_mut()) < 0 {
ff::avio_closep(&mut (*oc).pb);
ff::avformat_free_context(oc);
cleanup(frames_ref, enc, hw_device);
return Err("avformat_write_header failed".into());
}
// The muxer may rewrite the stream time_base in write_header.
let stream_tb = (*stream).time_base;
Ok(Self {
drm,
renderer,
hw_device,
frames_ref,
enc,
pkt: ff::av_packet_alloc(),
oc,
enc_tb: (*enc).time_base,
stream_tb,
width,
height,
pts: 0,
cache: HashMap::new(),
})
}
}
/// The wgpu device frames must be rendered on (so the RGBA texture is importable).
pub fn device(&self) -> &wgpu::Device {
&self.drm.device
}
pub fn queue(&self) -> &wgpu::Queue {
&self.drm.queue
}
/// Render `rgba` (an `Rgba8Unorm` texture on [`Self::device`], `TEXTURE_BINDING`)
/// into a VAAPI surface and encode it. Appends any produced packets internally.
pub fn encode_rgba(&mut self, rgba: &wgpu::Texture) -> Result<(), String> {
unsafe {
let surf = ff::av_frame_alloc();
if ff::av_hwframe_get_buffer(self.frames_ref, surf, 0) < 0 {
ff::av_frame_free(&mut (surf as *mut _));
return Err("av_hwframe_get_buffer failed".into());
}
let id = (*surf).data[3] as usize; // VASurfaceID
if !self.cache.contains_key(&id) {
let drm_f = ff::av_frame_alloc();
(*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
let flags = ff::AV_HWFRAME_MAP_DIRECT as i32
| ff::AV_HWFRAME_MAP_READ as i32
| ff::AV_HWFRAME_MAP_WRITE as i32;
if ff::av_hwframe_map(drm_f, surf, flags) < 0 {
ff::av_frame_free(&mut (drm_f as *mut _));
ff::av_frame_free(&mut (surf as *mut _));
return Err("av_hwframe_map failed".into());
}
let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor;
let obj = &(*desc).objects[0];
let y = &(*desc).layers[0].planes[0];
let uv = &(*desc).layers[1].planes[0];
let buf = Nv12DmaBuf {
fd: obj.fd,
size: obj.size as u64,
modifier: obj.format_modifier,
width: self.width,
height: self.height,
y_offset: y.offset as u64,
y_pitch: y.pitch as u64,
uv_offset: uv.offset as u64,
uv_pitch: uv.pitch as u64,
};
let imported = match dmabuf::import_raw(&self.drm, &buf) {
Ok(i) => i,
Err(e) => {
ff::av_frame_free(&mut (drm_f as *mut _));
ff::av_frame_free(&mut (surf as *mut _));
return Err(e);
}
};
ff::av_frame_free(&mut (drm_f as *mut _)); // fd was dup'd into Vulkan
self.cache.insert(id, imported);
}
// Render RGBA -> NV12 directly into the surface planes.
let imp = self.cache.get(&id).unwrap();
let rgba_view = rgba.create_view(&Default::default());
let y_view = imp.y().create_view(&Default::default());
let uv_view = imp.uv().create_view(&Default::default());
let mut cmd = self.drm.device.create_command_encoder(&Default::default());
self.renderer.convert(&self.drm.device, &mut cmd, &rgba_view, &y_view, &uv_view);
self.drm.queue.submit(Some(cmd.finish()));
let _ = self.drm.device.poll(wgpu::PollType::wait_indefinitely());
// Encode the surface.
(*surf).pts = self.pts;
self.pts += 1;
let r = ff::avcodec_send_frame(self.enc, surf);
ff::av_frame_free(&mut (surf as *mut _));
if r < 0 {
return Err(format!("avcodec_send_frame failed: {r}"));
}
self.drain()
}
}
unsafe fn drain(&mut self) -> Result<(), String> {
loop {
let r = ff::avcodec_receive_packet(self.enc, self.pkt);
if r == averror(libc::EAGAIN) || r == ff::AVERROR_EOF {
break;
}
if r < 0 {
return Err(format!("avcodec_receive_packet failed: {r}"));
}
ff::av_packet_rescale_ts(self.pkt, self.enc_tb, self.stream_tb);
(*self.pkt).stream_index = 0;
// Takes ownership of the packet's buffer (unrefs it for us).
let w = ff::av_interleaved_write_frame(self.oc, self.pkt);
if w < 0 {
return Err(format!("av_interleaved_write_frame failed: {w}"));
}
}
Ok(())
}
/// Flush the encoder, write the container trailer, and close the output file.
pub fn finish(mut self) -> Result<(), String> {
unsafe {
ff::avcodec_send_frame(self.enc, ptr::null_mut());
self.drain()?;
if ff::av_write_trailer(self.oc) < 0 {
return Err("av_write_trailer failed".into());
}
ff::avio_closep(&mut (*self.oc).pb);
}
Ok(())
}
}
impl Drop for ZeroCopyEncoder {
fn drop(&mut self) {
unsafe {
self.cache.clear(); // frees imported Vulkan resources first
ff::av_packet_free(&mut (self.pkt as *mut _));
ff::avcodec_free_context(&mut (self.enc as *mut _));
let mut fr = self.frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut self.hw_device);
if !self.oc.is_null() {
// `finish` nulls pb via avio_closep; close here too if it wasn't called.
if !(*self.oc).pb.is_null() {
ff::avio_closep(&mut (*self.oc).pb);
}
ff::avformat_free_context(self.oc);
self.oc = ptr::null_mut();
}
}
}
}

61
lightningbeam-ui/gpu-video-encoder/src/lib.rs Normal file
View File

@ -0,0 +1,61 @@
//! Zero-copy GPU video encoding.
//!
//! Converts a rendered RGBA texture to the encoder's pixel format (NV12) on the GPU
//! and feeds it to a hardware video encoder without a CPU round-trip. All the unsafe
//! GPU↔encoder interop (Vulkan external memory / DMA-BUF → VAAPI on Linux, etc.) is
//! isolated in this crate.
//!
//! Status: scaffolding. Headless GPU probe + (next) NV12 compute live here first so
//! the GPU-side conversion can be validated against a CPU reference before any unsafe
//! interop is written. See `lightningbeam-ui/ZEROCOPY_GPU_ENCODE_PLAN.md`.
pub mod nv12;
/// Fragment-shader RGBA→NV12 conversion that renders into plane textures.
pub mod render_nv12;
/// VAAPI hardware encode (Linux-only; libva).
#[cfg(target_os = "linux")]
pub mod vaapi;
/// Custom Vulkan device with DMA-BUF import extensions (Linux).
#[cfg(target_os = "linux")]
pub mod vk_device;
/// Import a VAAPI NV12 DMA-BUF as wgpu textures (Linux).
#[cfg(target_os = "linux")]
pub mod dmabuf;
/// End-to-end zero-copy `h264_vaapi` encoder (Linux).
#[cfg(target_os = "linux")]
pub mod encoder;
#[cfg(test)]
mod probe_tests {
/// Confirm a headless GPU adapter is reachable (Vulkan on Linux/Intel). This gates
/// whether the GPU-side conversion can be tested on real hardware in this env.
#[test]
fn headless_adapter_available() {
let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor {
backends: wgpu::Backends::VULKAN | wgpu::Backends::GL,
..Default::default()
});
let adapter = pollster::block_on(instance.request_adapter(
&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::HighPerformance,
force_fallback_adapter: false,
compatible_surface: None,
},
));
match adapter {
Ok(a) => {
let info = a.get_info();
eprintln!(
"[gpu-probe] adapter: {} | backend={:?} | type={:?} | driver={}",
info.name, info.backend, info.device_type, info.driver
);
}
Err(e) => panic!("no GPU adapter available headless: {e:?}"),
}
}
}

207
lightningbeam-ui/gpu-video-encoder/src/nv12.rs Normal file
View File

@ -0,0 +1,207 @@
//! GPU RGBA→NV12 conversion (BT.709 full-range), the pixel format hardware video
//! encoders (VAAPI/QSV/NVENC/VideoToolbox) consume.
//!
//! NV12 layout (what this writes, tight-packed into a storage buffer):
//! - `[0, W*H)` Y plane, one byte/pixel, row stride `W`
//! - `[W*H, W*H + W*H/2)` UV plane, interleaved `U,V` at 4:2:0, row stride `W`
//! (`W/2` chroma columns × 2 bytes), `H/2` rows
//!
//! Same BT.709 full-range matrix as the editor's planar YUV420p path, so colors match.
//! Requires `W % 8 == 0 && H % 2 == 0` (the shader packs 4 bytes per `u32`).
/// `true` when [`Nv12Converter`] can handle these dimensions (else caller pads/falls back).
pub fn supports(width: u32, height: u32) -> bool {
width % 8 == 0 && height % 2 == 0 && width > 0 && height > 0
}
/// Tight NV12 byte length for `width`×`height`.
pub fn nv12_len(width: u32, height: u32) -> usize {
(width * height + width * (height / 2)) as usize
}
/// Compute pipeline: `Rgba8Unorm` texture → tight NV12 storage buffer.
pub struct Nv12Converter {
y_pipeline: wgpu::ComputePipeline,
uv_pipeline: wgpu::ComputePipeline,
bind_group_layout: wgpu::BindGroupLayout,
}
impl Nv12Converter {
pub fn new(device: &wgpu::Device) -> Self {
let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("nv12_bgl"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: false },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
],
});
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("nv12_pl"),
bind_group_layouts: &[&bind_group_layout],
push_constant_ranges: &[],
});
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("nv12_shader"),
source: wgpu::ShaderSource::Wgsl(SHADER.into()),
});
let mk = |entry: &str| {
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some("nv12_pipeline"),
layout: Some(&pipeline_layout),
module: &shader,
entry_point: Some(entry),
compilation_options: wgpu::PipelineCompilationOptions::default(),
cache: None,
})
};
Self {
y_pipeline: mk("y_main"),
uv_pipeline: mk("uv_main"),
bind_group_layout,
}
}
/// Record RGBA→NV12 into `encoder`. `out_buffer` must be `STORAGE | COPY_SRC` of at
/// least [`nv12_len`] bytes. Caller must ensure [`supports`]`(width, height)`.
pub fn convert(
&self,
device: &wgpu::Device,
encoder: &mut wgpu::CommandEncoder,
rgba_view: &wgpu::TextureView,
out_buffer: &wgpu::Buffer,
width: u32,
height: u32,
) {
debug_assert!(supports(width, height));
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("nv12_bg"),
layout: &self.bind_group_layout,
entries: &[
wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(rgba_view) },
wgpu::BindGroupEntry { binding: 1, resource: out_buffer.as_entire_binding() },
],
});
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("nv12_pass"),
timestamp_writes: None,
});
pass.set_bind_group(0, &bind_group, &[]);
let wg = 8u32;
// Y: one thread per 4 horizontal luma samples.
pass.set_pipeline(&self.y_pipeline);
pass.dispatch_workgroups(((width / 4) + wg - 1) / wg, (height + wg - 1) / wg, 1);
// UV: one thread per 4 interleaved UV bytes = 2 chroma columns; (W/4)×(H/2) threads.
pass.set_pipeline(&self.uv_pipeline);
pass.dispatch_workgroups(((width / 4) + wg - 1) / wg, ((height / 2) + wg - 1) / wg, 1);
}
}
/// CPU reference producing the exact bytes the shader should — used by tests to verify
/// the GPU output on real hardware.
pub fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec<u8> {
let w = width as usize;
let h = height as usize;
let mut out = vec![0u8; nv12_len(width, height)];
let to_byte = |v: f32| (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8;
let px = |x: usize, y: usize| {
let i = (y * w + x) * 4;
[rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0]
};
// Y
for y in 0..h {
for x in 0..w {
let p = px(x, y);
out[y * w + x] = to_byte(0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2]);
}
}
// Interleaved UV (2x2 box average)
let y_size = w * h;
for cy in 0..h / 2 {
for cx in 0..w / 2 {
let mut acc = [0.0f32; 3];
for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] {
let p = px(2 * cx + dx, 2 * cy + dy);
acc[0] += p[0]; acc[1] += p[1]; acc[2] += p[2];
}
let a = [acc[0] / 4.0, acc[1] / 4.0, acc[2] / 4.0];
let u = -0.1146 * a[0] - 0.3854 * a[1] + 0.5000 * a[2] + 0.5;
let v = 0.5000 * a[0] - 0.4542 * a[1] - 0.0458 * a[2] + 0.5;
out[y_size + cy * w + 2 * cx] = to_byte(u);
out[y_size + cy * w + 2 * cx + 1] = to_byte(v);
}
}
out
}
const SHADER: &str = r#"
@group(0) @binding(0) var input_rgba: texture_2d<f32>;
@group(0) @binding(1) var<storage, read_write> out_buf: array<u32>;
fn to_byte(v: f32) -> u32 { return u32(clamp(v, 0.0, 1.0) * 255.0 + 0.5); }
// Y plane: pack 4 horizontal luma bytes.
@compute @workgroup_size(8, 8, 1)
fn y_main(@builtin(global_invocation_id) gid: vec3<u32>) {
let dims = textureDimensions(input_rgba);
let w = dims.x;
let h = dims.y;
let x4 = gid.x * 4u;
let y = gid.y;
if (x4 >= w || y >= h) { return; }
var packed: u32 = 0u;
for (var i = 0u; i < 4u; i = i + 1u) {
let c = textureLoad(input_rgba, vec2<u32>(x4 + i, y), 0).rgb;
let yy = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b;
packed = packed | (to_byte(yy) << (8u * i));
}
out_buf[(y * w + x4) / 4u] = packed;
}
// UV plane: each thread writes 4 interleaved bytes = U0 V0 U1 V1 for 2 chroma columns.
@compute @workgroup_size(8, 8, 1)
fn uv_main(@builtin(global_invocation_id) gid: vec3<u32>) {
let dims = textureDimensions(input_rgba);
let w = dims.x;
let h = dims.y;
let k = gid.x; // chroma-column pair index: covers columns 2k, 2k+1
let cy = gid.y;
if (k * 2u >= w / 2u || cy >= h / 2u) { return; }
let y_size = w * h;
var packed: u32 = 0u;
for (var j = 0u; j < 2u; j = j + 1u) {
let cx = 2u * k + j; // chroma column
let sx = 2u * cx;
let sy = 2u * cy;
let p00 = textureLoad(input_rgba, vec2<u32>(sx, sy), 0).rgb;
let p10 = textureLoad(input_rgba, vec2<u32>(sx + 1u, sy), 0).rgb;
let p01 = textureLoad(input_rgba, vec2<u32>(sx, sy + 1u), 0).rgb;
let p11 = textureLoad(input_rgba, vec2<u32>(sx + 1u, sy + 1u), 0).rgb;
let a = (p00 + p10 + p01 + p11) * 0.25;
let u = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b + 0.5;
let v = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b + 0.5;
packed = packed | (to_byte(u) << (16u * j)); // byte 0 or 2
packed = packed | (to_byte(v) << (16u * j + 8u)); // byte 1 or 3
}
// UV row stride is w bytes; this thread writes 4 bytes at column 4k.
out_buf[(y_size + cy * w + 4u * k) / 4u] = packed;
}
"#;

145
lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs Normal file
View File

@ -0,0 +1,145 @@
//! Fragment-shader RGBA→NV12 conversion that **renders** luma/chroma into the encoder
//! surface's plane textures (R8 Y, RG8 UV). Render targets (not compute storage) so it
//! works with the DMA-BUF-imported plane images, which aren't storage-writable.
//!
//! BT.709 full-range, matching `nv12::cpu_reference` and the encoder's color tags.
/// Converts a bound RGBA texture into a Y plane (R8) and a UV plane (RG8) via two passes.
pub struct Rgba2Nv12 {
y_pipeline: wgpu::RenderPipeline,
uv_pipeline: wgpu::RenderPipeline,
bgl: wgpu::BindGroupLayout,
}
impl Rgba2Nv12 {
pub fn new(device: &wgpu::Device) -> Self {
let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("rgba2nv12_bgl"),
entries: &[wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: false },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
}],
});
let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("rgba2nv12_pl"),
bind_group_layouts: &[&bgl],
push_constant_ranges: &[],
});
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("rgba2nv12_shader"),
source: wgpu::ShaderSource::Wgsl(SHADER.into()),
});
let mk = |fs: &str, fmt: wgpu::TextureFormat| {
device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("rgba2nv12_pipeline"),
layout: Some(&layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: Some("vs_main"),
buffers: &[],
compilation_options: Default::default(),
},
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: Some(fs),
targets: &[Some(fmt.into())],
compilation_options: Default::default(),
}),
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleList,
..Default::default()
},
depth_stencil: None,
multisample: Default::default(),
multiview: None,
cache: None,
})
};
Self {
y_pipeline: mk("y_fs", wgpu::TextureFormat::R8Unorm),
uv_pipeline: mk("uv_fs", wgpu::TextureFormat::Rg8Unorm),
bgl,
}
}
/// Record both plane passes. `y_view`/`uv_view` are the R8/RG8 plane render targets.
pub fn convert(
&self,
device: &wgpu::Device,
encoder: &mut wgpu::CommandEncoder,
rgba_view: &wgpu::TextureView,
y_view: &wgpu::TextureView,
uv_view: &wgpu::TextureView,
) {
let bg = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("rgba2nv12_bg"),
layout: &self.bgl,
entries: &[wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::TextureView(rgba_view),
}],
});
for (pipeline, view) in [(&self.y_pipeline, y_view), (&self.uv_pipeline, uv_view)] {
let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("rgba2nv12_pass"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view,
resolve_target: None,
depth_slice: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
store: wgpu::StoreOp::Store,
},
})],
depth_stencil_attachment: None,
timestamp_writes: None,
occlusion_query_set: None,
});
pass.set_pipeline(pipeline);
pass.set_bind_group(0, &bg, &[]);
pass.draw(0..3, 0..1);
}
}
}
const SHADER: &str = r#"
@group(0) @binding(0) var input_rgba: texture_2d<f32>;
// Fullscreen triangle.
@vertex
fn vs_main(@builtin(vertex_index) vi: u32) -> @builtin(position) vec4<f32> {
let x = f32((vi << 1u) & 2u);
let y = f32(vi & 2u);
return vec4<f32>(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0);
}
fn load(p: vec2<i32>) -> vec3<f32> {
return textureLoad(input_rgba, p, 0).rgb;
}
// Y plane (full res): one luma byte per pixel.
@fragment
fn y_fs(@builtin(position) pos: vec4<f32>) -> @location(0) vec4<f32> {
let c = load(vec2<i32>(i32(pos.x), i32(pos.y)));
let y = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b;
return vec4<f32>(y, 0.0, 0.0, 1.0);
}
// UV plane (half res): 2x2 box-averaged chroma, interleaved into RG.
@fragment
fn uv_fs(@builtin(position) pos: vec4<f32>) -> @location(0) vec4<f32> {
let sx = 2 * i32(pos.x);
let sy = 2 * i32(pos.y);
let a = (load(vec2<i32>(sx, sy)) + load(vec2<i32>(sx + 1, sy))
+ load(vec2<i32>(sx, sy + 1)) + load(vec2<i32>(sx + 1, sy + 1))) * 0.25;
let u = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b + 0.5;
let v = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b + 0.5;
return vec4<f32>(u, v, 0.0, 1.0);
}
"#;

513
lightningbeam-ui/gpu-video-encoder/src/vaapi.rs Normal file
View File

@ -0,0 +1,513 @@
//! VAAPI hardware H.264 encoding (Linux/Intel/AMD).
//!
//! Level 1 (this module first): a CPU-fed encoder — upload NV12 frames to VAAPI
//! surfaces (`av_hwframe_transfer_data`) and encode with `h264_vaapi`. This proves the
//! encoder works and establishes the FFI scaffolding. Level 2 (zero-copy: GPU writes
//! NV12 straight into the VAAPI surface via DMA-BUF) builds on this.
//!
//! All `unsafe` FFmpeg FFI is contained here.
use ffmpeg_sys_next as ff;
use std::ffi::CString;
use std::ptr;
#[inline]
fn averror(e: i32) -> i32 {
-e
}
/// Create a VAAPI hwdevice on `/dev/dri/renderD128`, trying driver names in turn.
///
/// libva's auto-selection can pick a driver that doesn't support the GPU — notably it
/// chooses the legacy `i965` driver on newer Intel parts (Gen 11+) where the modern `iHD`
/// driver is required. Each `av_hwdevice_ctx_create` opens a fresh VADisplay, so
/// `LIBVA_DRIVER_NAME` is re-read per attempt. We try `iHD` first (modern Intel), then the
/// caller's original setting, then `i965` (older Intel) and `radeonsi` (AMD). On success the
/// working driver name is left in the env; on total failure the original value is restored.
pub fn create_device() -> Result<*mut ff::AVBufferRef, String> {
unsafe {
let node = CString::new("/dev/dri/renderD128").unwrap();
let original = std::env::var_os("LIBVA_DRIVER_NAME");
let attempts: [Option<&str>; 4] = [Some("iHD"), None, Some("i965"), Some("radeonsi")];
for drv in attempts {
match drv {
Some(d) => std::env::set_var("LIBVA_DRIVER_NAME", d),
// `None` = the caller's original setting (or libva auto if unset).
None => match &original {
Some(v) => std::env::set_var("LIBVA_DRIVER_NAME", v),
None => std::env::remove_var("LIBVA_DRIVER_NAME"),
},
}
let mut hw: *mut ff::AVBufferRef = ptr::null_mut();
if ff::av_hwdevice_ctx_create(
&mut hw,
ff::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
node.as_ptr(),
ptr::null_mut(),
0,
) >= 0
{
return Ok(hw);
}
}
match &original {
Some(v) => std::env::set_var("LIBVA_DRIVER_NAME", v),
None => std::env::remove_var("LIBVA_DRIVER_NAME"),
}
Err("av_hwdevice_ctx_create(VAAPI) failed for all drivers (iHD/i965/radeonsi)".into())
}
}
/// Copy tight NV12 (`Y` then interleaved `UV`) into an AVFrame's planes, respecting
/// each plane's linesize (which FFmpeg may pad).
unsafe fn fill_nv12(frame: *mut ff::AVFrame, nv12: &[u8], width: u32, height: u32) {
let w = width as usize;
let h = height as usize;
// Y plane: h rows of w bytes.
let dst_y = (*frame).data[0];
let ls_y = (*frame).linesize[0] as usize;
for row in 0..h {
let src = &nv12[row * w..row * w + w];
ptr::copy_nonoverlapping(src.as_ptr(), dst_y.add(row * ls_y), w);
}
// UV plane: h/2 rows of w bytes (interleaved U,V), source offset starts at w*h.
let dst_uv = (*frame).data[1];
let ls_uv = (*frame).linesize[1] as usize;
let uv_off = w * h;
for row in 0..h / 2 {
let src = &nv12[uv_off + row * w..uv_off + row * w + w];
ptr::copy_nonoverlapping(src.as_ptr(), dst_uv.add(row * ls_uv), w);
}
}
/// A VAAPI NV12 surface mapped to a DMA-BUF, with its layout extracted for Vulkan import.
/// Keeps the FFmpeg handles alive; the `fd` stays valid until drop (dup it for Vulkan).
pub struct MappedSurface {
hw_device: *mut ff::AVBufferRef,
frames_ref: *mut ff::AVBufferRef,
surf: *mut ff::AVFrame,
drm: *mut ff::AVFrame,
pub width: u32,
pub height: u32,
pub fd: i32,
pub size: u64,
pub modifier: u64,
pub y_offset: u64,
pub y_pitch: u64,
pub uv_offset: u64,
pub uv_pitch: u64,
}
impl MappedSurface {
/// Allocate a VAAPI NV12 surface and map it to DRM-PRIME.
pub fn alloc(width: u32, height: u32) -> Result<Self, String> {
unsafe {
let mut hw_device: *mut ff::AVBufferRef = ptr::null_mut();
let node = CString::new("/dev/dri/renderD128").unwrap();
if ff::av_hwdevice_ctx_create(
&mut hw_device,
ff::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
node.as_ptr(),
ptr::null_mut(),
0,
) < 0
{
return Err("av_hwdevice_ctx_create failed".into());
}
let frames_ref = ff::av_hwframe_ctx_alloc(hw_device);
if frames_ref.is_null() {
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_ctx_alloc failed".into());
}
{
let fctx = (*frames_ref).data as *mut ff::AVHWFramesContext;
(*fctx).format = ff::AVPixelFormat::AV_PIX_FMT_VAAPI;
(*fctx).sw_format = ff::AVPixelFormat::AV_PIX_FMT_NV12;
(*fctx).width = width as i32;
(*fctx).height = height as i32;
(*fctx).initial_pool_size = 4;
}
if ff::av_hwframe_ctx_init(frames_ref) < 0 {
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_ctx_init failed".into());
}
let surf = ff::av_frame_alloc();
if ff::av_hwframe_get_buffer(frames_ref, surf, 0) < 0 {
ff::av_frame_free(&mut (surf as *mut _));
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_get_buffer failed".into());
}
let drm = ff::av_frame_alloc();
(*drm).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
let flags = ff::AV_HWFRAME_MAP_DIRECT as i32
| ff::AV_HWFRAME_MAP_READ as i32
| ff::AV_HWFRAME_MAP_WRITE as i32;
if ff::av_hwframe_map(drm, surf, flags) < 0 {
ff::av_frame_free(&mut (drm as *mut _));
ff::av_frame_free(&mut (surf as *mut _));
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_map failed".into());
}
let desc = (*drm).data[0] as *const ff::AVDRMFrameDescriptor;
// Expect 1 object, 2 layers (Y=R8, UV=GR88).
if (*desc).nb_objects != 1 || (*desc).nb_layers != 2 {
return Err(format!(
"unexpected DRM layout: {} objects, {} layers",
(*desc).nb_objects, (*desc).nb_layers
));
}
let obj = &(*desc).objects[0];
let y = &(*desc).layers[0].planes[0];
let uv = &(*desc).layers[1].planes[0];
Ok(MappedSurface {
hw_device,
frames_ref,
surf,
drm,
width,
height,
fd: obj.fd,
size: obj.size as u64,
modifier: obj.format_modifier,
y_offset: y.offset as u64,
y_pitch: y.pitch as u64,
uv_offset: uv.offset as u64,
uv_pitch: uv.pitch as u64,
})
}
}
/// The underlying VASurface AVFrame (to hand to the encoder).
pub fn av_frame(&self) -> *mut ff::AVFrame {
self.surf
}
/// Read the surface back to tight CPU NV12 (for verifying what the GPU wrote).
pub fn readback_nv12(&self) -> Result<Vec<u8>, String> {
unsafe {
let sw = ff::av_frame_alloc();
(*sw).format = ff::AVPixelFormat::AV_PIX_FMT_NV12 as i32;
(*sw).width = self.width as i32;
(*sw).height = self.height as i32;
if ff::av_frame_get_buffer(sw, 0) < 0 {
ff::av_frame_free(&mut (sw as *mut _));
return Err("av_frame_get_buffer failed".into());
}
if ff::av_hwframe_transfer_data(sw, self.surf, 0) < 0 {
ff::av_frame_free(&mut (sw as *mut _));
return Err("av_hwframe_transfer_data (download) failed".into());
}
let w = self.width as usize;
let h = self.height as usize;
let mut out = vec![0u8; w * h + w * (h / 2)];
let ls_y = (*sw).linesize[0] as usize;
for row in 0..h {
let src = (*sw).data[0].add(row * ls_y);
ptr::copy_nonoverlapping(src, out.as_mut_ptr().add(row * w), w);
}
let ls_uv = (*sw).linesize[1] as usize;
let uv_off = w * h;
for row in 0..h / 2 {
let src = (*sw).data[1].add(row * ls_uv);
ptr::copy_nonoverlapping(src, out.as_mut_ptr().add(uv_off + row * w), w);
}
ff::av_frame_free(&mut (sw as *mut _));
Ok(out)
}
}
}
impl Drop for MappedSurface {
fn drop(&mut self) {
unsafe {
ff::av_frame_free(&mut (self.drm as *mut _));
ff::av_frame_free(&mut (self.surf as *mut _));
let mut fr = self.frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut self.hw_device);
}
}
}
/// Allocate one VAAPI NV12 surface, map it to a DRM-PRIME descriptor, and return a
/// human-readable dump of its DMA-BUF layout (object fds/size/modifier; layer fourcc;
/// per-plane object/offset/pitch). The format **modifier** decides the zero-copy path:
/// `0` = LINEAR (compute can write a linear NV12 buffer/image), anything else = tiled
/// (needs a GPU copy into the tiled surface, or a linear import VAAPI accepts).
pub fn probe_surface_drm(width: u32, height: u32) -> Result<String, String> {
unsafe {
let mut hw_device: *mut ff::AVBufferRef = ptr::null_mut();
let node = CString::new("/dev/dri/renderD128").unwrap();
if ff::av_hwdevice_ctx_create(
&mut hw_device,
ff::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
node.as_ptr(),
ptr::null_mut(),
0,
) < 0
{
return Err("av_hwdevice_ctx_create(VAAPI) failed".into());
}
let frames_ref = ff::av_hwframe_ctx_alloc(hw_device);
if frames_ref.is_null() {
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_ctx_alloc failed".into());
}
{
let fctx = (*frames_ref).data as *mut ff::AVHWFramesContext;
(*fctx).format = ff::AVPixelFormat::AV_PIX_FMT_VAAPI;
(*fctx).sw_format = ff::AVPixelFormat::AV_PIX_FMT_NV12;
(*fctx).width = width as i32;
(*fctx).height = height as i32;
(*fctx).initial_pool_size = 2;
}
if ff::av_hwframe_ctx_init(frames_ref) < 0 {
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_ctx_init failed".into());
}
let surf = ff::av_frame_alloc();
if ff::av_hwframe_get_buffer(frames_ref, surf, 0) < 0 {
ff::av_frame_free(&mut (surf as *mut _));
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut hw_device);
return Err("av_hwframe_get_buffer failed".into());
}
let drm = ff::av_frame_alloc();
(*drm).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
let flags = ff::AV_HWFRAME_MAP_DIRECT as i32
| ff::AV_HWFRAME_MAP_READ as i32
| ff::AV_HWFRAME_MAP_WRITE as i32;
let r = ff::av_hwframe_map(drm, surf, flags);
if r < 0 {
ff::av_frame_free(&mut (drm as *mut _));
ff::av_frame_free(&mut (surf as *mut _));
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut hw_device);
return Err(format!("av_hwframe_map(DRM_PRIME) failed: {r}"));
}
let desc = (*drm).data[0] as *const ff::AVDRMFrameDescriptor;
let mut s = format!("VAAPI NV12 {width}x{height} surface as DRM-PRIME:\n");
s += &format!(" nb_objects = {}\n", (*desc).nb_objects);
for o in 0..(*desc).nb_objects as usize {
let obj = &(*desc).objects[o];
s += &format!(
" object[{o}]: fd={} size={} format_modifier=0x{:016x}{}\n",
obj.fd,
obj.size,
obj.format_modifier,
if obj.format_modifier == 0 { " (LINEAR)" } else { " (tiled)" },
);
}
s += &format!(" nb_layers = {}\n", (*desc).nb_layers);
for l in 0..(*desc).nb_layers as usize {
let lay = &(*desc).layers[l];
let f = lay.format;
let fourcc = [(f & 0xff) as u8, ((f >> 8) & 0xff) as u8, ((f >> 16) & 0xff) as u8, ((f >> 24) & 0xff) as u8];
s += &format!(
" layer[{l}]: format='{}' (0x{:08x}) nb_planes={}\n",
String::from_utf8_lossy(&fourcc),
f,
lay.nb_planes,
);
for p in 0..lay.nb_planes as usize {
let pl = &lay.planes[p];
s += &format!(
" plane[{p}]: object_index={} offset={} pitch={}\n",
pl.object_index, pl.offset, pl.pitch,
);
}
}
ff::av_frame_free(&mut (drm as *mut _));
ff::av_frame_free(&mut (surf as *mut _));
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::av_buffer_unref(&mut hw_device);
Ok(s)
}
}
/// Encode NV12 frames with `h264_vaapi` and write the raw Annex-B H.264 to `out_path`.
/// Returns the number of encoded packets. `Err` (rather than panic) when VAAPI/the
/// encoder is unavailable, so callers can fall back.
pub fn encode_nv12_to_file(
width: u32,
height: u32,
frames: &[Vec<u8>],
framerate: i32,
out_path: &str,
) -> Result<usize, String> {
unsafe {
// 1. VAAPI device.
let mut hw_device: *mut ff::AVBufferRef = ptr::null_mut();
let node = CString::new("/dev/dri/renderD128").unwrap();
let r = ff::av_hwdevice_ctx_create(
&mut hw_device,
ff::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI,
node.as_ptr(),
ptr::null_mut(),
0,
);
if r < 0 {
return Err(format!("av_hwdevice_ctx_create(VAAPI) failed: {r}"));
}
let cleanup_dev = |dev: *mut ff::AVBufferRef| {
let mut d = dev;
ff::av_buffer_unref(&mut d);
};
// 2. Encoder.
let name = CString::new("h264_vaapi").unwrap();
let codec = ff::avcodec_find_encoder_by_name(name.as_ptr());
if codec.is_null() {
cleanup_dev(hw_device);
return Err("encoder h264_vaapi not found in this FFmpeg build".into());
}
let enc = ff::avcodec_alloc_context3(codec);
if enc.is_null() {
cleanup_dev(hw_device);
return Err("avcodec_alloc_context3 failed".into());
}
(*enc).width = width as i32;
(*enc).height = height as i32;
(*enc).time_base = ff::AVRational { num: 1, den: framerate };
(*enc).framerate = ff::AVRational { num: framerate, den: 1 };
(*enc).pix_fmt = ff::AVPixelFormat::AV_PIX_FMT_VAAPI;
// 3. HW frames context (VAAPI surfaces with NV12 sw layout).
let frames_ref = ff::av_hwframe_ctx_alloc(hw_device);
if frames_ref.is_null() {
ff::avcodec_free_context(&mut (enc as *mut _));
cleanup_dev(hw_device);
return Err("av_hwframe_ctx_alloc failed".into());
}
{
let fctx = (*frames_ref).data as *mut ff::AVHWFramesContext;
(*fctx).format = ff::AVPixelFormat::AV_PIX_FMT_VAAPI;
(*fctx).sw_format = ff::AVPixelFormat::AV_PIX_FMT_NV12;
(*fctx).width = width as i32;
(*fctx).height = height as i32;
(*fctx).initial_pool_size = 8;
}
let r = ff::av_hwframe_ctx_init(frames_ref);
if r < 0 {
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::avcodec_free_context(&mut (enc as *mut _));
cleanup_dev(hw_device);
return Err(format!("av_hwframe_ctx_init failed: {r}"));
}
(*enc).hw_frames_ctx = ff::av_buffer_ref(frames_ref);
// 4. Open.
let r = ff::avcodec_open2(enc, codec, ptr::null_mut());
if r < 0 {
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::avcodec_free_context(&mut (enc as *mut _));
cleanup_dev(hw_device);
return Err(format!("avcodec_open2(h264_vaapi) failed: {r}"));
}
let mut out: Vec<u8> = Vec::new();
let pkt = ff::av_packet_alloc();
let mut count = 0usize;
// Drain helper: pull packets and append to `out`.
let drain = |enc: *mut ff::AVCodecContext, out: &mut Vec<u8>, count: &mut usize| -> Result<(), String> {
loop {
let r = ff::avcodec_receive_packet(enc, pkt);
if r == averror(libc::EAGAIN) || r == ff::AVERROR_EOF {
break;
}
if r < 0 {
return Err(format!("avcodec_receive_packet failed: {r}"));
}
let data = std::slice::from_raw_parts((*pkt).data, (*pkt).size as usize);
out.extend_from_slice(data);
*count += 1;
ff::av_packet_unref(pkt);
}
Ok(())
};
let mut err: Option<String> = None;
for (i, nv12) in frames.iter().enumerate() {
// Software NV12 frame.
let sw = ff::av_frame_alloc();
(*sw).format = ff::AVPixelFormat::AV_PIX_FMT_NV12 as i32;
(*sw).width = width as i32;
(*sw).height = height as i32;
if ff::av_frame_get_buffer(sw, 0) < 0 {
err = Some("av_frame_get_buffer(sw) failed".into());
ff::av_frame_free(&mut (sw as *mut _));
break;
}
fill_nv12(sw, nv12, width, height);
// VAAPI surface frame + upload.
let hw = ff::av_frame_alloc();
if ff::av_hwframe_get_buffer(frames_ref, hw, 0) < 0 {
err = Some("av_hwframe_get_buffer failed".into());
ff::av_frame_free(&mut (sw as *mut _));
ff::av_frame_free(&mut (hw as *mut _));
break;
}
if ff::av_hwframe_transfer_data(hw, sw, 0) < 0 {
err = Some("av_hwframe_transfer_data failed".into());
ff::av_frame_free(&mut (sw as *mut _));
ff::av_frame_free(&mut (hw as *mut _));
break;
}
(*hw).pts = i as i64;
let r = ff::avcodec_send_frame(enc, hw);
ff::av_frame_free(&mut (sw as *mut _));
ff::av_frame_free(&mut (hw as *mut _));
if r < 0 {
err = Some(format!("avcodec_send_frame failed: {r}"));
break;
}
if let Err(e) = drain(enc, &mut out, &mut count) {
err = Some(e);
break;
}
}
// Flush.
if err.is_none() {
ff::avcodec_send_frame(enc, ptr::null_mut());
if let Err(e) = drain(enc, &mut out, &mut count) {
err = Some(e);
}
}
// Cleanup.
ff::av_packet_free(&mut (pkt as *mut _));
let mut fr = frames_ref;
ff::av_buffer_unref(&mut fr);
ff::avcodec_free_context(&mut (enc as *mut _));
cleanup_dev(hw_device);
if let Some(e) = err {
return Err(e);
}
std::fs::write(out_path, &out).map_err(|e| format!("write {out_path}: {e}"))?;
Ok(count)
}
}

152
lightningbeam-ui/gpu-video-encoder/src/vk_device.rs Normal file
View File

@ -0,0 +1,152 @@
//! Custom wgpu Vulkan device that additionally enables `VK_EXT_image_drm_format_modifier`
//! (plus the external-memory extensions wgpu-hal already turns on), so we can import a
//! tiled VAAPI NV12 DMA-BUF as a Vulkan image. wgpu's safe API can't add arbitrary device
//! extensions, so we build the `VkDevice` ourselves and wrap it via `device_from_raw`.
//!
//! All `unsafe` is contained here. Returns owned handles the caller must keep alive
//! together (instance → adapter → device/queue).
use ash::vk;
use std::ffi::CStr;
/// A wgpu device/queue backed by a hand-built Vulkan device with DMA-BUF import enabled.
pub struct DrmDevice {
// Order matters for drop; wgpu handles refcount internally but we keep these owned.
pub device: wgpu::Device,
pub queue: wgpu::Queue,
pub adapter: wgpu::Adapter,
pub instance: wgpu::Instance,
/// The raw VkDevice (for the ash image-import calls in `dmabuf.rs`).
pub raw_device: ash::Device,
pub raw_physical_device: vk::PhysicalDevice,
pub raw_instance: ash::Instance,
}
/// Create the device, or `Err` if Vulkan/the extension isn't available (caller falls back).
pub fn create() -> Result<DrmDevice, String> {
unsafe { create_inner() }
}
unsafe fn create_inner() -> Result<DrmDevice, String> {
use wgpu_hal::vulkan::Api as Vk;
// Bring the HAL Instance trait into scope for `init` / `enumerate_adapters`.
use wgpu_hal::Instance as _;
// 1. HAL instance.
let hal_instance = wgpu_hal::vulkan::Instance::init(&wgpu_hal::InstanceDescriptor {
name: "gpu-video-encoder",
flags: wgpu::InstanceFlags::empty(),
memory_budget_thresholds: Default::default(),
backend_options: Default::default(),
})
.map_err(|e| format!("vulkan instance init failed: {e:?}"))?;
let ash_instance = hal_instance.shared_instance().raw_instance().clone();
// 2. Pick an adapter (prefer the integrated/discrete GPU).
let mut exposed_adapters = hal_instance.enumerate_adapters(None);
if exposed_adapters.is_empty() {
return Err("no Vulkan adapters".into());
}
// Prefer a real GPU over CPU/llvmpipe.
exposed_adapters.sort_by_key(|a| match a.info.device_type {
wgpu::DeviceType::DiscreteGpu => 0,
wgpu::DeviceType::IntegratedGpu => 1,
_ => 2,
});
let exposed = exposed_adapters.into_iter().next().unwrap();
let phys = exposed.adapter.raw_physical_device();
// 3. Queue family with graphics + compute.
let qf_props = ash_instance.get_physical_device_queue_family_properties(phys);
let family_index = qf_props
.iter()
.position(|p| {
p.queue_flags
.contains(vk::QueueFlags::GRAPHICS | vk::QueueFlags::COMPUTE)
})
.ok_or("no graphics+compute queue family")? as u32;
// 4. Extensions: what wgpu-hal wants + DRM modifier import set.
let mut ext_names: Vec<&'static CStr> =
exposed.adapter.required_device_extensions(exposed.features);
// Only the genuine extensions; external_memory / bind_memory2 / ycbcr / format_list
// are core in Vulkan 1.1+ (this device is 1.3) so they need no enabling.
let extra: &[&'static CStr] = &[
ash::ext::image_drm_format_modifier::NAME,
ash::khr::external_memory_fd::NAME,
ash::ext::external_memory_dma_buf::NAME,
ash::ext::queue_family_foreign::NAME,
];
for e in extra {
if !ext_names.contains(e) {
ext_names.push(e);
}
}
let ext_ptrs: Vec<*const i8> = ext_names.iter().map(|c| c.as_ptr()).collect();
// 5. Enable all supported physical-device features (so wgpu has what it needs) plus
// sampler YCbCr conversion (required for the NV12 multi-planar image).
let supported = ash_instance.get_physical_device_features(phys);
let mut ycbcr =
vk::PhysicalDeviceSamplerYcbcrConversionFeatures::default().sampler_ycbcr_conversion(true);
let priorities = [1.0f32];
let queue_info = vk::DeviceQueueCreateInfo::default()
.queue_family_index(family_index)
.queue_priorities(&priorities);
let queue_infos = [queue_info];
let create_info = vk::DeviceCreateInfo::default()
.queue_create_infos(&queue_infos)
.enabled_extension_names(&ext_ptrs)
.enabled_features(&supported)
.push_next(&mut ycbcr);
let ash_device = ash_instance
.create_device(phys, &create_info, None)
.map_err(|e| format!("vkCreateDevice failed: {e:?}"))?;
// 6. Wrap the raw device into a hal OpenDevice, then a wgpu device.
let open_device = exposed
.adapter
.device_from_raw(
ash_device.clone(),
None,
&ext_names,
exposed.features,
&wgpu::MemoryHints::default(),
family_index,
0,
)
.map_err(|e| format!("device_from_raw failed: {e:?}"))?;
let raw_physical_device = phys;
let wgpu_instance = wgpu::Instance::from_hal::<Vk>(hal_instance);
let wgpu_adapter = wgpu_instance.create_adapter_from_hal::<Vk>(exposed);
let (device, queue) = wgpu_adapter
.create_device_from_hal::<Vk>(
open_device,
&wgpu::DeviceDescriptor {
label: Some("drm-import-device"),
required_features: wgpu::Features::empty(),
// Vello's compute pipelines need more than downlevel limits (e.g.
// max_storage_buffers_per_shader_stage >= 5). This device only ever runs on a
// real VAAPI-capable GPU, so request the adapter's full limits.
required_limits: wgpu_adapter.limits(),
..Default::default()
},
)
.map_err(|e| format!("create_device_from_hal failed: {e:?}"))?;
Ok(DrmDevice {
device,
queue,
adapter: wgpu_adapter,
instance: wgpu_instance,
raw_device: ash_device,
raw_physical_device,
raw_instance: ash_instance,
})
}

42
lightningbeam-ui/gpu-video-encoder/tests/drm_device.rs Normal file
View File

@ -0,0 +1,42 @@
//! Step 1 of zero-copy: the custom Vulkan device with DMA-BUF import extensions builds
//! and can do a trivial GPU op. Skips (passes) when Vulkan is unavailable.
#![cfg(target_os = "linux")]
#[test]
fn drm_device_creates_and_works() {
let dev = match gpu_video_encoder::vk_device::create() {
Ok(d) => d,
Err(e) => {
eprintln!("[drm-device] unavailable, skipping: {e}");
return;
}
};
eprintln!("[drm-device] created custom Vulkan device OK");
// Trivial sanity op: write+read a small buffer, proving the wrapped device is usable.
let data: Vec<u8> = (0..256u32).map(|i| i as u8).collect();
let src = wgpu::util::DeviceExt::create_buffer_init(
&dev.device,
&wgpu::util::BufferInitDescriptor {
label: Some("src"),
contents: &data,
usage: wgpu::BufferUsages::COPY_SRC,
},
);
let dst = dev.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("dst"),
size: 256,
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
mapped_at_creation: false,
});
let mut enc = dev.device.create_command_encoder(&Default::default());
enc.copy_buffer_to_buffer(&src, 0, &dst, 0, 256);
dev.queue.submit(Some(enc.finish()));
let slice = dst.slice(..);
slice.map_async(wgpu::MapMode::Read, |_| {});
let _ = dev.device.poll(wgpu::PollType::wait_indefinitely());
let got = slice.get_mapped_range().to_vec();
assert_eq!(got, data, "round-trip through custom device failed");
eprintln!("[drm-device] buffer round-trip OK on custom device");
}

117
lightningbeam-ui/gpu-video-encoder/tests/gpu_nv12.rs Normal file
View File

@ -0,0 +1,117 @@
//! Real-hardware test: run the RGBA→NV12 compute on the GPU and check it byte-matches
//! the CPU reference. Skips (passes) if no GPU adapter is available.
use gpu_video_encoder::nv12::{cpu_reference, nv12_len, Nv12Converter};
fn device_queue() -> Option<(wgpu::Device, wgpu::Queue)> {
let instance = wgpu::Instance::new(&wgpu::InstanceDescriptor {
backends: wgpu::Backends::VULKAN | wgpu::Backends::GL,
..Default::default()
});
let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::HighPerformance,
force_fallback_adapter: false,
compatible_surface: None,
}))
.ok()?;
pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {
label: Some("nv12-test"),
required_features: wgpu::Features::empty(),
required_limits: wgpu::Limits::downlevel_defaults(),
..Default::default()
}))
.ok()
}
/// A deterministic, varied RGBA pattern so luma and 2x2 chroma subsampling are exercised.
fn pattern(w: u32, h: u32) -> Vec<u8> {
let mut v = Vec::with_capacity((w * h * 4) as usize);
for y in 0..h {
for x in 0..w {
v.push(((x * 37 + y * 11) % 256) as u8); // R
v.push(((x * 5 + y * 53) % 256) as u8); // G
v.push(((x * 97 + y * 17) % 256) as u8); // B
v.push(255);
}
}
v
}
#[test]
fn gpu_nv12_matches_cpu_reference() {
let Some((device, queue)) = device_queue() else {
eprintln!("[gpu_nv12] no GPU adapter; skipping");
return;
};
let (w, h) = (64u32, 16u32);
let rgba = pattern(w, h);
// Source RGBA texture.
let tex = device.create_texture(&wgpu::TextureDescriptor {
label: Some("src_rgba"),
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &tex,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
&rgba,
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w * 4), rows_per_image: Some(h) },
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
let view = tex.create_view(&Default::default());
let len = nv12_len(w, h) as u64;
let out = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("nv12_out"),
size: len,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let staging = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("nv12_staging"),
size: len,
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
mapped_at_creation: false,
});
let conv = Nv12Converter::new(&device);
let mut enc = device.create_command_encoder(&Default::default());
conv.convert(&device, &mut enc, &view, &out, w, h);
enc.copy_buffer_to_buffer(&out, 0, &staging, 0, len);
queue.submit(Some(enc.finish()));
let slice = staging.slice(..);
slice.map_async(wgpu::MapMode::Read, |_| {});
let _ = device.poll(wgpu::PollType::wait_indefinitely());
let gpu = slice.get_mapped_range().to_vec();
let cpu = cpu_reference(&rgba, w, h);
assert_eq!(gpu.len(), cpu.len(), "length mismatch");
// Allow ±1 for rounding differences between GPU and CPU float paths.
let mut max_diff = 0i32;
let mut nbad = 0;
for (i, (g, c)) in gpu.iter().zip(cpu.iter()).enumerate() {
let d = (*g as i32 - *c as i32).abs();
max_diff = max_diff.max(d);
if d > 1 {
nbad += 1;
if nbad <= 8 {
eprintln!("[gpu_nv12] byte {i}: gpu={g} cpu={c} (diff {d})");
}
}
}
eprintln!("[gpu_nv12] {}x{} NV12, max byte diff = {max_diff}", w, h);
assert_eq!(nbad, 0, "{nbad} bytes differ from CPU reference by >1");
}

66
lightningbeam-ui/gpu-video-encoder/tests/vaapi_encode.rs Normal file
View File

@ -0,0 +1,66 @@
//! Level-1 spike: prove `h264_vaapi` encodes NV12 in this environment. Skips (passes)
//! when VAAPI isn't available so it's a no-op on CI/macOS/Windows.
#![cfg(target_os = "linux")]
use gpu_video_encoder::nv12::{cpu_reference, nv12_len};
use gpu_video_encoder::vaapi::encode_nv12_to_file;
/// A moving-gradient RGBA pattern → NV12 via the CPU reference, so we feed valid frames.
fn nv12_frames(w: u32, h: u32, n: usize) -> Vec<Vec<u8>> {
(0..n)
.map(|f| {
let mut rgba = Vec::with_capacity((w * h * 4) as usize);
for y in 0..h {
for x in 0..w {
rgba.push(((x + f as u32 * 4) % 256) as u8);
rgba.push(((y + f as u32 * 2) % 256) as u8);
rgba.push(((x + y) % 256) as u8);
rgba.push(255);
}
}
let v = cpu_reference(&rgba, w, h);
assert_eq!(v.len(), nv12_len(w, h));
v
})
.collect()
}
#[test]
fn vaapi_surface_drm_layout() {
match gpu_video_encoder::vaapi::probe_surface_drm(1920, 1088) {
Ok(s) => eprintln!("[vaapi-drm]\n{s}"),
Err(e) => eprintln!("[vaapi-drm] unavailable, skipping: {e}"),
}
}
#[test]
fn vaapi_h264_encode_smoke() {
let (w, h) = (320u32, 240u32);
let frames = nv12_frames(w, h, 30);
let out = std::env::temp_dir().join("gpu_video_encoder_vaapi_smoke.h264");
let out_str = out.to_str().unwrap();
match encode_nv12_to_file(w, h, &frames, 30, out_str) {
Ok(packets) => {
let meta = std::fs::metadata(&out).expect("output file missing");
eprintln!(
"[vaapi] encoded {} packets, {} bytes -> {}",
packets,
meta.len(),
out_str
);
assert!(packets > 0, "no packets produced");
assert!(meta.len() > 0, "empty output file");
// First frame should be an IDR; Annex-B starts with a start code.
let head = std::fs::read(&out).unwrap();
assert!(
head.starts_with(&[0, 0, 0, 1]) || head.starts_with(&[0, 0, 1]),
"output is not Annex-B H.264 (no start code)"
);
}
Err(e) => {
eprintln!("[vaapi] unavailable, skipping: {e}");
}
}
}

169
lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs Normal file
View File

@ -0,0 +1,169 @@
//! End-to-end zero-copy proof: import a VAAPI NV12 surface as wgpu textures, render
//! known values into them via Vulkan, read the surface back, and verify the bytes —
//! proving the GPU wrote straight into the encoder's surface with no CPU upload.
#![cfg(target_os = "linux")]
use gpu_video_encoder::{dmabuf, nv12, render_nv12, vaapi, vk_device};
/// Render a real RGBA frame into the VAAPI surface (zero-copy) and verify the surface's
/// NV12 matches the CPU reference for that frame.
#[test]
fn zerocopy_real_frame_render() {
let drm = match vk_device::create() {
Ok(d) => d,
Err(e) => {
eprintln!("[zerocopy-real] no Vulkan, skipping: {e}");
return;
}
};
let (w, h) = (640u32, 480u32);
let surf = match vaapi::MappedSurface::alloc(w, h) {
Ok(s) => s,
Err(e) => {
eprintln!("[zerocopy-real] no VAAPI, skipping: {e}");
return;
}
};
let imported = dmabuf::import(&drm, &surf).expect("import");
// A varied RGBA pattern.
let mut rgba = Vec::with_capacity((w * h * 4) as usize);
for y in 0..h {
for x in 0..w {
rgba.push(((x * 3 + y) % 256) as u8);
rgba.push(((x + y * 2) % 256) as u8);
rgba.push(((x * 2 + y * 3) % 256) as u8);
rgba.push(255);
}
}
let src = drm.device.create_texture(&wgpu::TextureDescriptor {
label: Some("rgba_src"),
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
drm.queue.write_texture(
wgpu::TexelCopyTextureInfo { texture: &src, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
&rgba,
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w * 4), rows_per_image: Some(h) },
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
let conv = render_nv12::Rgba2Nv12::new(&drm.device);
let src_view = src.create_view(&Default::default());
let y_view = imported.y().create_view(&Default::default());
let uv_view = imported.uv().create_view(&Default::default());
let mut enc = drm.device.create_command_encoder(&Default::default());
conv.convert(&drm.device, &mut enc, &src_view, &y_view, &uv_view);
drm.queue.submit(Some(enc.finish()));
let _ = drm.device.poll(wgpu::PollType::wait_indefinitely());
let got = surf.readback_nv12().expect("readback");
let want = nv12::cpu_reference(&rgba, w, h);
assert_eq!(got.len(), want.len());
let mut max_diff = 0i32;
let mut nbad = 0;
for (g, c) in got.iter().zip(want.iter()) {
let d = (*g as i32 - *c as i32).abs();
max_diff = max_diff.max(d);
if d > 2 {
nbad += 1;
}
}
eprintln!("[zerocopy-real] {}x{} real-frame render, max diff={max_diff}, bad={nbad}/{}", w, h, got.len());
assert!(nbad * 100 < got.len(), "too many bytes differ from CPU NV12 reference");
eprintln!("[zerocopy-real] ✅ real RGBA frame rendered into VAAPI surface, NV12 matches reference");
}
#[test]
fn zerocopy_render_into_vaapi_surface() {
let drm = match vk_device::create() {
Ok(d) => d,
Err(e) => {
eprintln!("[zerocopy] no Vulkan device, skipping: {e}");
return;
}
};
let surf = match vaapi::MappedSurface::alloc(640, 480) {
Ok(s) => s,
Err(e) => {
eprintln!("[zerocopy] no VAAPI surface, skipping: {e}");
return;
}
};
eprintln!(
"[zerocopy] surface: modifier=0x{:016x} y(off={},pitch={}) uv(off={},pitch={}) size={}",
surf.modifier, surf.y_offset, surf.y_pitch, surf.uv_offset, surf.uv_pitch, surf.size
);
let imported = match dmabuf::import(&drm, &surf) {
Ok(i) => i,
Err(e) => panic!("dma-buf import failed: {e}"),
};
eprintln!("[zerocopy] imported surface as wgpu Y(R8) + UV(RG8) textures");
// Render known constants via clear: Y=0.5(->128), U=0.25(->64), V=0.75(->191).
let y_view = imported.y().create_view(&Default::default());
let uv_view = imported.uv().create_view(&Default::default());
let mut enc = drm.device.create_command_encoder(&Default::default());
{
enc.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("clear-y"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view: &y_view,
resolve_target: None,
depth_slice: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Clear(wgpu::Color { r: 0.5, g: 0.0, b: 0.0, a: 0.0 }),
store: wgpu::StoreOp::Store,
},
})],
depth_stencil_attachment: None,
timestamp_writes: None,
occlusion_query_set: None,
});
enc.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("clear-uv"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view: &uv_view,
resolve_target: None,
depth_slice: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Clear(wgpu::Color { r: 0.25, g: 0.75, b: 0.0, a: 0.0 }),
store: wgpu::StoreOp::Store,
},
})],
depth_stencil_attachment: None,
timestamp_writes: None,
occlusion_query_set: None,
});
}
drm.queue.submit(Some(enc.finish()));
let _ = drm.device.poll(wgpu::PollType::wait_indefinitely());
// Read the VAAPI surface back and check what the GPU wrote.
let nv12 = surf.readback_nv12().expect("readback");
let (w, h) = (640usize, 480usize);
let y_plane = &nv12[..w * h];
let uv_plane = &nv12[w * h..];
let near = |v: u8, t: i32| (v as i32 - t).abs() <= 3;
let y_ok = y_plane.iter().filter(|&&v| near(v, 128)).count();
let u_ok = uv_plane.iter().step_by(2).filter(|&&v| near(v, 64)).count();
let v_ok = uv_plane.iter().skip(1).step_by(2).filter(|&&v| near(v, 191)).count();
eprintln!(
"[zerocopy] Y~128: {}/{}, U~64: {}/{}, V~191: {}/{}",
y_ok, w * h, u_ok, uv_plane.len() / 2, v_ok, uv_plane.len() / 2
);
let frac = |ok: usize, n: usize| ok as f64 / n as f64;
assert!(frac(y_ok, w * h) > 0.98, "Y plane not the rendered value (sample {:?})", &y_plane[..8]);
assert!(frac(u_ok, uv_plane.len() / 2) > 0.98, "U not rendered value");
assert!(frac(v_ok, uv_plane.len() / 2) > 0.98, "V not rendered value");
eprintln!("[zerocopy] ✅ GPU rendered straight into the VAAPI surface (verified via readback)");
}

76
lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs Normal file
View File

@ -0,0 +1,76 @@
//! Capstone: encode RGBA frames fully zero-copy (GPU render → VAAPI surface → h264_vaapi)
//! and verify the output is real H.264. Skips when VAAPI is unavailable.
#![cfg(target_os = "linux")]
use gpu_video_encoder::encoder::ZeroCopyEncoder;
#[test]
fn zerocopy_encode_h264() {
let (w, h) = (640u32, 480u32);
let out = std::env::temp_dir().join("gpu_video_encoder_zerocopy.mp4");
let _ = std::fs::remove_file(&out);
let mut enc = match ZeroCopyEncoder::new(w, h, 30, 4000, &out) {
Ok(e) => e,
Err(e) => {
eprintln!("[zc-encode] unavailable, skipping: {e}");
return;
}
};
// Build one reusable RGBA source texture; update it per frame with a moving pattern.
let device = enc.device();
let src = device.create_texture(&wgpu::TextureDescriptor {
label: Some("rgba"),
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
let n = 30;
for f in 0..n {
let mut rgba = Vec::with_capacity((w * h * 4) as usize);
for y in 0..h {
for x in 0..w {
rgba.push(((x + f * 8) % 256) as u8);
rgba.push(((y + f * 4) % 256) as u8);
rgba.push(((x + y) % 256) as u8);
rgba.push(255);
}
}
enc.queue().write_texture(
wgpu::TexelCopyTextureInfo { texture: &src, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
&rgba,
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w * 4), rows_per_image: Some(h) },
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
enc.encode_rgba(&src).expect("encode_rgba");
}
enc.finish().expect("finish");
let meta = std::fs::metadata(&out).expect("output .mp4 missing");
eprintln!("[zc-encode] {} frames -> {} bytes mp4 at {}", n, meta.len(), out.display());
assert!(meta.len() > 1000, "implausibly small output");
// ffprobe-verify the container: H.264 stream, right dims, ~n frames.
let o = std::process::Command::new("ffprobe")
.args([
"-hide_banner", "-v", "error", "-count_frames",
"-show_entries", "stream=codec_name,width,height,nb_read_frames",
"-of", "default=noprint_wrappers=1",
])
.arg(&out)
.output()
.expect("run ffprobe");
let s = String::from_utf8_lossy(&o.stdout);
eprintln!("[zc-encode] ffprobe:\n{s}");
assert!(s.contains("codec_name=h264"), "ffprobe didn't see H.264");
assert!(s.contains(&format!("width={w}")), "wrong width");
assert!(s.contains(&format!("height={h}")), "wrong height");
assert!(s.contains(&format!("nb_read_frames={n}")), "expected {n} frames");
eprintln!("[zc-encode] ✅ zero-copy H.264 mp4 encode verified");
}

5
lightningbeam-ui/lightningbeam-core/Cargo.toml
View File

@ -29,6 +29,8 @@ daw-backend = { path = "../../daw-backend" }
# Video decoding # Video decoding
ffmpeg-next = "8.0" ffmpeg-next = "8.0"
# AVIO-over-Read+Seek shim: decode packed video by streaming from the SQLite blob.
ffmpeg-blob-io = { path = "../../ffmpeg-blob-io" }
lru = "0.12" lru = "0.12"
# File I/O # File I/O
@ -73,3 +75,6 @@ windows-sys = { version = "0.60", features = [
version = "0.11" version = "0.11"
[dev-dependencies] [dev-dependencies]
# For the packed-video streaming integration test (blob -> AVIO -> ffmpeg Input).
ffmpeg-blob-io = { path = "../../ffmpeg-blob-io" }
ffmpeg-next = "8.0"

2
lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs
View File

@ -12,7 +12,7 @@
//! correct). If the base is somehow not resident we skip rather than corrupt. //! correct). If the base is somehow not resident we skip rather than corrupt.
/// Normalize a buffer to full length `n`; an empty/short buffer becomes transparent. /// Normalize a buffer to full length `n`; an empty/short buffer becomes transparent.
fn normalize(buf: &[u8], n: usize) -> std::borrow::Cow<[u8]> { fn normalize(buf: &[u8], n: usize) -> std::borrow::Cow<'_, [u8]> {
if buf.len() == n { if buf.len() == n {
std::borrow::Cow::Borrowed(buf) std::borrow::Cow::Borrowed(buf)
} else { } else {

8
lightningbeam-ui/lightningbeam-core/src/clip.rs
View File

@ -343,6 +343,13 @@ pub struct VideoClip {
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub linked_audio_clip_id: Option<Uuid>, pub linked_audio_clip_id: Option<Uuid>,
/// When set, the video bytes are packed into the `.beam` container under this
/// media id (== the clip id) and decoded by streaming from the SQLite blob.
/// `None` means the video is referenced externally via [`Self::file_path`].
/// Reconstructed from the archive on load (see `load_beam_sqlite`).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub media_id: Option<Uuid>,
/// Folder this clip belongs to (None = root of category) /// Folder this clip belongs to (None = root of category)
#[serde(default)] #[serde(default)]
pub folder_id: Option<Uuid>, pub folder_id: Option<Uuid>,
@ -367,6 +374,7 @@ impl VideoClip {
duration, duration,
frame_rate, frame_rate,
linked_audio_clip_id: None, linked_audio_clip_id: None,
media_id: None,
folder_id: None, folder_id: None,
} }
} }

126
lightningbeam-ui/lightningbeam-core/src/file_io.rs
View File

@ -9,7 +9,7 @@
//! too (via [`load_beam_zip_legacy`]). Saving always writes the SQLite form, so //! too (via [`load_beam_zip_legacy`]). Saving always writes the SQLite form, so
//! opening a legacy file and saving migrates it. //! opening a legacy file and saving migrates it.
use crate::beam_archive::{BeamArchive, MediaKind, MediaMeta, LARGE_MEDIA_THRESHOLD}; use crate::beam_archive::{BeamArchive, MediaKind, MediaMeta, MediaStorage, LARGE_MEDIA_THRESHOLD};
use crate::document::Document; use crate::document::Document;
use daw_backend::audio::pool::AudioPoolEntry; use daw_backend::audio::pool::AudioPoolEntry;
use daw_backend::audio::project::Project as AudioProject; use daw_backend::audio::project::Project as AudioProject;
@ -88,15 +88,6 @@ impl Default for LargeMediaMode {
/// Settings for saving a project /// Settings for saving a project
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct SaveSettings { pub struct SaveSettings {
/// Automatically embed files smaller than this size (in bytes)
pub auto_embed_threshold_bytes: u64,
/// Force embedding all media files
pub force_embed_all: bool,
/// Force linking all media files (don't embed any)
pub force_link_all: bool,
/// How to store files at/above [`LARGE_MEDIA_THRESHOLD`] (pack vs reference). /// How to store files at/above [`LARGE_MEDIA_THRESHOLD`] (pack vs reference).
/// `Ask` behaves as `Reference` here (safe default: don't bloat the DB). /// `Ask` behaves as `Reference` here (safe default: don't bloat the DB).
pub large_media_mode: LargeMediaMode, pub large_media_mode: LargeMediaMode,
@ -105,9 +96,6 @@ pub struct SaveSettings {
impl Default for SaveSettings { impl Default for SaveSettings {
fn default() -> Self { fn default() -> Self {
Self { Self {
auto_embed_threshold_bytes: 10_000_000, // 10 MB
force_embed_all: false,
force_link_all: false,
large_media_mode: LargeMediaMode::Ask, large_media_mode: LargeMediaMode::Ask,
} }
} }
@ -269,7 +257,7 @@ pub fn save_beam(
audio_pool_entries: Vec<AudioPoolEntry>, audio_pool_entries: Vec<AudioPoolEntry>,
layer_to_track_map: &std::collections::HashMap<uuid::Uuid, u32>, layer_to_track_map: &std::collections::HashMap<uuid::Uuid, u32>,
thumbnail_blobs: &std::collections::HashMap<uuid::Uuid, Vec<u8>>, thumbnail_blobs: &std::collections::HashMap<uuid::Uuid, Vec<u8>>,
_settings: &SaveSettings, settings: &SaveSettings,
) -> Result<(), String> { ) -> Result<(), String> {
let fn_start = std::time::Instant::now(); let fn_start = std::time::Instant::now();
eprintln!("📊 [SAVE_BEAM] Starting save_beam() (SQLite container)..."); eprintln!("📊 [SAVE_BEAM] Starting save_beam() (SQLite container)...");
@ -349,7 +337,7 @@ pub fn save_beam(
// (`Ask` == reference); smaller files are always packed. // (`Ask` == reference); smaller files are always packed.
let reference_it = entry.is_video_audio let reference_it = entry.is_video_audio
|| (size >= LARGE_MEDIA_THRESHOLD || (size >= LARGE_MEDIA_THRESHOLD
&& _settings.large_media_mode != LargeMediaMode::Pack); && settings.large_media_mode != LargeMediaMode::Pack);
if reference_it { if reference_it {
referenced = Some(rel.clone()); referenced = Some(rel.clone());
} else { } else {
@ -462,6 +450,65 @@ pub fn save_beam(
} }
} }
// --- video clips -> media rows (Video bytes), keyed by the clip id ---
// Mirror the audio pack/reference decision: pack unless the file is
// >= LARGE_MEDIA_THRESHOLD and the user didn't choose Pack. Packed video is
// decoded by streaming from the blob (see video.rs `VideoSource` + ffmpeg-blob-io).
for (clip_id, clip) in &document.video_clips {
// In-place re-save: an unchanged packed video keeps its row, never re-streamed.
if clip.media_id == Some(*clip_id) && txn.media_exists(*clip_id)? {
live_media.insert(*clip_id);
continue;
}
if clip.file_path.is_empty() {
continue;
}
let full = if Path::new(&clip.file_path).is_absolute() {
PathBuf::from(&clip.file_path)
} else {
project_dir.join(&clip.file_path)
};
if !full.is_file() {
continue; // source gone — reported missing on load
}
let size = std::fs::metadata(&full).map(|m| m.len()).unwrap_or(0);
let codec = full
.extension()
.and_then(|x| x.to_str())
.unwrap_or("mp4")
.to_lowercase();
let meta = MediaMeta {
width: Some(clip.width as u32),
height: Some(clip.height as u32),
..Default::default()
};
let reference_it =
size >= LARGE_MEDIA_THRESHOLD && settings.large_media_mode != LargeMediaMode::Pack;
if reference_it {
txn.put_media_referenced(*clip_id, MediaKind::Video, &codec, &clip.file_path, meta)?;
} else {
txn.put_media_packed_from_path(*clip_id, MediaKind::Video, &codec, &full, meta)?;
// The video's audio track lives in this same blob. Point the linked
// video-audio pool entry at the video's media row so its audio streams
// from the blob too (instead of re-probing the external file on load).
if let Some(aid) = clip.linked_audio_clip_id {
if let Some(crate::clip::AudioClipType::Sampled { audio_pool_index }) =
document.audio_clips.get(&aid).map(|c| &c.clip_type)
{
if let Some(e) = modified_entries
.iter_mut()
.find(|e| e.pool_index == *audio_pool_index)
{
e.media_id = Some(clip_id.to_string());
e.relative_path = None;
e.embedded_data = None;
}
}
}
}
live_media.insert(*clip_id);
}
// --- image assets -> media rows (original file bytes), keyed by asset id --- // --- image assets -> media rows (original file bytes), keyed by asset id ---
let mut image_count = 0usize; let mut image_count = 0usize;
for (id, asset) in &document.image_assets { for (id, asset) in &document.image_assets {
@ -498,7 +545,7 @@ pub fn save_beam(
modified: now.clone(), modified: now.clone(),
ui_state: document.clone(), ui_state: document.clone(),
audio_backend: SerializedAudioBackend { audio_backend: SerializedAudioBackend {
sample_rate: 48000, // TODO: Get from audio engine sample_rate: audio_project.sample_rate(),
project: audio_project.clone(), project: audio_project.clone(),
audio_pool_entries: modified_entries, audio_pool_entries: modified_entries,
layer_to_track_map: layer_to_track_map.clone(), layer_to_track_map: layer_to_track_map.clone(),
@ -658,7 +705,7 @@ fn load_beam_sqlite(path: &Path) -> Result<LoadedProject, String> {
// Missing external files (referenced entries whose file no longer exists). // Missing external files (referenced entries whose file no longer exists).
let project_dir = path.parent().unwrap_or_else(|| Path::new(".")); let project_dir = path.parent().unwrap_or_else(|| Path::new("."));
let missing_files: Vec<MissingFileInfo> = restored_entries let mut missing_files: Vec<MissingFileInfo> = restored_entries
.iter() .iter()
.enumerate() .enumerate()
.filter_map(|(idx, entry)| { .filter_map(|(idx, entry)| {
@ -682,6 +729,42 @@ fn load_beam_sqlite(path: &Path) -> Result<LoadedProject, String> {
}) })
.collect(); .collect();
// Resolve packed/referenced video bytes per clip (mirrors audio resolution).
// A `MediaKind::Video` row keyed by the clip id means the bytes are in the
// container; the decoder streams from the blob. Referenced rows update the
// external path and report a missing source. Old projects (no Video row) stay
// referenced via their existing `file_path`.
for (clip_id, clip) in document.video_clips.iter_mut() {
match archive.media_info(*clip_id) {
Ok(Some(info)) if info.kind == MediaKind::Video => match info.storage {
MediaStorage::Packed => {
clip.media_id = Some(*clip_id);
}
MediaStorage::Referenced => {
clip.media_id = None;
if let Some(p) = info.ext_path.clone() {
clip.file_path = p;
}
let full = if Path::new(&clip.file_path).is_absolute() {
PathBuf::from(&clip.file_path)
} else {
project_dir.join(&clip.file_path)
};
if !clip.file_path.is_empty() && !full.exists() {
missing_files.push(MissingFileInfo {
pool_index: 0,
original_path: full,
file_type: MediaFileType::Video,
});
}
}
},
_ => {
clip.media_id = None;
}
}
}
// Persisted video thumbnail packs (opaque LBTN blobs), keyed by clip id. The // Persisted video thumbnail packs (opaque LBTN blobs), keyed by clip id. The
// editor decodes + inserts them and skips regeneration for these clips. // editor decodes + inserts them and skips regeneration for these clips.
let mut thumbnail_blobs = std::collections::HashMap::new(); let mut thumbnail_blobs = std::collections::HashMap::new();
@ -879,8 +962,11 @@ fn load_beam_zip_legacy(path: &Path) -> Result<LoadedProject, String> {
for layer in document.root.children.iter_mut() { for layer in document.root.children.iter_mut() {
if let crate::layer::AnyLayer::Raster(rl) = layer { if let crate::layer::AnyLayer::Raster(rl) = layer {
for kf in &mut rl.keyframes { for kf in &mut rl.keyframes {
if !kf.media_path.is_empty() { // Legacy ZIP raster entries are named "media/raster/<uuid>.png",
match zip.by_name(&kf.media_path) { // derivable from the keyframe id (the old `media_path` field).
let entry_path = format!("media/raster/{}.png", kf.id);
{
match zip.by_name(&entry_path) {
Ok(mut png_file) => { Ok(mut png_file) => {
let mut png_bytes = Vec::new(); let mut png_bytes = Vec::new();
let _ = png_file.read_to_end(&mut png_bytes); let _ = png_file.read_to_end(&mut png_bytes);
@ -890,7 +976,7 @@ fn load_beam_zip_legacy(path: &Path) -> Result<LoadedProject, String> {
kf.raw_pixels = rgba.into_raw(); kf.raw_pixels = rgba.into_raw();
raster_load_count += 1; raster_load_count += 1;
} }
Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to decode raster PNG {}: {}", kf.media_path, e), Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to decode raster PNG {}: {}", entry_path, e),
} }
} }
Err(_) => { Err(_) => {

8
lightningbeam-ui/lightningbeam-core/src/raster_layer.rs
View File

@ -111,8 +111,6 @@ pub struct RasterKeyframe {
pub time: f64, pub time: f64,
pub width: u32, pub width: u32,
pub height: u32, pub height: u32,
/// ZIP-relative path: `"media/raster/<uuid>.png"`
pub media_path: String,
/// Stroke history (for potential replay / future non-destructive editing) /// Stroke history (for potential replay / future non-destructive editing)
pub stroke_log: Vec<StrokeRecord>, pub stroke_log: Vec<StrokeRecord>,
pub tween_after: TweenType, pub tween_after: TweenType,
@ -157,13 +155,11 @@ impl RasterKeyframe {
pub fn new(time: f64, width: u32, height: u32) -> Self { pub fn new(time: f64, width: u32, height: u32) -> Self {
let id = Uuid::new_v4(); let id = Uuid::new_v4();
let media_path = format!("media/raster/{}.png", id);
Self { Self {
id, id,
time, time,
width, width,
height, height,
media_path,
stroke_log: Vec::new(), stroke_log: Vec::new(),
tween_after: TweenType::Hold, tween_after: TweenType::Hold,
raw_pixels: Vec::new(), raw_pixels: Vec::new(),
@ -264,10 +260,6 @@ impl RasterLayer {
.map(|pos| self.keyframes.remove(pos)) .map(|pos| self.keyframes.remove(pos))
} }
/// Return the ZIP-relative PNG path for the active keyframe at `time`, or `None`.
pub fn buffer_path_at_time(&self, time: f64) -> Option<&str> {
self.keyframe_at(time).map(|kf| kf.media_path.as_str())
}
} }
// Delegate all LayerTrait methods to self.layer // Delegate all LayerTrait methods to self.layer

174
lightningbeam-ui/lightningbeam-core/src/renderer.rs
View File

@ -232,6 +232,21 @@ pub struct VideoRenderInstance {
pub opacity: f32, pub opacity: f32,
} }
/// Sink for pulling video frames out of a container layer's scene recursion, so
/// they composite via the fast GPU Video path instead of baking into Vello.
///
/// Threaded as `Option<&mut VideoExtract>` through the isolated-render scene
/// functions. When present, [`render_video_layer`] pushes a [`VideoRenderInstance`]
/// instead of drawing into the Vello scene. `drew_other` is set whenever any
/// non-video primitive (vector graph, image asset, raster) is emitted; that forces
/// the safe Vello fallback because the container mixes video with other content
/// and we can't preserve z-order by extraction alone.
#[derive(Default)]
struct VideoExtract {
instances: Vec<VideoRenderInstance>,
drew_other: bool,
}
/// Type of rendered layer for compositor handling /// Type of rendered layer for compositor handling
pub enum RenderedLayerType { pub enum RenderedLayerType {
/// Vector / group layer — Vello scene in `RenderedLayer::scene` is used. /// Vector / group layer — Vello scene in `RenderedLayer::scene` is used.
@ -428,6 +443,30 @@ pub fn render_document_for_compositing(
} }
} }
// One-shot diagnostic: dump what the compositor actually receives. Set
// LB_LAYER_DEBUG=1 to print a single snapshot of each layer's resolved type.
if std::env::var("LB_LAYER_DEBUG").is_ok() {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| {
eprintln!("[LB_LAYER_DEBUG] composite layers = {}", rendered_layers.len());
for (i, l) in rendered_layers.iter().enumerate() {
let desc = match &l.layer_type {
RenderedLayerType::Vector =>
format!("Vector (has_content={}, scene_empty={})", l.has_content, l.cpu_pixmap.is_none()),
RenderedLayerType::Raster { width, height, dirty, .. } =>
format!("Raster {width}x{height} dirty={dirty}"),
RenderedLayerType::Video { instances } =>
format!("Video ({} instance(s))", instances.len()),
RenderedLayerType::Float { width, height, .. } =>
format!("Float {width}x{height}"),
RenderedLayerType::Effect { effect_instances } =>
format!("Effect ({} instance(s))", effect_instances.len()),
};
eprintln!("[LB_LAYER_DEBUG] layer[{i}] id={} type={desc}", l.layer_id);
}
});
}
CompositeRenderResult { CompositeRenderResult {
background, background,
background_cpu: None, background_cpu: None,
@ -462,6 +501,9 @@ pub fn render_layer_isolated(
// Render layer content with full opacity (1.0) - opacity applied during compositing // Render layer content with full opacity (1.0) - opacity applied during compositing
match layer { match layer {
AnyLayer::Vector(vector_layer) => { AnyLayer::Vector(vector_layer) => {
// Render into the scene with an extraction sink so a clip that is purely
// video (no vector geometry) composites via the fast GPU Video path.
let mut ex = VideoExtract::default();
render_vector_layer_to_scene( render_vector_layer_to_scene(
document, document,
time, time,
@ -471,10 +513,28 @@ pub fn render_layer_isolated(
1.0, // Full opacity - layer opacity handled in compositing 1.0, // Full opacity - layer opacity handled in compositing
image_cache, image_cache,
video_manager, video_manager,
Some(&mut ex),
); );
rendered.has_content = vector_layer.graph_at_time(time) if !ex.instances.is_empty() && !ex.drew_other {
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted)) // Pure video: discard the (now-empty) scene and emit GPU instances.
|| !vector_layer.clip_instances.is_empty(); rendered.scene = Scene::new();
rendered.has_content = true;
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
} else {
if !ex.instances.is_empty() {
// Mixed video + vector: the first pass diverted the video out of
// the scene, so re-render with no sink to bake it back in (correct
// z-order; Vello path). Rare — fast-splitting is deferred.
rendered.scene = Scene::new();
render_vector_layer_to_scene(
document, time, vector_layer, &mut rendered.scene,
base_transform, 1.0, image_cache, video_manager, None,
);
}
rendered.has_content = vector_layer.graph_at_time(time)
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted))
|| !vector_layer.clip_instances.is_empty();
}
} }
AnyLayer::Audio(_) => { AnyLayer::Audio(_) => {
// Audio layers don't render visually // Audio layers don't render visually
@ -577,15 +637,34 @@ pub fn render_layer_isolated(
return RenderedLayer::effect_layer(layer_id, opacity, active_effects); return RenderedLayer::effect_layer(layer_id, opacity, active_effects);
} }
AnyLayer::Group(group_layer) => { AnyLayer::Group(group_layer) => {
// Render each child layer's content into the group's scene // Render each child into the group's scene with an extraction sink. The
// common imported-video case is a Group[Video, Audio] — audio draws
// nothing, so it's pure video and composites via the GPU Video path.
let mut ex = VideoExtract::default();
for child in &group_layer.children { for child in &group_layer.children {
render_layer( render_layer(
document, time, child, &mut rendered.scene, base_transform, document, time, child, &mut rendered.scene, base_transform,
1.0, // Full opacity - layer opacity handled in compositing 1.0, // Full opacity - layer opacity handled in compositing
image_cache, video_manager, camera_frame, image_cache, video_manager, camera_frame, Some(&mut ex),
); );
} }
rendered.has_content = !group_layer.children.is_empty(); if !ex.instances.is_empty() && !ex.drew_other {
rendered.scene = Scene::new();
rendered.has_content = true;
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
} else {
if !ex.instances.is_empty() {
// Mixed: re-render with no sink to bake the video back into the scene.
rendered.scene = Scene::new();
for child in &group_layer.children {
render_layer(
document, time, child, &mut rendered.scene, base_transform,
1.0, image_cache, video_manager, camera_frame, None,
);
}
}
rendered.has_content = !group_layer.children.is_empty();
}
} }
AnyLayer::Raster(raster_layer) => { AnyLayer::Raster(raster_layer) => {
if let Some(kf) = raster_layer.keyframe_at(time) { if let Some(kf) = raster_layer.keyframe_at(time) {
@ -614,6 +693,7 @@ fn render_vector_layer_to_scene(
parent_opacity: f64, parent_opacity: f64,
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
extract: Option<&mut VideoExtract>,
) { ) {
render_vector_layer( render_vector_layer(
document, document,
@ -624,6 +704,7 @@ fn render_vector_layer_to_scene(
parent_opacity, parent_opacity,
image_cache, image_cache,
video_manager, video_manager,
extract,
); );
} }
@ -691,10 +772,10 @@ pub fn render_document_with_transform(
for layer in document.visible_layers() { for layer in document.visible_layers() {
if any_soloed { if any_soloed {
if layer.soloed() { if layer.soloed() {
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None); render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
} }
} else { } else {
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None); render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
} }
} }
} }
@ -755,10 +836,11 @@ fn render_layer(
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
camera_frame: Option<&crate::webcam::CaptureFrame>, camera_frame: Option<&crate::webcam::CaptureFrame>,
mut extract: Option<&mut VideoExtract>,
) { ) {
match layer { match layer {
AnyLayer::Vector(vector_layer) => { AnyLayer::Vector(vector_layer) => {
render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager) render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager, extract)
} }
AnyLayer::Audio(_) => { AnyLayer::Audio(_) => {
// Audio layers don't render visually // Audio layers don't render visually
@ -766,18 +848,20 @@ fn render_layer(
AnyLayer::Video(video_layer) => { AnyLayer::Video(video_layer) => {
let mut video_mgr = video_manager.lock().unwrap(); let mut video_mgr = video_manager.lock().unwrap();
let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None }; let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None };
render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame); render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame, extract);
} }
AnyLayer::Effect(_) => { AnyLayer::Effect(_) => {
// Effect layers are processed during GPU compositing, not rendered to scene // Effect layers are processed during GPU compositing, not rendered to scene
} }
AnyLayer::Group(group_layer) => { AnyLayer::Group(group_layer) => {
// Render each child layer in the group // Render each child layer in the group, passing the extract sink down.
for child in &group_layer.children { for child in &group_layer.children {
render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame); render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame, extract.as_deref_mut());
} }
} }
AnyLayer::Raster(raster_layer) => { AnyLayer::Raster(raster_layer) => {
// Raster is non-video content — force the Vello fallback if extracting.
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
render_raster_layer_to_scene(raster_layer, time, scene, base_transform); render_raster_layer_to_scene(raster_layer, time, scene, base_transform);
} }
} }
@ -816,6 +900,7 @@ pub fn render_single_clip_instance(
render_clip_instance( render_clip_instance(
document, time, clip_instance, layer_opacity, scene, base_transform, document, time, clip_instance, layer_opacity, scene, base_transform,
&vector_layer.layer.animation_data, image_cache, video_manager, group_end_time, &vector_layer.layer.animation_data, image_cache, video_manager, group_end_time,
None, // edit-inside-clip overlay keeps the Vello path
); );
} }
@ -831,6 +916,7 @@ fn render_clip_instance(
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
group_end_time: Option<f64>, group_end_time: Option<f64>,
mut extract: Option<&mut VideoExtract>,
) { ) {
// Try to find the clip in the document's clip libraries // Try to find the clip in the document's clip libraries
// For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented) // For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented)
@ -972,7 +1058,7 @@ fn render_clip_instance(
if !layer_node.data.visible() { if !layer_node.data.visible() {
continue; continue;
} }
render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None); render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None, extract.as_deref_mut());
} }
} }
@ -986,6 +1072,7 @@ fn render_video_layer(
parent_opacity: f64, parent_opacity: f64,
video_manager: &mut crate::video::VideoManager, video_manager: &mut crate::video::VideoManager,
camera_frame: Option<&crate::webcam::CaptureFrame>, camera_frame: Option<&crate::webcam::CaptureFrame>,
mut extract: Option<&mut VideoExtract>,
) { ) {
use crate::animation::TransformProperty; use crate::animation::TransformProperty;
@ -1151,14 +1238,26 @@ fn render_video_layer(
Affine::IDENTITY Affine::IDENTITY
}; };
// Render video frame as image fill // Extract to the GPU Video path when a sink is present; otherwise bake into
scene.fill( // the Vello scene. The combined frame-pixel → document transform is
Fill::NonZero, // instance_transform * brush_transform (matching the top-level Video path).
instance_transform, if let Some(ex) = extract.as_deref_mut() {
&image_with_alpha, ex.instances.push(VideoRenderInstance {
Some(brush_transform), rgba_data: frame.rgba_data.clone(),
&video_rect, width: frame.width,
); height: frame.height,
transform: instance_transform * brush_transform,
opacity: final_opacity,
});
} else {
scene.fill(
Fill::NonZero,
instance_transform,
&image_with_alpha,
Some(brush_transform),
&video_rect,
);
}
clip_rendered = true; clip_rendered = true;
} }
@ -1194,13 +1293,25 @@ fn render_video_layer(
* Affine::translate((offset_x, offset_y)) * Affine::translate((offset_x, offset_y))
* Affine::scale(uniform_scale); * Affine::scale(uniform_scale);
scene.fill( // preview_transform maps frame-pixel space → document directly, so it
Fill::NonZero, // is exactly the instance transform for the GPU path.
preview_transform, if let Some(ex) = extract.as_deref_mut() {
&image_with_alpha, ex.instances.push(VideoRenderInstance {
None, rgba_data: frame.rgba_data.clone(),
&frame_rect, width: frame.width,
); height: frame.height,
transform: preview_transform,
opacity: final_opacity,
});
} else {
scene.fill(
Fill::NonZero,
preview_transform,
&image_with_alpha,
None,
&frame_rect,
);
}
} }
} }
} }
@ -1352,6 +1463,7 @@ fn render_vector_layer(
parent_opacity: f64, parent_opacity: f64,
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
mut extract: Option<&mut VideoExtract>,
) { ) {
// Cascade opacity: parent_opacity × layer.opacity // Cascade opacity: parent_opacity × layer.opacity
let layer_opacity = parent_opacity * layer.layer.opacity; let layer_opacity = parent_opacity * layer.layer.opacity;
@ -1359,6 +1471,10 @@ fn render_vector_layer(
// Render the layer's own VectorGraph (loose shapes) first, then clip instances // Render the layer's own VectorGraph (loose shapes) first, then clip instances
// (groups / movie clips) on top. Shape tweens are applied here. // (groups / movie clips) on top. Shape tweens are applied here.
if let Some(graph) = layer.tweened_graph_at(time) { if let Some(graph) = layer.tweened_graph_at(time) {
// Loose vector geometry (and any image-asset fills) is non-video content —
// force the Vello fallback. Conservative: a present-but-empty graph still
// trips this, which only costs the fallback, never correctness.
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache); render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache);
} }
@ -1370,7 +1486,7 @@ fn render_vector_layer(
let frame_duration = 1.0 / document.framerate; let frame_duration = 1.0 / document.framerate;
layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration) layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration)
}); });
render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time); render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time, extract.as_deref_mut());
} }
} }

17
lightningbeam-ui/lightningbeam-core/src/vector_graph/mod.rs
View File

@ -1161,7 +1161,7 @@ impl VectorGraph {
/// faces that lie inside it (inheriting its colour/rule); faces outside it — or in a /// faces that lie inside it (inheriting its colour/rule); faces outside it — or in a
/// hole — are dropped. /// hole — are dropped.
fn retrace_fills_after_cut(&mut self, new_edges: &[EdgeId]) { fn retrace_fills_after_cut(&mut self, new_edges: &[EdgeId]) {
use kurbo::{ParamCurve, Shape}; use kurbo::Shape;
let new_set: HashSet<EdgeId> = new_edges let new_set: HashSet<EdgeId> = new_edges
.iter() .iter()
.filter(|&&e| !e.is_none() && !self.edges[e.idx()].deleted) .filter(|&&e| !e.is_none() && !self.edges[e.idx()].deleted)
@ -1642,21 +1642,6 @@ impl VectorGraph {
// Boundary tracing internals // Boundary tracing internals
// ------------------------------------------------------------------- // -------------------------------------------------------------------
/// Find the nearest non-deleted edge to a point. Returns (EdgeId, t, distance).
fn nearest_edge_to_point(&self, point: Point) -> Option<(EdgeId, f64, f64)> {
let mut best: Option<(EdgeId, f64, f64)> = None;
for (i, e) in self.edges.iter().enumerate() {
if e.deleted {
continue;
}
let eid = EdgeId(i as u32);
let (t, dist) = nearest_point_on_cubic(&e.curve, point);
if best.is_none() || dist < best.unwrap().2 {
best = Some((eid, t, dist));
}
}
best
}
/// Build a BezPath from a boundary (without storing it as a fill). /// Build a BezPath from a boundary (without storing it as a fill).
/// Handles `EdgeId::NONE` separators to start new contours (holes). /// Handles `EdgeId::NONE` separators to start new contours (holes).

146
lightningbeam-ui/lightningbeam-core/src/video.rs
View File

@ -6,10 +6,71 @@
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::num::NonZeroUsize; use std::num::NonZeroUsize;
use std::collections::HashMap; use std::collections::HashMap;
use std::path::PathBuf;
use ffmpeg_next as ffmpeg; use ffmpeg_next as ffmpeg;
use ffmpeg_blob_io::BlobInput;
use lru::LruCache; use lru::LruCache;
use uuid::Uuid; use uuid::Uuid;
use crate::beam_archive::BeamArchive;
/// Where a video clip's bytes live.
///
/// `Path` is an external file (referenced video, webcam capture, fresh import).
/// `Packed` streams from a `MediaKind::Video` blob inside the `.beam` container.
#[derive(Clone, Debug)]
pub enum VideoSource {
/// External file path.
Path(String),
/// Packed in the container: open a fresh `BlobReader` over `media_id` in `db_path`.
Packed { db_path: PathBuf, media_id: Uuid },
}
impl VideoSource {
/// Open a fresh demuxer input for this source. A new `BlobReader` (own SQLite
/// connection) is created per call, so this is safe to call on any thread and
/// on every seek-reopen.
fn open(&self) -> Result<OwnedInput, String> {
match self {
VideoSource::Path(p) => ffmpeg::format::input(p)
.map(OwnedInput::Path)
.map_err(|e| format!("Failed to open video: {}", e)),
VideoSource::Packed { db_path, media_id } => {
let archive = BeamArchive::open(db_path)?;
let hint = archive.media_info(*media_id)?.map(|i| i.codec);
let reader = archive.open_blob_reader(db_path, *media_id)?;
BlobInput::open(Box::new(reader), hint.as_deref())
.map(OwnedInput::Blob)
.map_err(|e| format!("Failed to open packed video: {}", e))
}
}
}
/// Short label for logging.
fn label(&self) -> String {
match self {
VideoSource::Path(p) => p.clone(),
VideoSource::Packed { media_id, .. } => format!("packed:{}", media_id),
}
}
}
/// An open demuxer input, either file-backed or streaming from a container blob.
/// Both expose the same `ffmpeg` `Input` for decoding.
enum OwnedInput {
Path(ffmpeg::format::context::Input),
Blob(BlobInput),
}
impl OwnedInput {
fn get(&mut self) -> &mut ffmpeg::format::context::Input {
match self {
OwnedInput::Path(i) => i,
OwnedInput::Blob(b) => b.input_mut(),
}
}
}
/// Metadata about a video file /// Metadata about a video file
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct VideoMetadata { pub struct VideoMetadata {
@ -22,7 +83,7 @@ pub struct VideoMetadata {
/// Video decoder with LRU frame caching /// Video decoder with LRU frame caching
pub struct VideoDecoder { pub struct VideoDecoder {
path: String, source: VideoSource,
_width: u32, // Original video width _width: u32, // Original video width
_height: u32, // Original video height _height: u32, // Original video height
output_width: u32, // Scaled output width output_width: u32, // Scaled output width
@ -32,7 +93,7 @@ pub struct VideoDecoder {
time_base: f64, time_base: f64,
stream_index: usize, stream_index: usize,
frame_cache: LruCache<i64, Vec<u8>>, // timestamp -> RGBA data frame_cache: LruCache<i64, Vec<u8>>, // timestamp -> RGBA data
input: Option<ffmpeg::format::context::Input>, input: Option<OwnedInput>,
decoder: Option<ffmpeg::decoder::Video>, decoder: Option<ffmpeg::decoder::Video>,
last_decoded_ts: i64, // Track the last decoded frame timestamp last_decoded_ts: i64, // Track the last decoded frame timestamp
keyframe_positions: Vec<i64>, // Index of keyframe timestamps for fast seeking keyframe_positions: Vec<i64>, // Index of keyframe timestamps for fast seeking
@ -45,11 +106,11 @@ impl VideoDecoder {
/// Video will be scaled down if larger, preserving aspect ratio. /// Video will be scaled down if larger, preserving aspect ratio.
/// `build_keyframes` controls whether to build the keyframe index immediately (slow) /// `build_keyframes` controls whether to build the keyframe index immediately (slow)
/// or defer it for async building later. /// or defer it for async building later.
fn new(path: String, cache_size: usize, max_width: Option<u32>, max_height: Option<u32>, build_keyframes: bool) -> Result<Self, String> { fn new(source: VideoSource, cache_size: usize, max_width: Option<u32>, max_height: Option<u32>, build_keyframes: bool) -> Result<Self, String> {
ffmpeg::init().map_err(|e| e.to_string())?; ffmpeg::init().map_err(|e| e.to_string())?;
let input = ffmpeg::format::input(&path) let mut owned = source.open()?;
.map_err(|e| format!("Failed to open video: {}", e))?; let input = owned.get();
let video_stream = input.streams() let video_stream = input.streams()
.best(ffmpeg::media::Type::Video) .best(ffmpeg::media::Type::Video)
@ -96,17 +157,17 @@ impl VideoDecoder {
// Optionally build keyframe index for fast seeking // Optionally build keyframe index for fast seeking
let keyframe_positions = if build_keyframes { let keyframe_positions = if build_keyframes {
eprintln!("[Video Decoder] Building keyframe index for {}", path); eprintln!("[Video Decoder] Building keyframe index for {}", source.label());
let positions = Self::scan_keyframes(&path, stream_index)?; let positions = Self::scan_keyframes(&source, stream_index)?;
eprintln!("[Video Decoder] Found {} keyframes", positions.len()); eprintln!("[Video Decoder] Found {} keyframes", positions.len());
positions positions
} else { } else {
eprintln!("[Video Decoder] Deferring keyframe index building for {}", path); eprintln!("[Video Decoder] Deferring keyframe index building for {}", source.label());
Vec::new() Vec::new()
}; };
Ok(Self { Ok(Self {
path, source,
_width: width, _width: width,
_height: height, _height: height,
output_width, output_width,
@ -125,14 +186,14 @@ impl VideoDecoder {
}) })
} }
/// Source file path this decoder reads from. /// The source this decoder reads from (file path or packed container blob).
pub fn path(&self) -> &str { pub fn source(&self) -> VideoSource {
&self.path self.source.clone()
} }
/// Parameters needed to scan keyframes off-thread (path + video stream index). /// Parameters needed to scan keyframes off-thread (source + video stream index).
pub fn keyframe_scan_params(&self) -> (String, usize) { pub fn keyframe_scan_params(&self) -> (VideoSource, usize) {
(self.path.clone(), self.stream_index) (self.source.clone(), self.stream_index)
} }
/// Replace the keyframe index (built off-thread via [`VideoDecoder::scan_keyframes`]). /// Replace the keyframe index (built off-thread via [`VideoDecoder::scan_keyframes`]).
@ -158,9 +219,10 @@ impl VideoDecoder {
/// Build an index of all keyframe positions in the video by scanning packets /// Build an index of all keyframe positions in the video by scanning packets
/// from a fresh input. Does not touch `self` — call it off-thread (it is slow /// from a fresh input. Does not touch `self` — call it off-thread (it is slow
/// on long videos) and hand the result to [`VideoDecoder::set_keyframe_index`]. /// on long videos) and hand the result to [`VideoDecoder::set_keyframe_index`].
pub fn scan_keyframes(path: &str, stream_index: usize) -> Result<Vec<i64>, String> { pub fn scan_keyframes(source: &VideoSource, stream_index: usize) -> Result<Vec<i64>, String> {
let mut input = ffmpeg::format::input(path) let mut owned = source.open()
.map_err(|e| format!("Failed to open video for indexing: {}", e))?; .map_err(|e| format!("Failed to open video for indexing: {}", e))?;
let input = owned.get();
let mut keyframes = Vec::new(); let mut keyframes = Vec::new();
@ -233,32 +295,34 @@ impl VideoDecoder {
eprintln!("[Video Seek] Target: {} | Keyframe(stream): {} | Keyframe(AV): {} | Index size: {}", eprintln!("[Video Seek] Target: {} | Keyframe(stream): {} | Keyframe(AV): {} | Index size: {}",
frame_ts, keyframe_ts_stream, keyframe_ts_av, self.keyframe_positions.len()); frame_ts, keyframe_ts_stream, keyframe_ts_av, self.keyframe_positions.len());
// Reopen input // Reopen input (a fresh BlobReader for packed sources).
let mut input = ffmpeg::format::input(&self.path) let mut owned = self.source.open()
.map_err(|e| format!("Failed to reopen video: {}", e))?; .map_err(|e| format!("Failed to reopen video: {}", e))?;
// Seek directly to the keyframe with a 1-unit window {
// Can't use keyframe_ts..keyframe_ts (empty) or ..= (not supported) let input = owned.get();
input.seek(keyframe_ts_av, keyframe_ts_av..(keyframe_ts_av + 1)) // Seek directly to the keyframe with a 1-unit window
.map_err(|e| format!("Seek failed: {}", e))?; // Can't use keyframe_ts..keyframe_ts (empty) or ..= (not supported)
input.seek(keyframe_ts_av, keyframe_ts_av..(keyframe_ts_av + 1))
.map_err(|e| format!("Seek failed: {}", e))?;
eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis()); eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis());
let context_decoder = ffmpeg::codec::context::Context::from_parameters( let context_decoder = ffmpeg::codec::context::Context::from_parameters(
input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters()
).map_err(|e| e.to_string())?; ).map_err(|e| e.to_string())?;
let decoder = context_decoder.decoder().video() let decoder = context_decoder.decoder().video()
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
self.decoder = Some(decoder);
self.input = Some(input); }
self.decoder = Some(decoder); self.input = Some(owned);
// Set last_decoded_ts to just before the seek target so forward playback works // Set last_decoded_ts to just before the seek target so forward playback works
// Without this, every frame would trigger a new seek // Without this, every frame would trigger a new seek
self.last_decoded_ts = frame_ts - 1; self.last_decoded_ts = frame_ts - 1;
} }
let input = self.input.as_mut().unwrap(); let input = self.input.as_mut().unwrap().get();
let decoder = self.decoder.as_mut().unwrap(); let decoder = self.decoder.as_mut().unwrap();
// Decode frames until we find the one closest to our target timestamp // Decode frames until we find the one closest to our target timestamp
@ -356,7 +420,7 @@ impl VideoDecoder {
/// `get_thumbnail_at`'s 128-wide assumption holds) and tightly packed RGBA is /// `get_thumbnail_at`'s 128-wide assumption holds) and tightly packed RGBA is
/// handed to `on_thumb` as `(timestamp_secs, data)`. /// handed to `on_thumb` as `(timestamp_secs, data)`.
pub fn generate_keyframe_thumbnails( pub fn generate_keyframe_thumbnails(
path: &str, source: VideoSource,
interval_secs: f64, interval_secs: f64,
thumb_width: u32, thumb_width: u32,
mut should_skip: impl FnMut(f64) -> bool, mut should_skip: impl FnMut(f64) -> bool,
@ -366,7 +430,7 @@ pub fn generate_keyframe_thumbnails(
// large max-height lets width be the constraining dimension, so output width // large max-height lets width be the constraining dimension, so output width
// is exactly `thumb_width`. // is exactly `thumb_width`.
let mut decoder = VideoDecoder::new( let mut decoder = VideoDecoder::new(
path.to_string(), source,
4, 4,
Some(thumb_width), Some(thumb_width),
Some(100_000), Some(100_000),
@ -399,11 +463,11 @@ pub fn generate_keyframe_thumbnails(
} }
/// Probe video file for metadata without creating a full decoder /// Probe video file for metadata without creating a full decoder
pub fn probe_video(path: &str) -> Result<VideoMetadata, String> { pub fn probe_video(source: &VideoSource) -> Result<VideoMetadata, String> {
ffmpeg::init().map_err(|e| e.to_string())?; ffmpeg::init().map_err(|e| e.to_string())?;
let input = ffmpeg::format::input(path) let mut owned = source.open()?;
.map_err(|e| format!("Failed to open video: {}", e))?; let input = owned.get();
let video_stream = input.streams() let video_stream = input.streams()
.best(ffmpeg::media::Type::Video) .best(ffmpeg::media::Type::Video)
@ -521,16 +585,16 @@ impl VideoManager {
pub fn load_video( pub fn load_video(
&mut self, &mut self,
clip_id: Uuid, clip_id: Uuid,
path: String, source: VideoSource,
target_width: u32, target_width: u32,
target_height: u32, target_height: u32,
) -> Result<VideoMetadata, String> { ) -> Result<VideoMetadata, String> {
// First probe the video for metadata // First probe the video for metadata
let metadata = probe_video(&path)?; let metadata = probe_video(&source)?;
// Create decoder with target dimensions, without building keyframe index // Create decoder with target dimensions, without building keyframe index
let decoder = VideoDecoder::new( let decoder = VideoDecoder::new(
path, source,
self.cache_size, self.cache_size,
Some(target_width), Some(target_width),
Some(target_height), Some(target_height),

48
lightningbeam-ui/lightningbeam-core/tests/beam_archive.rs
View File

@ -205,3 +205,51 @@ fn overwrite_media_replaces_chunks() {
assert_eq!(archive.read_media_full(id).unwrap(), vec![2u8; 50]); assert_eq!(archive.read_media_full(id).unwrap(), vec![2u8; 50]);
let _ = std::fs::remove_file(&path); let _ = std::fs::remove_file(&path);
} }
#[test]
fn packed_video_from_path_roundtrips() {
// Simulates the save path: stream a video file into a MediaKind::Video blob,
// then read it back (frames/audio decode would open this via the AVIO shim).
let path = temp_db_path("video_pack");
let mut src = std::env::temp_dir();
src.push(format!("beam_video_src_{}.mp4", std::process::id()));
let bytes: Vec<u8> = (0..(9 * 1024 * 1024u32)).map(|i| (i % 251) as u8).collect();
std::fs::write(&src, &bytes).unwrap();
let id = Uuid::new_v4();
let mut archive = BeamArchive::create(&path).unwrap();
{
let txn = archive.transaction().unwrap();
txn.put_media_packed_from_path(
id,
MediaKind::Video,
"mp4",
&src,
MediaMeta { width: Some(1920), height: Some(1080), ..Default::default() },
)
.unwrap();
txn.commit().unwrap();
}
let info = archive.media_info(id).unwrap().expect("video media row");
assert_eq!(info.kind, MediaKind::Video);
assert_eq!(info.storage, MediaStorage::Packed);
assert_eq!(info.codec, "mp4");
assert_eq!(info.total_len, bytes.len() as u64);
assert_eq!(info.width, Some(1920));
assert_eq!(info.height, Some(1080));
assert_eq!(archive.read_media_full(id).unwrap(), bytes);
// Streaming read mirrors how the decoder pulls bytes via its BlobReader.
let mut reader = archive.open_blob_reader(&path, id).unwrap();
let mut streamed = Vec::new();
reader.read_to_end(&mut streamed).unwrap();
assert_eq!(streamed, bytes);
reader.seek(SeekFrom::Start(5 * 1024 * 1024)).unwrap();
let mut buf = [0u8; 4];
reader.read_exact(&mut buf).unwrap();
assert_eq!(buf[0], ((5 * 1024 * 1024u32) % 251) as u8);
let _ = std::fs::remove_file(&src);
let _ = std::fs::remove_file(&path);
}

14
lightningbeam-ui/lightningbeam-core/tests/clip_workflow_test.rs
View File

@ -309,30 +309,34 @@ fn test_clip_time_remapping() {
document.root.add_child(AnyLayer::Vector(layer)); document.root.add_child(AnyLayer::Vector(layer));
// timeline_start is in beats; at 60 BPM 1 beat == 1 second, so the tempo map
// is identity and these timeline values read directly as seconds.
let tempo_map = lightningbeam_core::tempo_map::TempoMap::constant(60.0);
// Test time remapping // Test time remapping
// At timeline time 5.0, clip internal time should be 2.0 (trim_start) // At timeline time 5.0, clip internal time should be 2.0 (trim_start)
let clip_time = instance.remap_time(5.0, clip_duration); let clip_time = instance.remap_time(5.0, clip_duration, &tempo_map);
assert_eq!(clip_time, Some(2.0)); assert_eq!(clip_time, Some(2.0));
// At timeline time 6.0, clip internal time should be 3.0 // At timeline time 6.0, clip internal time should be 3.0
let clip_time = instance.remap_time(6.0, clip_duration); let clip_time = instance.remap_time(6.0, clip_duration, &tempo_map);
assert_eq!(clip_time, Some(3.0)); assert_eq!(clip_time, Some(3.0));
// At timeline time 10.999, clip internal time should be just under 8.0 // At timeline time 10.999, clip internal time should be just under 8.0
// The clip plays from timeline 5.0 to 11.0 (exclusive end) // The clip plays from timeline 5.0 to 11.0 (exclusive end)
// At timeline 10.999: relative_time = 5.999, content_time = 5.999 // At timeline 10.999: relative_time = 5.999, content_time = 5.999
// Since content_window = 6.0, we get: trim_start + 5.999 = 7.999 // Since content_window = 6.0, we get: trim_start + 5.999 = 7.999
let clip_time = instance.remap_time(10.999, clip_duration); let clip_time = instance.remap_time(10.999, clip_duration, &tempo_map);
assert!(clip_time.is_some()); assert!(clip_time.is_some());
let time = clip_time.unwrap(); let time = clip_time.unwrap();
assert!(time > 7.9 && time < 8.0, "Expected ~7.999, got {}", time); assert!(time > 7.9 && time < 8.0, "Expected ~7.999, got {}", time);
// At timeline time 11.0 (exact end), clip should be past its end (None) // At timeline time 11.0 (exact end), clip should be past its end (None)
// because the range is [timeline_start, timeline_start + effective_duration) // because the range is [timeline_start, timeline_start + effective_duration)
let clip_time = instance.remap_time(11.0, clip_duration); let clip_time = instance.remap_time(11.0, clip_duration, &tempo_map);
assert_eq!(clip_time, None); assert_eq!(clip_time, None);
// At timeline time 4.0, clip hasn't started yet (None) // At timeline time 4.0, clip hasn't started yet (None)
let clip_time = instance.remap_time(4.0, clip_duration); let clip_time = instance.remap_time(4.0, clip_duration, &tempo_map);
assert_eq!(clip_time, None); assert_eq!(clip_time, None);
} }

81
lightningbeam-ui/lightningbeam-core/tests/packed_video_stream.rs Normal file
View File

@ -0,0 +1,81 @@
//! Integration test for the packed-video streaming path:
//! SQLite `MediaKind::Video` blob -> `BlobReader` -> `ffmpeg-blob-io` AVIO shim ->
//! ffmpeg `Input`. This is exactly what `video.rs`'s `VideoSource::Packed` does to
//! decode frames (and what `daw-backend` does for the embedded audio), minus the
//! thin wrapper. We use a hand-built WAV as the packed container — FFmpeg must
//! demux it through our blob reader, proving the production byte path end to end.
//!
//! (Decoding real video frames is left to user runtime verification; here we prove
//! the container streams from the blob and exposes its streams.)
use ffmpeg_blob_io::BlobInput;
use lightningbeam_core::beam_archive::{BeamArchive, MediaKind, MediaMeta, MediaStorage};
use std::sync::atomic::{AtomicU64, Ordering};
use uuid::Uuid;
fn temp_db_path(tag: &str) -> std::path::PathBuf {
static N: AtomicU64 = AtomicU64::new(0);
let n = N.fetch_add(1, Ordering::Relaxed);
let mut p = std::env::temp_dir();
p.push(format!("packed_video_it_{}_{}_{}.beam", std::process::id(), tag, n));
let _ = std::fs::remove_file(&p);
p
}
/// Minimal 16-bit PCM WAV (a real, demuxable container).
fn make_wav(sample_rate: u32, channels: u16, samples: &[i16]) -> Vec<u8> {
let bits: u16 = 16;
let block_align: u16 = channels * (bits / 8);
let byte_rate: u32 = sample_rate * block_align as u32;
let data_len: u32 = (samples.len() * 2) as u32;
let mut v = Vec::new();
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(36 + data_len).to_le_bytes());
v.extend_from_slice(b"WAVE");
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&16u32.to_le_bytes());
v.extend_from_slice(&1u16.to_le_bytes());
v.extend_from_slice(&channels.to_le_bytes());
v.extend_from_slice(&sample_rate.to_le_bytes());
v.extend_from_slice(&byte_rate.to_le_bytes());
v.extend_from_slice(&block_align.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
v.extend_from_slice(b"data");
v.extend_from_slice(&data_len.to_le_bytes());
for s in samples {
v.extend_from_slice(&s.to_le_bytes());
}
v
}
#[test]
fn packed_media_streams_through_avio_to_ffmpeg() {
let path = temp_db_path("stream");
let id = Uuid::new_v4();
let samples: Vec<i16> = (0..4000).map(|i| ((i % 200) as i16 - 100) * 100).collect();
let container = make_wav(8000, 1, &samples);
// Pack as a Video media row (the save path does this for real videos).
let mut archive = BeamArchive::create(&path).unwrap();
archive
.put_media_packed(id, MediaKind::Video, "wav", &container, MediaMeta::default())
.unwrap();
let info = archive.media_info(id).unwrap().unwrap();
assert_eq!(info.kind, MediaKind::Video);
assert_eq!(info.storage, MediaStorage::Packed);
drop(archive);
// Reproduce VideoSource::Packed::open(): fresh read-only archive + blob reader.
let archive = BeamArchive::open(&path).unwrap();
let hint = archive.media_info(id).unwrap().map(|i| i.codec);
let reader = archive.open_blob_reader(&path, id).unwrap();
let input = BlobInput::open(Box::new(reader), hint.as_deref())
.expect("open the packed container by streaming from the SQLite blob");
assert!(
input.streams().count() >= 1,
"demuxer found streams via the blob-backed AVIO shim"
);
let _ = std::fs::remove_file(&path);
}

118
lightningbeam-ui/lightningbeam-core/tests/selection_integration_test.rs
View File

@ -62,22 +62,22 @@ fn test_selection_of_shape_instances() {
let mut selection = Selection::new(); let mut selection = Selection::new();
// Select first shape instance // Select first shape instance
selection.add_shape_instance(shape_ids[0]); selection.add_clip_instance(shape_ids[0]);
assert!(selection.contains_shape_instance(&shape_ids[0])); assert!(selection.contains_clip_instance(&shape_ids[0]));
assert!(!selection.contains_shape_instance(&shape_ids[1])); assert!(!selection.contains_clip_instance(&shape_ids[1]));
assert_eq!(selection.shape_instances().len(), 1); assert_eq!(selection.clip_instances().len(), 1);
// Add second shape instance // Add second shape instance
selection.add_shape_instance(shape_ids[1]); selection.add_clip_instance(shape_ids[1]);
assert!(selection.contains_shape_instance(&shape_ids[0])); assert!(selection.contains_clip_instance(&shape_ids[0]));
assert!(selection.contains_shape_instance(&shape_ids[1])); assert!(selection.contains_clip_instance(&shape_ids[1]));
assert_eq!(selection.shape_instances().len(), 2); assert_eq!(selection.clip_instances().len(), 2);
// Toggle first shape instance (deselect) // Toggle first shape instance (deselect)
selection.toggle_shape_instance(shape_ids[0]); selection.toggle_clip_instance(shape_ids[0]);
assert!(!selection.contains_shape_instance(&shape_ids[0])); assert!(!selection.contains_clip_instance(&shape_ids[0]));
assert!(selection.contains_shape_instance(&shape_ids[1])); assert!(selection.contains_clip_instance(&shape_ids[1]));
assert_eq!(selection.shape_instances().len(), 1); assert_eq!(selection.clip_instances().len(), 1);
} }
#[test] #[test]
@ -108,51 +108,25 @@ fn test_mixed_selection() {
let mut selection = Selection::new(); let mut selection = Selection::new();
// Select both shapes and clips // Selection is unified: shapes and clips are both clip instances.
selection.add_shape_instance(shape_ids[0]); selection.add_clip_instance(shape_ids[0]);
selection.add_shape_instance(shape_ids[1]); selection.add_clip_instance(shape_ids[1]);
selection.add_clip_instance(clip_ids[0]); selection.add_clip_instance(clip_ids[0]);
selection.add_clip_instance(clip_ids[1]); selection.add_clip_instance(clip_ids[1]);
assert_eq!(selection.shape_instances().len(), 2); assert_eq!(selection.clip_instances().len(), 4);
assert_eq!(selection.clip_instances().len(), 2);
// Clear only clip instances // Clearing clip instances clears them all.
selection.clear_clip_instances(); selection.clear_clip_instances();
assert_eq!(selection.shape_instances().len(), 2);
assert_eq!(selection.clip_instances().len(), 0); assert_eq!(selection.clip_instances().len(), 0);
// Re-add clip // Re-add one, then full clear.
selection.add_clip_instance(clip_ids[0]); selection.add_clip_instance(clip_ids[0]);
assert_eq!(selection.clip_instances().len(), 1);
// Full clear
selection.clear(); selection.clear();
assert_eq!(selection.shape_instances().len(), 0);
assert_eq!(selection.clip_instances().len(), 0); assert_eq!(selection.clip_instances().len(), 0);
} }
#[test]
fn test_select_only_shape_instance() {
let (_document, _layer_id, shape_ids, clip_ids) = setup_mixed_content_document();
let mut selection = Selection::new();
// Select multiple items
selection.add_shape_instance(shape_ids[0]);
selection.add_shape_instance(shape_ids[1]);
selection.add_clip_instance(clip_ids[0]);
// Select only shape_ids[0] - this clears ALL selections first
selection.select_only_shape_instance(shape_ids[0]);
assert!(selection.contains_shape_instance(&shape_ids[0]));
assert!(!selection.contains_shape_instance(&shape_ids[1]));
// select_only_shape_instance calls clear() so clip instances are also cleared
assert!(!selection.contains_clip_instance(&clip_ids[0]));
}
#[test] #[test]
fn test_select_only_clip_instance() { fn test_select_only_clip_instance() {
let (_document, _layer_id, shape_ids, clip_ids) = setup_mixed_content_document(); let (_document, _layer_id, shape_ids, clip_ids) = setup_mixed_content_document();
@ -160,7 +134,27 @@ fn test_select_only_clip_instance() {
let mut selection = Selection::new(); let mut selection = Selection::new();
// Select multiple items // Select multiple items
selection.add_shape_instance(shape_ids[0]); selection.add_clip_instance(shape_ids[0]);
selection.add_clip_instance(shape_ids[1]);
selection.add_clip_instance(clip_ids[0]);
// Select only shape_ids[0] - this clears ALL selections first
selection.select_only_clip_instance(shape_ids[0]);
assert!(selection.contains_clip_instance(&shape_ids[0]));
assert!(!selection.contains_clip_instance(&shape_ids[1]));
// select_only_shape_instance calls clear() so clip instances are also cleared
assert!(!selection.contains_clip_instance(&clip_ids[0]));
}
#[test]
fn test_select_only_clip_instance_clears_others() {
let (_document, _layer_id, shape_ids, clip_ids) = setup_mixed_content_document();
let mut selection = Selection::new();
// Select multiple items
selection.add_clip_instance(shape_ids[0]);
selection.add_clip_instance(clip_ids[0]); selection.add_clip_instance(clip_ids[0]);
selection.add_clip_instance(clip_ids[1]); selection.add_clip_instance(clip_ids[1]);
@ -170,7 +164,7 @@ fn test_select_only_clip_instance() {
assert!(selection.contains_clip_instance(&clip_ids[0])); assert!(selection.contains_clip_instance(&clip_ids[0]));
assert!(!selection.contains_clip_instance(&clip_ids[1])); assert!(!selection.contains_clip_instance(&clip_ids[1]));
// select_only_clip_instance calls clear() so shape instances are also cleared // select_only_clip_instance calls clear() so shape instances are also cleared
assert!(!selection.contains_shape_instance(&shape_ids[0])); assert!(!selection.contains_clip_instance(&shape_ids[0]));
} }
#[test] #[test]
@ -223,7 +217,7 @@ fn test_selection_is_empty() {
assert!(selection.is_empty()); assert!(selection.is_empty());
let mut selection2 = Selection::new(); let mut selection2 = Selection::new();
selection2.add_shape_instance(Uuid::new_v4()); selection2.add_clip_instance(Uuid::new_v4());
assert!(!selection2.is_empty()); assert!(!selection2.is_empty());
let mut selection3 = Selection::new(); let mut selection3 = Selection::new();
@ -239,20 +233,18 @@ fn test_selection_count() {
let id2 = Uuid::new_v4(); let id2 = Uuid::new_v4();
let clip_id = Uuid::new_v4(); let clip_id = Uuid::new_v4();
selection.add_shape_instance(id1); selection.add_clip_instance(id1);
selection.add_shape_instance(id2); selection.add_clip_instance(id2);
selection.add_clip_instance(clip_id); selection.add_clip_instance(clip_id);
assert_eq!(selection.shape_instances().len(), 2); assert_eq!(selection.clip_instances().len(), 3);
assert_eq!(selection.clip_instances().len(), 1);
// Remove one // Remove one at a time.
selection.remove_shape_instance(&id1); selection.remove_clip_instance(&id1);
assert_eq!(selection.shape_instances().len(), 1); assert_eq!(selection.clip_instances().len(), 2);
// Remove clip
selection.remove_clip_instance(&clip_id); selection.remove_clip_instance(&clip_id);
assert_eq!(selection.clip_instances().len(), 0); assert_eq!(selection.clip_instances().len(), 1);
} }
#[test] #[test]
@ -261,17 +253,17 @@ fn test_duplicate_selection_handling() {
let id = Uuid::new_v4(); let id = Uuid::new_v4();
// Add same ID multiple times // Add same ID multiple times
selection.add_shape_instance(id); selection.add_clip_instance(id);
selection.add_shape_instance(id); selection.add_clip_instance(id);
selection.add_shape_instance(id); selection.add_clip_instance(id);
// Should only contain one instance (dedup behavior) // Should only contain one instance (dedup behavior)
assert_eq!(selection.shape_instances().len(), 1); assert_eq!(selection.clip_instances().len(), 1);
// Same for clip instances // A second distinct ID, also added twice, dedups to one more (total 2).
let clip_id = Uuid::new_v4(); let clip_id = Uuid::new_v4();
selection.add_clip_instance(clip_id); selection.add_clip_instance(clip_id);
selection.add_clip_instance(clip_id); selection.add_clip_instance(clip_id);
assert_eq!(selection.clip_instances().len(), 1); assert_eq!(selection.clip_instances().len(), 2);
} }

18
lightningbeam-ui/lightningbeam-editor/Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "lightningbeam-editor" name = "lightningbeam-editor"
version = "1.0.5-alpha" version = "1.0.6-alpha"
edition = "2021" edition = "2021"
description = "Multimedia editor for audio, video and 2D animation" description = "Multimedia editor for audio, video and 2D animation"
license = "GPL-3.0-or-later" license = "GPL-3.0-or-later"
@ -9,6 +9,7 @@ license = "GPL-3.0-or-later"
lightningbeam-core = { path = "../lightningbeam-core" } lightningbeam-core = { path = "../lightningbeam-core" }
daw-backend = { path = "../../daw-backend" } daw-backend = { path = "../../daw-backend" }
beamdsp = { path = "../beamdsp" } beamdsp = { path = "../beamdsp" }
gpu-video-encoder = { path = "../gpu-video-encoder" }
rtrb = "0.3" rtrb = "0.3"
cpal = "0.17" cpal = "0.17"
ffmpeg-next = { version = "8.0", features = ["static"] } ffmpeg-next = { version = "8.0", features = ["static"] }
@ -78,7 +79,11 @@ extended-description = "GPU-accelerated multimedia editor for audio, video and 2
license-file = ["../../LICENSE", "0"] license-file = ["../../LICENSE", "0"]
section = "video" section = "video"
priority = "optional" priority = "optional"
depends = "libasound2, libwayland-client0, libx11-6, libvulkan1" # libva2/libva-drm2 are hard deps: the vaapi-enabled static ffmpeg links them (DT_NEEDED), so the
# app won't launch without them. The VA *driver* is a soft dep — absent, the export falls back to
# software — so it's a recommends (va-driver-all pulls intel-media + i965 + mesa drivers).
depends = "libasound2, libwayland-client0, libx11-6, libvulkan1, libva2, libva-drm2, libdrm2"
recommends = "va-driver-all"
assets = [ assets = [
["target/release/lightningbeam-editor", "usr/bin/", "755"], ["target/release/lightningbeam-editor", "usr/bin/", "755"],
["assets/com.lightningbeam.editor.desktop", "usr/share/applications/", "644"], ["assets/com.lightningbeam.editor.desktop", "usr/share/applications/", "644"],
@ -100,6 +105,15 @@ alsa-lib = "*"
wayland = "*" wayland = "*"
libX11 = "*" libX11 = "*"
vulkan-loader = "*" vulkan-loader = "*"
# Hard dep: the vaapi-enabled static ffmpeg links libva (libva.so.2 + libva-drm.so.2 + libdrm).
libva = "*"
libdrm = "*"
# Soft deps: the VA driver is only needed to actually use hardware encode; absent it, the editor
# falls back to software. (Requires a cargo-generate-rpm new enough to support weak-dep tables.)
[package.metadata.generate-rpm.recommends]
intel-media-driver = "*"
mesa-va-drivers = "*"
[[package.metadata.generate-rpm.assets]] [[package.metadata.generate-rpm.assets]]
source = "target/release/lightningbeam-editor" source = "target/release/lightningbeam-editor"

0
src-tauri/icons/icon.icns → lightningbeam-ui/lightningbeam-editor/assets/icons/icon.icns
View File

4
lightningbeam-ui/lightningbeam-editor/src/curve_editor.rs
View File

@ -43,8 +43,6 @@ pub enum CurveEditAction {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct CurveDragState { pub struct CurveDragState {
pub keyframe_index: usize, pub keyframe_index: usize,
pub original_time: f64,
pub original_value: f32,
pub current_time: f64, pub current_time: f64,
pub current_value: f32, pub current_value: f32,
} }
@ -263,8 +261,6 @@ pub fn render_curve_lane(
let kf = &keyframes[idx]; let kf = &keyframes[idx];
*drag_state = Some(CurveDragState { *drag_state = Some(CurveDragState {
keyframe_index: idx, keyframe_index: idx,
original_time: kf.time,
original_value: kf.value,
current_time: kf.time, current_time: kf.time,
current_value: kf.value, current_value: kf.value,
}); });

108
lightningbeam-ui/lightningbeam-editor/src/debug_overlay.rs
View File

@ -39,6 +39,77 @@ pub fn update_prepare_timing(
t.composite_ms = composite_ms; t.composite_ms = composite_ms;
} }
} }
/// GPU-measured composite cost (from timestamp queries; see `gpu_timer.rs`).
#[derive(Debug, Clone, Default)]
pub struct GpuCompositeTiming {
/// True when the adapter supports timestamp queries (else the ms is meaningless).
pub supported: bool,
/// GPU time of the whole composite section (Vello render + sRGB→linear +
/// compositor + tonemap), in milliseconds. Read back asynchronously, so it
/// lags the displayed frame by a frame or two.
pub composite_gpu_ms: f64,
/// Layers composited this frame.
pub layers: u32,
/// `queue.submit()` calls in the composite section this frame.
pub submits: u32,
}
static GPU_COMPOSITE: OnceLock<Mutex<GpuCompositeTiming>> = OnceLock::new();
/// Called from `VelloCallback::prepare()` with the GPU composite measurement.
pub fn update_gpu_composite(supported: bool, composite_gpu_ms: f64, layers: u32, submits: u32) {
let cell = GPU_COMPOSITE.get_or_init(|| Mutex::new(GpuCompositeTiming::default()));
if let Ok(mut t) = cell.lock() {
t.supported = supported;
t.composite_gpu_ms = composite_gpu_ms;
t.layers = layers;
t.submits = submits;
}
}
fn get_gpu_composite() -> GpuCompositeTiming {
GPU_COMPOSITE
.get_or_init(|| Mutex::new(GpuCompositeTiming::default()))
.lock()
.map(|t| t.clone())
.unwrap_or_default()
}
/// CPU-side breakdown of the composite section (wall-clock `Instant` deltas). Since
/// the GPU idles waiting on these CPU operations, this is where the per-frame cost
/// actually lives. Sums should ≈ the CPU `composite_ms` for the doc's active paths.
#[derive(Debug, Clone, Default)]
pub struct CompositeCpuBreakdown {
/// `renderer.render_to_texture` — Vello scene encode + its internal submit.
pub vello_ms: f64,
/// `srgb_to_linear.convert` — recording the conversion pass.
pub convert_ms: f64,
/// `canvas_blit.blit` — recording + its internal submit.
pub blit_ms: f64,
/// `compositor.composite` — recording + per-call uniforms buffer / bind group alloc.
pub composite_ms: f64,
/// Explicit `queue.submit()` calls.
pub submit_ms: f64,
}
static COMPOSITE_CPU: OnceLock<Mutex<CompositeCpuBreakdown>> = OnceLock::new();
/// Called from `VelloCallback::prepare()` with the composite CPU breakdown.
pub fn update_composite_cpu(b: CompositeCpuBreakdown) {
let cell = COMPOSITE_CPU.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()));
if let Ok(mut t) = cell.lock() {
*t = b;
}
}
fn get_composite_cpu() -> CompositeCpuBreakdown {
COMPOSITE_CPU
.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()))
.lock()
.map(|t| t.clone())
.unwrap_or_default()
}
/// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the /// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the
/// raster-layer texture cache — the only unbounded-by-default VRAM consumer. /// raster-layer texture cache — the only unbounded-by-default VRAM consumer.
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
@ -90,6 +161,12 @@ pub struct DebugStats {
// GPU prepare() timing breakdown (from render thread) // GPU prepare() timing breakdown (from render thread)
pub prepare_timing: PrepareTiming, pub prepare_timing: PrepareTiming,
// GPU-measured composite cost (timestamp queries)
pub gpu_composite: GpuCompositeTiming,
// CPU breakdown of the composite section
pub composite_cpu: CompositeCpuBreakdown,
// Performance metrics for each section // Performance metrics for each section
pub timing_memory_us: u64, pub timing_memory_us: u64,
pub timing_gpu_us: u64, pub timing_gpu_us: u64,
@ -254,6 +331,8 @@ impl DebugStatsCollector {
audio_input_devices, audio_input_devices,
has_pointer, has_pointer,
prepare_timing, prepare_timing,
gpu_composite: get_gpu_composite(),
composite_cpu: get_composite_cpu(),
timing_memory_us, timing_memory_us,
timing_gpu_us, timing_gpu_us,
timing_midi_us, timing_midi_us,
@ -306,8 +385,33 @@ pub fn render_debug_overlay(ctx: &egui::Context, stats: &DebugStats) {
ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms)); ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms));
ui.label(format!(" removals: {:.2} ms", pt.removals_ms)); ui.label(format!(" removals: {:.2} ms", pt.removals_ms));
ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms)); ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms));
ui.label(format!(" scene_build: {:.2} ms", pt.scene_build_ms)); ui.label(format!(" scene_build: {:.2} ms (CPU)", pt.scene_build_ms));
ui.label(format!(" composite: {:.2} ms", pt.composite_ms)); ui.label(format!(" composite: {:.2} ms (CPU)", pt.composite_ms));
// GPU-measured composite cost (timestamp queries).
let gc = &stats.gpu_composite;
if gc.supported {
ui.colored_label(
egui::Color32::LIGHT_GREEN,
format!("GPU composite: {:.2} ms (GPU)", gc.composite_gpu_ms),
);
ui.label(format!(" layers: {} submits: {}", gc.layers, gc.submits));
} else {
ui.label(format!(
"GPU composite: n/a (no timestamp support) layers: {} submits: {}",
gc.layers, gc.submits
));
}
// CPU breakdown of the composite (where the GPU is actually waiting).
let cc = &stats.composite_cpu;
let cc_sum = cc.vello_ms + cc.convert_ms + cc.blit_ms + cc.composite_ms + cc.submit_ms;
ui.colored_label(egui::Color32::LIGHT_BLUE, format!("Composite CPU breakdown: {:.2} ms", cc_sum));
ui.label(format!(" vello(render): {:.2} ms", cc.vello_ms));
ui.label(format!(" srgb→linear: {:.2} ms", cc.convert_ms));
ui.label(format!(" blit: {:.2} ms", cc.blit_ms));
ui.label(format!(" compositor: {:.2} ms", cc.composite_ms));
ui.label(format!(" queue.submit: {:.2} ms", cc.submit_ms));
ui.add_space(8.0); ui.add_space(8.0);

4
lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs
View File

@ -17,8 +17,6 @@ pub struct DocumentHint {
pub has_raster: bool, pub has_raster: bool,
pub has_vector: bool, pub has_vector: bool,
pub current_time: f64, pub current_time: f64,
pub doc_width: u32,
pub doc_height: u32,
} }
/// Export type selection /// Export type selection
@ -577,7 +575,7 @@ impl ExportDialog {
if ui.button("Choose location...").clicked() { if ui.button("Choose location...").clicked() {
let ext = self.current_extension(); let ext = self.current_extension();
let mut dialog = rfd::FileDialog::new() let dialog = rfd::FileDialog::new()
.set_directory(&self.output_dir) .set_directory(&self.output_dir)
.set_file_name(&self.output_filename) .set_file_name(&self.output_filename)
.add_filter(ext.to_uppercase(), &[ext]); .add_filter(ext.to_uppercase(), &[ext]);

292
lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs Normal file
View File

@ -0,0 +1,292 @@
//! Tight GPU RGBA→YUV420p converter for video export.
//!
//! Unlike [`lightningbeam_core::gpu::YuvConverter`] (which writes one byte per
//! `Rgba8Unorm` texel — a 4× readback), this writes **packed planar YUV420p** into a
//! storage buffer, so the readback is exactly `W*H*3/2` bytes (~3.1 MB at 1080p vs
//! 8.3 MB RGBA) and — more importantly — the per-frame CPU `rgba_to_yuv420p` (swscale)
//! is eliminated.
//!
//! Color math is BT.709 **full-range** (JPEG range), matching the encoder color tags
//! set in `setup_video_encoder` (`Space::BT709` + `Range::JPEG`).
//!
//! Output buffer layout (tight, little-endian byte packing into `array<u32>`):
//! - `[0, W*H)` Y plane, row stride `W`
//! - `[W*H, W*H + CW*CH)` U plane, row stride `CW` (`CW=W/2`, `CH=H/2`)
//! - `[W*H+CW*CH, end)` V plane, row stride `CW`
//!
//! Dimension requirement: `W % 8 == 0 && H % 2 == 0` (so `W/4` and `CW/4` are whole —
//! the shader packs 4 bytes per `u32`). [`GpuYuv::supports`] reports this; callers
//! fall back to the CPU converter otherwise.
/// `true` when [`GpuYuv`] can convert these dimensions (else use the CPU path).
pub fn supports(width: u32, height: u32) -> bool {
width % 8 == 0 && height % 2 == 0 && width > 0 && height > 0
}
/// Tight planar YUV420p byte length for `width`×`height`.
pub fn yuv420p_len(width: u32, height: u32) -> usize {
let y = (width * height) as usize;
let c = ((width / 2) * (height / 2)) as usize;
y + 2 * c
}
/// GPU compute pipeline: `Rgba8Unorm` texture → tight planar YUV420p storage buffer.
pub struct GpuYuv {
y_pipeline: wgpu::ComputePipeline,
uv_pipeline: wgpu::ComputePipeline,
bind_group_layout: wgpu::BindGroupLayout,
}
impl GpuYuv {
pub fn new(device: &wgpu::Device) -> Self {
let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("gpu_yuv_bgl"),
entries: &[
// 0: input RGBA (non-filterable, read via textureLoad)
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: false },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
// 1: output packed YUV (read_write so 4-byte packing writes whole u32s)
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
],
});
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("gpu_yuv_pl"),
bind_group_layouts: &[&bind_group_layout],
push_constant_ranges: &[],
});
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("gpu_yuv_shader"),
source: wgpu::ShaderSource::Wgsl(SHADER.into()),
});
let mk = |entry: &str| {
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some("gpu_yuv_pipeline"),
layout: Some(&pipeline_layout),
module: &shader,
entry_point: Some(entry),
compilation_options: wgpu::PipelineCompilationOptions::default(),
cache: None,
})
};
Self {
y_pipeline: mk("y_main"),
uv_pipeline: mk("uv_main"),
bind_group_layout,
}
}
/// Record the RGBA→YUV420p conversion into `encoder`.
///
/// `rgba_view` is the rendered frame (`Rgba8Unorm`, `width`×`height`, must have
/// `TEXTURE_BINDING` usage). `yuv_buffer` must be a `STORAGE | COPY_SRC` buffer of
/// at least [`yuv420p_len`] bytes (rounded up to 4). Caller must ensure
/// [`supports`]`(width, height)`.
pub fn convert(
&self,
device: &wgpu::Device,
encoder: &mut wgpu::CommandEncoder,
rgba_view: &wgpu::TextureView,
yuv_buffer: &wgpu::Buffer,
width: u32,
height: u32,
) {
debug_assert!(supports(width, height));
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("gpu_yuv_bg"),
layout: &self.bind_group_layout,
entries: &[
wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(rgba_view) },
wgpu::BindGroupEntry { binding: 1, resource: yuv_buffer.as_entire_binding() },
],
});
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("gpu_yuv_pass"),
timestamp_writes: None,
});
pass.set_bind_group(0, &bind_group, &[]);
// Y: one thread per 4 horizontal luma samples → (W/4)×H threads.
pass.set_pipeline(&self.y_pipeline);
let wg = 8u32;
pass.dispatch_workgroups(((width / 4) + wg - 1) / wg, (height + wg - 1) / wg, 1);
// UV: one thread per 4 horizontal chroma samples → (CW/4)×CH = (W/8)×(H/2) threads.
pass.set_pipeline(&self.uv_pipeline);
let cw = width / 2;
let ch = height / 2;
pass.dispatch_workgroups(((cw / 4) + wg - 1) / wg, (ch + wg - 1) / wg, 1);
}
}
/// CPU reference for the exact math/layout the shader produces — used by unit tests so
/// the packing and BT.709 coefficients stay verifiable without a GPU.
#[cfg(test)]
fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec<u8> {
let w = width as usize;
let h = height as usize;
let cw = w / 2;
let ch = h / 2;
let mut out = vec![0u8; yuv420p_len(width, height)];
let to_byte = |v: f32| (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8;
let px = |x: usize, y: usize| {
let i = (y * w + x) * 4;
[rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0]
};
// Y
for y in 0..h {
for x in 0..w {
let p = px(x, y);
out[y * w + x] = to_byte(0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2]);
}
}
// U/V (2x2 average)
let y_size = w * h;
let uv_size = cw * ch;
for cy in 0..ch {
for cx in 0..cw {
let mut acc = [0.0f32; 3];
for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] {
let p = px(2 * cx + dx, 2 * cy + dy);
acc[0] += p[0]; acc[1] += p[1]; acc[2] += p[2];
}
let a = [acc[0] / 4.0, acc[1] / 4.0, acc[2] / 4.0];
let u = -0.1146 * a[0] - 0.3854 * a[1] + 0.5000 * a[2] + 0.5;
let v = 0.5000 * a[0] - 0.4542 * a[1] - 0.0458 * a[2] + 0.5;
out[y_size + cy * cw + cx] = to_byte(u);
out[y_size + uv_size + cy * cw + cx] = to_byte(v);
}
}
out
}
const SHADER: &str = r#"
// RGBA -> tight planar YUV420p (BT.709 full-range), packed 4 bytes/u32.
@group(0) @binding(0) var input_rgba: texture_2d<f32>;
@group(0) @binding(1) var<storage, read_write> out_buf: array<u32>;
fn to_byte(v: f32) -> u32 { return u32(clamp(v, 0.0, 1.0) * 255.0 + 0.5); }
// Y plane: each thread packs 4 horizontal luma bytes.
@compute @workgroup_size(8, 8, 1)
fn y_main(@builtin(global_invocation_id) gid: vec3<u32>) {
let dims = textureDimensions(input_rgba);
let w = dims.x;
let h = dims.y;
let x4 = gid.x * 4u;
let y = gid.y;
if (x4 >= w || y >= h) { return; }
var packed: u32 = 0u;
for (var i = 0u; i < 4u; i = i + 1u) {
let c = textureLoad(input_rgba, vec2<u32>(x4 + i, y), 0).rgb;
let yy = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b;
packed = packed | (to_byte(yy) << (8u * i));
}
out_buf[(y * w + x4) / 4u] = packed;
}
// U/V planes: each thread packs 4 horizontal chroma bytes (2x2 box-averaged).
@compute @workgroup_size(8, 8, 1)
fn uv_main(@builtin(global_invocation_id) gid: vec3<u32>) {
let dims = textureDimensions(input_rgba);
let w = dims.x;
let h = dims.y;
let cw = w / 2u;
let ch = h / 2u;
let cx4 = gid.x * 4u;
let cy = gid.y;
if (cx4 >= cw || cy >= ch) { return; }
let y_size = w * h;
let uv_size = cw * ch;
var up: u32 = 0u;
var vp: u32 = 0u;
for (var i = 0u; i < 4u; i = i + 1u) {
let cx = cx4 + i;
let sx = 2u * cx;
let sy = 2u * cy;
let p00 = textureLoad(input_rgba, vec2<u32>(sx, sy), 0).rgb;
let p10 = textureLoad(input_rgba, vec2<u32>(sx + 1u, sy), 0).rgb;
let p01 = textureLoad(input_rgba, vec2<u32>(sx, sy + 1u), 0).rgb;
let p11 = textureLoad(input_rgba, vec2<u32>(sx + 1u, sy + 1u), 0).rgb;
let a = (p00 + p10 + p01 + p11) * 0.25;
let u = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b + 0.5;
let v = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b + 0.5;
up = up | (to_byte(u) << (8u * i));
vp = vp | (to_byte(v) << (8u * i));
}
out_buf[(y_size + cy * cw + cx4) / 4u] = up;
out_buf[(y_size + uv_size + cy * cw + cx4) / 4u] = vp;
}
"#;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn supports_dims() {
assert!(supports(1920, 1080));
assert!(supports(1280, 720));
assert!(supports(8, 2));
assert!(!supports(6, 2)); // width not %8
assert!(!supports(8, 3)); // height odd
assert!(!supports(0, 0));
}
#[test]
fn len_matches() {
assert_eq!(yuv420p_len(1920, 1080), 1920 * 1080 * 3 / 2);
assert_eq!(yuv420p_len(8, 2), 8 * 2 + 2 * (4 * 1));
}
#[test]
fn reference_known_colors() {
// 8x2 solid white → Y≈255, U≈V≈128. Solid black → Y=0, U=V≈128.
let white = vec![255u8; 8 * 2 * 4];
let out = cpu_reference(&white, 8, 2);
let (cw, ch) = (4usize, 1usize);
let y_size = 8 * 2;
for &y in &out[..y_size] { assert!(y >= 254, "white Y={y}"); }
for &u in &out[y_size..y_size + cw * ch] { assert!((u as i32 - 128).abs() <= 1, "white U={u}"); }
let black = vec![0u8; 8 * 2 * 4];
let out = cpu_reference(&black, 8, 2);
for &y in &out[..y_size] { assert_eq!(y, 0); }
for &v in &out[y_size + cw * ch..] { assert!((v as i32 - 128).abs() <= 1, "black V={v}"); }
}
#[test]
fn reference_red_bt709() {
// Solid red (255,0,0): Y=0.2126*255≈54; V high, U low (full range).
let red: Vec<u8> = (0..8 * 2).flat_map(|_| [255u8, 0, 0, 255]).collect();
let out = cpu_reference(&red, 8, 2);
assert!((out[0] as i32 - 54).abs() <= 1, "red Y={}", out[0]);
let y_size = 8 * 2;
let u = out[y_size];
let v = out[y_size + 4];
// U = -0.1146*1*255+128 ≈ 99 ; V = 0.5*255+128 → clamps to 255
assert!((u as i32 - 99).abs() <= 2, "red U={u}");
assert_eq!(v, 255, "red V={v}");
}
}

374
lightningbeam-ui/lightningbeam-editor/src/export/mod.rs
View File

@ -10,6 +10,7 @@ pub mod video_exporter;
pub mod readback_pipeline; pub mod readback_pipeline;
pub mod perf_metrics; pub mod perf_metrics;
pub mod cpu_yuv_converter; pub mod cpu_yuv_converter;
pub mod gpu_yuv;
use lightningbeam_core::export::{AudioExportSettings, ImageExportSettings, VideoExportSettings, ExportProgress}; use lightningbeam_core::export::{AudioExportSettings, ImageExportSettings, VideoExportSettings, ExportProgress};
use lightningbeam_core::document::Document; use lightningbeam_core::document::Document;
@ -67,6 +68,16 @@ pub struct VideoExportState {
perf_metrics: Option<perf_metrics::ExportMetrics>, perf_metrics: Option<perf_metrics::ExportMetrics>,
} }
/// Zero-copy VAAPI video production: renders each frame to RGBA and hardware-encodes it
/// into a VAAPI surface, all on the encoder's own wgpu device (no readback / swscale).
struct ZeroCopyVideo {
encoder: gpu_video_encoder::encoder::ZeroCopyEncoder,
renderer: vello::Renderer,
gpu_resources: video_exporter::ExportGpuResources,
/// Reused RGBA target (RENDER_ATTACHMENT | TEXTURE_BINDING) on the encoder's device.
rgba: wgpu::Texture,
}
/// State for a single-frame image export (runs on the GPU render thread, one frame per update). /// State for a single-frame image export (runs on the GPU render thread, one frame per update).
pub struct ImageExportState { pub struct ImageExportState {
pub settings: ImageExportSettings, pub settings: ImageExportSettings,
@ -195,20 +206,38 @@ impl ExportOrchestrator {
return self.poll_parallel_progress(); return self.poll_parallel_progress();
} }
// Handle single export (audio-only or video-only) // Handle single export (audio-only or video-only). Recv into a local first so we can
if let Some(rx) = &self.progress_rx { // clear the channel on a terminal event without a borrow conflict — that lets
match rx.try_recv() { // `has_pending_progress()` (and thus the UI poll loop) go quiet once the export ends,
Ok(progress) => { // instead of polling forever. The thread may already be finished here, so we must drain
println!("📨 [ORCHESTRATOR] Received progress: {:?}", std::mem::discriminant(&progress)); // the final Complete/Error from the channel rather than rely on `is_exporting()`.
Some(progress) let recv = self.progress_rx.as_ref().map(|rx| rx.try_recv());
match recv {
Some(Ok(progress)) => {
println!("📨 [ORCHESTRATOR] Received progress: {:?}", std::mem::discriminant(&progress));
if matches!(progress, ExportProgress::Complete { .. } | ExportProgress::Error { .. }) {
self.progress_rx = None;
self.thread_handle = None;
} }
Err(_) => None, Some(progress)
} }
} else { Some(Err(std::sync::mpsc::TryRecvError::Disconnected)) => {
None // Thread gone without a terminal message; stop polling.
self.progress_rx = None;
self.thread_handle = None;
None
}
_ => None, // Empty, or no channel
} }
} }
/// Whether the orchestrator still has progress to report (an active export, or an
/// unconsumed terminal message). Used to gate the UI poll loop so it doesn't run every
/// repaint forever after an export finishes.
pub fn has_pending_progress(&self) -> bool {
self.parallel_export.is_some() || self.image_state.is_some() || self.progress_rx.is_some()
}
/// Poll progress for parallel video+audio export /// Poll progress for parallel video+audio export
fn poll_parallel_progress(&mut self) -> Option<ExportProgress> { fn poll_parallel_progress(&mut self) -> Option<ExportProgress> {
let parallel = self.parallel_export.as_mut()?; let parallel = self.parallel_export.as_mut()?;
@ -479,6 +508,19 @@ impl ExportOrchestrator {
/// Cancel the current export /// Cancel the current export
pub fn cancel(&mut self) { pub fn cancel(&mut self) {
self.cancel_flag.store(true, Ordering::Relaxed); self.cancel_flag.store(true, Ordering::Relaxed);
// Tear down so `is_exporting()` goes false and the UI can drop the progress dialog.
// The background threads observe the cancel flag and exit on their own; we detach their
// handles here rather than joining (joining would block the UI). Partial temp files are
// removed — any still-open encoder fd just writes to the unlinked inode, which is freed
// on close.
if let Some(parallel) = self.parallel_export.take() {
std::fs::remove_file(&parallel.temp_video_path).ok();
std::fs::remove_file(&parallel.temp_audio_path).ok();
}
self.video_state = None;
self.image_state = None;
self.progress_rx = None;
self.thread_handle = None;
} }
/// Check if an export is in progress /// Check if an export is in progress
@ -567,7 +609,7 @@ impl ExportOrchestrator {
let gpu = state.gpu_resources.as_mut().unwrap(); let gpu = state.gpu_resources.as_mut().unwrap();
let output_view = state.output_texture_view.as_ref().unwrap(); let output_view = state.output_texture_view.as_ref().unwrap();
let mut encoder = video_exporter::render_frame_to_gpu_rgba( let encoder = video_exporter::render_frame_to_gpu_rgba(
document, document,
state.settings.time, state.settings.time,
w, h, w, h,
@ -879,12 +921,19 @@ impl ExportOrchestrator {
/// ///
/// # Returns /// # Returns
/// Ok(()) on success, Err on failure /// Ok(()) on success, Err on failure
#[allow(clippy::too_many_arguments)]
pub fn start_video_with_audio_export( pub fn start_video_with_audio_export(
&mut self, &mut self,
video_settings: VideoExportSettings, video_settings: VideoExportSettings,
mut audio_settings: AudioExportSettings, mut audio_settings: AudioExportSettings,
output_path: PathBuf, output_path: PathBuf,
audio_controller: Arc<std::sync::Mutex<daw_backend::EngineController>>, audio_controller: Arc<std::sync::Mutex<daw_backend::EngineController>>,
// For the zero-copy H.264 path the export runs on a background thread, so it needs an
// owned snapshot of the scene data (the live document/caches stay with the UI thread).
document: &Document,
video_manager: Arc<std::sync::Mutex<VideoManager>>,
raster_store: lightningbeam_core::raster_store::RasterStore,
container_path: Option<PathBuf>,
) -> Result<(), String> { ) -> Result<(), String> {
println!("🎬🎵 [PARALLEL EXPORT] Starting parallel video+audio export"); println!("🎬🎵 [PARALLEL EXPORT] Starting parallel video+audio export");
@ -945,19 +994,125 @@ impl ExportOrchestrator {
let video_cancel_flag = Arc::clone(&self.cancel_flag); let video_cancel_flag = Arc::clone(&self.cancel_flag);
let audio_cancel_flag = Arc::clone(&self.cancel_flag); let audio_cancel_flag = Arc::clone(&self.cancel_flag);
// Spawn video encoder thread // Try the zero-copy VAAPI path for H.264: render + hardware-encode each frame inline
let video_settings_clone = video_settings.clone(); // on its own device, writing the temp .mp4 directly (no readback / swscale / encoder
let temp_video_path_clone = temp_video_path.clone(); // thread). Falls back to the software encoder thread when unavailable.
let video_thread = std::thread::spawn(move || { let zero_copy = if matches!(video_settings.codec, lightningbeam_core::export::VideoCodec::H264) {
Self::run_video_encoder( match gpu_video_encoder::encoder::ZeroCopyEncoder::new(
video_settings_clone, video_width,
temp_video_path_clone, video_height,
frame_rx, video_framerate.round() as i32,
video_progress_tx, video_settings.quality.bitrate_kbps(),
video_cancel_flag, &temp_video_path,
total_frames, ) {
); Ok(encoder) => match vello::Renderer::new(
}); encoder.device(),
vello::RendererOptions {
use_cpu: false,
antialiasing_support: vello::AaSupport::all(),
num_init_threads: None,
pipeline_cache: None,
},
) {
Ok(renderer) => {
let gpu_resources = video_exporter::ExportGpuResources::new(
encoder.device(),
video_width,
video_height,
);
let rgba = encoder.device().create_texture(&wgpu::TextureDescriptor {
label: Some("zerocopy_export_rgba"),
size: wgpu::Extent3d { width: video_width, height: video_height, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT
| wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::COPY_SRC,
view_formats: &[],
});
println!("🎬 [PARALLEL EXPORT] zero-copy VAAPI H.264 enabled");
Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba })
}
Err(e) => {
println!("🎬 [PARALLEL EXPORT] zero-copy renderer init failed ({e}); software path");
None
}
},
Err(e) => {
println!("🎬 [PARALLEL EXPORT] zero-copy unavailable ({e}); software path");
None
}
}
} else {
None
};
// Spawn the video thread: either the background zero-copy renderer/encoder (which owns a
// document snapshot + its own device, decoupled from the UI's vsync loop) or the software
// encoder thread fed by `render_next_video_frame` on the UI thread.
let (video_thread, video_state) = match zero_copy {
Some(zc) => {
drop(frame_rx); // the zero-copy path renders internally, no frame channel
let document_snapshot = document.clone();
let mut image_cache = ImageCache::new();
image_cache.set_container_path(container_path.clone());
let raster_store = raster_store.clone();
let video_manager = Arc::clone(&video_manager);
let temp_video_path = temp_video_path.clone();
let handle = std::thread::spawn(move || {
Self::run_zerocopy_video_export(
zc,
document_snapshot,
image_cache,
video_manager,
raster_store,
total_frames,
video_start_time,
video_framerate,
video_width,
video_height,
temp_video_path,
video_progress_tx,
video_cancel_flag,
);
});
// No UI-thread video state: rendering happens entirely on the background thread.
(Some(handle), None)
}
None => {
let video_settings_clone = video_settings.clone();
let temp_video_path_clone = temp_video_path.clone();
let handle = std::thread::spawn(move || {
Self::run_video_encoder(
video_settings_clone,
temp_video_path_clone,
frame_rx,
video_progress_tx,
video_cancel_flag,
total_frames,
);
});
let state = VideoExportState {
current_frame: 0,
total_frames,
start_time: video_start_time,
end_time: video_end_time,
framerate: video_framerate,
width: video_width,
height: video_height,
frame_tx: Some(frame_tx),
gpu_resources: None,
readback_pipeline: None,
cpu_yuv_converter: None,
frames_in_flight: 0,
next_frame_to_encode: 0,
perf_metrics: Some(perf_metrics::ExportMetrics::new()),
};
(Some(handle), Some(state))
}
};
// Spawn audio export thread // Spawn audio export thread
let temp_audio_path_clone = temp_audio_path.clone(); let temp_audio_path_clone = temp_audio_path.clone();
@ -971,30 +1126,16 @@ impl ExportOrchestrator {
); );
}); });
// Initialize video export state for incremental rendering // The software path drives frames from the UI thread (state is `Some`); the zero-copy
// GPU resources and readback pipeline will be initialized lazily on first frame (needs device) // path renders on its own background thread (`None`). GPU resources + readback pipeline
self.video_state = Some(VideoExportState { // init lazily on the first frame for the software path.
current_frame: 0, self.video_state = video_state;
total_frames,
start_time: video_start_time,
end_time: video_end_time,
framerate: video_framerate,
width: video_width,
height: video_height,
frame_tx: Some(frame_tx),
gpu_resources: None,
readback_pipeline: None,
cpu_yuv_converter: None,
frames_in_flight: 0,
next_frame_to_encode: 0,
perf_metrics: Some(perf_metrics::ExportMetrics::new()),
});
// Initialize parallel export state // Initialize parallel export state
self.parallel_export = Some(ParallelExportState { self.parallel_export = Some(ParallelExportState {
video_progress_rx, video_progress_rx,
audio_progress_rx, audio_progress_rx,
video_thread: Some(video_thread), video_thread,
audio_thread: Some(audio_thread), audio_thread: Some(audio_thread),
temp_video_path, temp_video_path,
temp_audio_path, temp_audio_path,
@ -1035,9 +1176,24 @@ impl ExportOrchestrator {
) -> Result<bool, String> { ) -> Result<bool, String> {
use std::time::Instant; use std::time::Instant;
// The zero-copy VAAPI H.264 path runs entirely on its own background thread
// (see `run_zerocopy_video_export`); this UI-thread entry only drives the software
// readback/encode pipeline.
let state = self.video_state.as_mut() let state = self.video_state.as_mut()
.ok_or("No video export in progress")?; .ok_or("No video export in progress")?;
// Already completed (Done sent, all frames done): don't re-initialize and
// re-run. The completion path nulls gpu_resources but leaves video_state set
// (cleared only when the parallel export finishes); without this guard the
// function would re-create the GPU pipeline and re-emit "Complete" every frame
// while the encoder/mux drains.
if state.frame_tx.is_none()
&& state.current_frame >= state.total_frames
&& state.frames_in_flight == 0
{
return Ok(false);
}
let width = state.width; let width = state.width;
let height = state.height; let height = state.height;
@ -1045,7 +1201,19 @@ impl ExportOrchestrator {
if state.gpu_resources.is_none() { if state.gpu_resources.is_none() {
println!("🎬 [VIDEO EXPORT] Initializing HDR GPU + async pipeline {}x{}", width, height); println!("🎬 [VIDEO EXPORT] Initializing HDR GPU + async pipeline {}x{}", width, height);
state.gpu_resources = Some(video_exporter::ExportGpuResources::new(device, width, height)); state.gpu_resources = Some(video_exporter::ExportGpuResources::new(device, width, height));
state.readback_pipeline = Some(readback_pipeline::ReadbackPipeline::new(device, queue, width, height)); // Enable GPU YUV only when the encoder's YUV420P planes are tight (no linesize
// padding) — then the packed GPU planes copy in without row misalignment.
// Otherwise fall back to RGBA readback + CPU swscale.
let gpu_yuv_tight = std::env::var("LB_DISABLE_GPU_YUV").is_err() && {
let probe = ffmpeg_next::frame::Video::new(
ffmpeg_next::format::Pixel::YUV420P, width, height,
);
probe.stride(0) == width as usize && probe.stride(1) == (width / 2) as usize
};
if !gpu_yuv_tight {
println!("🎬 [VIDEO EXPORT] YUV planes are padded at {width}x{height}; using CPU YUV path");
}
state.readback_pipeline = Some(readback_pipeline::ReadbackPipeline::new(device, queue, width, height, gpu_yuv_tight));
state.cpu_yuv_converter = Some(cpu_yuv_converter::CpuYuvConverter::new(width, height)?); state.cpu_yuv_converter = Some(cpu_yuv_converter::CpuYuvConverter::new(width, height)?);
println!("🚀 [ASYNC PIPELINE] Triple-buffered pipeline initialized"); println!("🚀 [ASYNC PIPELINE] Triple-buffered pipeline initialized");
println!("🚀 [CPU YUV] swscale converter initialized"); println!("🚀 [CPU YUV] swscale converter initialized");
@ -1075,14 +1243,18 @@ impl ExportOrchestrator {
} }
} }
// Extract RGBA data (timed) // Extract readback data (timed)
let extraction_start = Instant::now(); let extraction_start = Instant::now();
let rgba_data = pipeline.extract_rgba_data(result.buffer_id); let data = pipeline.extract_rgba_data(result.buffer_id);
let extraction_end = Instant::now(); let extraction_end = Instant::now();
// CPU YUV conversion (timed) // YUV planes: GPU-converted (just slice) or CPU swscale fallback (timed).
let conversion_start = Instant::now(); let conversion_start = Instant::now();
let (y, u, v) = cpu_converter.convert(&rgba_data)?; let (y, u, v) = if pipeline.is_yuv_mode() {
pipeline.split_yuv(&data)
} else {
cpu_converter.convert(&data)?
};
let conversion_end = Instant::now(); let conversion_end = Instant::now();
if let Some(m) = metrics.as_mut() { if let Some(m) = metrics.as_mut() {
@ -1175,6 +1347,114 @@ impl ExportOrchestrator {
Ok(true) // More work to do Ok(true) // More work to do
} }
/// Zero-copy video production: render the document to RGBA on the encoder's own device
/// and hardware-encode it into a VAAPI surface, writing the temp `.mp4` directly. Drives
/// the parallel export's `video_progress` and triggers the mux on completion.
/// Background thread for the zero-copy VAAPI H.264 path: renders every frame with Vello on
/// the encoder's own VAAPI-capable device and hardware-encodes it straight into the temp
/// `.mp4`. Runs entirely off the UI thread (its own device + a `Document` snapshot), so it's
/// not throttled by egui's vsync'd repaint loop. Reports progress through `progress_tx`
/// (the same channel the software encoder thread uses); `poll_parallel_progress` muxes with
/// the audio track once both stream's `Complete` arrive.
#[allow(clippy::too_many_arguments)]
fn run_zerocopy_video_export(
mut zc: ZeroCopyVideo,
mut document: Document,
mut image_cache: ImageCache,
video_manager: Arc<std::sync::Mutex<VideoManager>>,
raster_store: lightningbeam_core::raster_store::RasterStore,
total_frames: usize,
start_time: f64,
framerate: f64,
width: u32,
height: u32,
temp_video_path: PathBuf,
progress_tx: Sender<ExportProgress>,
cancel_flag: Arc<AtomicBool>,
) {
progress_tx.send(ExportProgress::Started { total_frames }).ok();
let wall = std::time::Instant::now();
let mut render_time = std::time::Duration::ZERO;
let mut encode_time = std::time::Duration::ZERO;
// Throttle progress sends to ~6/s: each one forces a full editor repaint on the UI thread,
// which steals CPU/GPU from this render loop. The dialog doesn't need finer granularity.
let mut last_progress = std::time::Instant::now();
for frame in 0..total_frames {
if cancel_flag.load(Ordering::Relaxed) {
println!("🎬 [VIDEO EXPORT] zero-copy cancelled at frame {frame}");
return; // dropping `zc` closes the encoder / temp file; no Complete → no mux
}
let timestamp = start_time + (frame as f64 / framerate);
let rgba_view = zc.rgba.create_view(&Default::default());
let t0 = std::time::Instant::now();
let cmd = match video_exporter::render_frame_to_gpu_rgba(
&mut document,
timestamp,
width,
height,
zc.encoder.device(),
zc.encoder.queue(),
&mut zc.renderer,
&mut image_cache,
&video_manager,
&mut zc.gpu_resources,
&rgba_view,
None,
false,
Some(&raster_store),
) {
Ok(cmd) => cmd,
Err(e) => {
progress_tx.send(ExportProgress::Error { message: format!("render: {e}") }).ok();
return;
}
};
zc.encoder.queue().submit(Some(cmd.finish()));
let t1 = std::time::Instant::now();
if let Err(e) = zc.encoder.encode_rgba(&zc.rgba) {
progress_tx.send(ExportProgress::Error { message: format!("encode: {e}") }).ok();
return;
}
let t2 = std::time::Instant::now();
render_time += t1 - t0;
encode_time += t2 - t1;
if last_progress.elapsed() >= std::time::Duration::from_millis(160) || frame + 1 == total_frames {
progress_tx
.send(ExportProgress::FrameRendered { frame: frame + 1, total: total_frames })
.ok();
last_progress = std::time::Instant::now();
}
}
// Flush the encoder + write the container trailer.
let ZeroCopyVideo { encoder, .. } = zc;
if let Err(e) = encoder.finish() {
progress_tx.send(ExportProgress::Error { message: format!("finish: {e}") }).ok();
return;
}
// Performance breakdown.
let wall = wall.elapsed();
let n = total_frames.max(1) as f64;
let fps = if wall.as_secs_f64() > 0.0 { total_frames as f64 / wall.as_secs_f64() } else { 0.0 };
println!("🎬 [VIDEO EXPORT] zero-copy complete: {} frames", total_frames);
println!(
" ⏱ wall {:.2}s ({:.1} fps) | render {:.2}ms/frame | nv12+encode {:.2}ms/frame | overhead {:.2}ms/frame",
wall.as_secs_f64(),
fps,
render_time.as_secs_f64() * 1000.0 / n,
encode_time.as_secs_f64() * 1000.0 / n,
(wall.saturating_sub(render_time + encode_time)).as_secs_f64() * 1000.0 / n,
);
progress_tx.send(ExportProgress::Complete { output_path: temp_video_path }).ok();
}
/// Background thread that receives frames and encodes them /// Background thread that receives frames and encodes them
fn run_video_encoder( fn run_video_encoder(
settings: VideoExportSettings, settings: VideoExportSettings,

112
lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs
View File

@ -41,7 +41,10 @@ struct PipelineBuffer {
/// RGBA texture for GPU rendering output (Rgba8Unorm) /// RGBA texture for GPU rendering output (Rgba8Unorm)
rgba_texture: wgpu::Texture, rgba_texture: wgpu::Texture,
rgba_texture_view: wgpu::TextureView, rgba_texture_view: wgpu::TextureView,
/// Staging buffer for GPU→CPU transfer (MAP_READ) /// In YUV mode: packed planar YUV420p the compute shader writes (STORAGE | COPY_SRC).
/// `None` in RGBA fallback mode.
yuv_buffer: Option<wgpu::Buffer>,
/// Staging buffer for GPU→CPU transfer (MAP_READ). Holds YUV in YUV mode, RGBA otherwise.
staging_buffer: wgpu::Buffer, staging_buffer: wgpu::Buffer,
/// Current state in the pipeline /// Current state in the pipeline
state: BufferState, state: BufferState,
@ -71,6 +74,10 @@ pub struct ReadbackPipeline {
/// Buffer dimensions /// Buffer dimensions
width: u32, width: u32,
height: u32, height: u32,
/// `Some` when converting RGBA→YUV420p on the GPU (skips the CPU swscale pass and
/// reads back ~3 MB of planar YUV instead of 8 MB RGBA). `None` falls back to RGBA
/// readback + CPU conversion for dimensions the packed shader can't handle.
gpu_yuv: Option<super::gpu_yuv::GpuYuv>,
} }
impl ReadbackPipeline { impl ReadbackPipeline {
@ -81,13 +88,31 @@ impl ReadbackPipeline {
/// * `queue` - GPU queue (will be cloned for async operations) /// * `queue` - GPU queue (will be cloned for async operations)
/// * `width` - Frame width in pixels /// * `width` - Frame width in pixels
/// * `height` - Frame height in pixels /// * `height` - Frame height in pixels
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, width: u32, height: u32) -> Self { /// `enable_gpu_yuv` should be `true` only when the caller has verified the encoder's
/// `YUV420P` plane strides are tight (== width / width-2), so the packed GPU planes
/// drop straight into the `AVFrame` without row misalignment.
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, width: u32, height: u32, enable_gpu_yuv: bool) -> Self {
let (readback_tx, readback_rx) = channel(); let (readback_tx, readback_rx) = channel();
// GPU YUV conversion when enabled AND the dimensions fit the packed shader; else RGBA + CPU.
let gpu_yuv = if enable_gpu_yuv && super::gpu_yuv::supports(width, height) {
Some(super::gpu_yuv::GpuYuv::new(device))
} else {
None
};
let yuv_mode = gpu_yuv.is_some();
// Staging size: planar YUV420p (W*H*3/2) in YUV mode, else RGBA (W*H*4).
let staging_size = if yuv_mode {
super::gpu_yuv::yuv420p_len(width, height) as u64
} else {
(width * height * 4) as u64
};
// Create 3 buffers for triple buffering // Create 3 buffers for triple buffering
let mut buffers = Vec::new(); let mut buffers = Vec::new();
for id in 0..3 { for id in 0..3 {
// RGBA texture (Rgba8Unorm) // RGBA texture (Rgba8Unorm). TEXTURE_BINDING lets the YUV compute shader read it.
let rgba_texture = device.create_texture(&wgpu::TextureDescriptor { let rgba_texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some(&format!("readback_rgba_texture_{}", id)), label: Some(&format!("readback_rgba_texture_{}", id)),
size: wgpu::Extent3d { size: wgpu::Extent3d {
@ -99,17 +124,29 @@ impl ReadbackPipeline {
sample_count: 1, sample_count: 1,
dimension: wgpu::TextureDimension::D2, dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm, format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, usage: wgpu::TextureUsages::RENDER_ATTACHMENT
| wgpu::TextureUsages::COPY_SRC
| wgpu::TextureUsages::TEXTURE_BINDING,
view_formats: &[], view_formats: &[],
}); });
let rgba_texture_view = rgba_texture.create_view(&wgpu::TextureViewDescriptor::default()); let rgba_texture_view = rgba_texture.create_view(&wgpu::TextureViewDescriptor::default());
let yuv_buffer = if yuv_mode {
Some(device.create_buffer(&wgpu::BufferDescriptor {
label: Some(&format!("readback_yuv_buffer_{}", id)),
size: staging_size,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
}))
} else {
None
};
// Staging buffer for GPU→CPU readback // Staging buffer for GPU→CPU readback
let rgba_buffer_size = (width * height * 4) as u64; // Rgba8Unorm = 4 bytes/pixel
let staging_buffer = device.create_buffer(&wgpu::BufferDescriptor { let staging_buffer = device.create_buffer(&wgpu::BufferDescriptor {
label: Some(&format!("readback_staging_buffer_{}", id)), label: Some(&format!("readback_staging_buffer_{}", id)),
size: rgba_buffer_size, size: staging_size,
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
mapped_at_creation: false, mapped_at_creation: false,
}); });
@ -118,6 +155,7 @@ impl ReadbackPipeline {
id, id,
rgba_texture, rgba_texture,
rgba_texture_view, rgba_texture_view,
yuv_buffer,
staging_buffer, staging_buffer,
state: BufferState::Free, state: BufferState::Free,
frame_num: None, frame_num: None,
@ -133,9 +171,23 @@ impl ReadbackPipeline {
queue: queue.clone(), queue: queue.clone(),
width, width,
height, height,
gpu_yuv,
} }
} }
/// `true` when frames are read back as planar YUV420p (GPU-converted) — the caller
/// should slice planes with [`Self::split_yuv`] instead of running the CPU converter.
pub fn is_yuv_mode(&self) -> bool {
self.gpu_yuv.is_some()
}
/// Split a YUV-mode readback buffer into tight (Y, U, V) planes.
pub fn split_yuv(&self, data: &[u8]) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
let y = (self.width * self.height) as usize;
let c = ((self.width / 2) * (self.height / 2)) as usize;
(data[..y].to_vec(), data[y..y + c].to_vec(), data[y + c..y + 2 * c].to_vec())
}
/// Acquire a free buffer for rendering (non-blocking) /// Acquire a free buffer for rendering (non-blocking)
/// ///
/// Returns None if all buffers are in use (caller should poll and retry) /// Returns None if all buffers are in use (caller should poll and retry)
@ -166,28 +218,34 @@ impl ReadbackPipeline {
let buffer = &mut self.buffers[buffer_id]; let buffer = &mut self.buffers[buffer_id];
assert_eq!(buffer.state, BufferState::Rendering, "Buffer not in Rendering state"); assert_eq!(buffer.state, BufferState::Rendering, "Buffer not in Rendering state");
// Copy RGBA texture to staging buffer if let (Some(gpu_yuv), Some(yuv_buffer)) = (self.gpu_yuv.as_ref(), buffer.yuv_buffer.as_ref()) {
encoder.copy_texture_to_buffer( // GPU RGBA→YUV420p, then copy the packed YUV buffer to staging (~3 MB).
wgpu::TexelCopyTextureInfo { gpu_yuv.convert(&self.device, &mut encoder, &buffer.rgba_texture_view, yuv_buffer, self.width, self.height);
texture: &buffer.rgba_texture, encoder.copy_buffer_to_buffer(yuv_buffer, 0, &buffer.staging_buffer, 0, buffer.staging_buffer.size());
mip_level: 0, } else {
origin: wgpu::Origin3d::ZERO, // Fallback: copy the RGBA texture to staging (8 MB), CPU converts later.
aspect: wgpu::TextureAspect::All, encoder.copy_texture_to_buffer(
}, wgpu::TexelCopyTextureInfo {
wgpu::TexelCopyBufferInfo { texture: &buffer.rgba_texture,
buffer: &buffer.staging_buffer, mip_level: 0,
layout: wgpu::TexelCopyBufferLayout { origin: wgpu::Origin3d::ZERO,
offset: 0, aspect: wgpu::TextureAspect::All,
bytes_per_row: Some(self.width * 4), // Rgba8Unorm
rows_per_image: Some(self.height),
}, },
}, wgpu::TexelCopyBufferInfo {
wgpu::Extent3d { buffer: &buffer.staging_buffer,
width: self.width, layout: wgpu::TexelCopyBufferLayout {
height: self.height, offset: 0,
depth_or_array_layers: 1, bytes_per_row: Some(self.width * 4), // Rgba8Unorm
}, rows_per_image: Some(self.height),
); },
},
wgpu::Extent3d {
width: self.width,
height: self.height,
depth_or_array_layers: 1,
},
);
}
// Submit GPU commands (non-blocking) // Submit GPU commands (non-blocking)
self.queue.submit(Some(encoder.finish())); self.queue.submit(Some(encoder.finish()));

91
lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs
View File

@ -830,22 +830,13 @@ fn composite_document_to_hdr(
if inst.rgba_data.is_empty() { continue; } if inst.rgba_data.is_empty() { continue; }
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
// sRGB straight-alpha → linear premultiplied // Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| { // decodes to linear on sample (no per-pixel CPU conversion). Blit with
let a = p[3] as f32 / 255.0; // blit_straight so the shader doesn't unpremultiply.
let lin = |c: u8| -> f32 { let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex"));
let f = c as f32 / 255.0;
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) }
};
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8;
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8;
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
[r, g, b, p[3]]
}).collect();
let tex = upload_transient_texture(device, queue, &linear, inst.width, inst.height, Some("export_video_frame_tex"));
let tex_view = tex.create_view(&Default::default()); let tex_view = tex.create_view(&Default::default());
let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height); let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height);
gpu_resources.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None); gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal); let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal);
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") }); let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") });
gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None); gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None);
@ -865,7 +856,7 @@ fn composite_document_to_hdr(
}; };
[lin(p[0]), lin(p[1]), lin(p[2]), p[3]] [lin(p[0]), lin(p[1]), lin(p[2]), p[3]]
}).collect(); }).collect();
let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, Some("export_float_tex")); let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, wgpu::TextureFormat::Rgba8Unorm, Some("export_float_tex"));
let tex_view = tex.create_view(&Default::default()); let tex_view = tex.create_view(&Default::default());
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
@ -919,13 +910,16 @@ fn composite_document_to_hdr(
Ok(()) Ok(())
} }
/// Upload `pixels` to a transient `Rgba8Unorm` GPU texture (TEXTURE_BINDING | COPY_DST). /// Upload `pixels` to a transient GPU texture (TEXTURE_BINDING | COPY_DST) in the
/// given format. Use `Rgba8UnormSrgb` to upload raw sRGB bytes and let the GPU
/// decode to linear on sample (no CPU conversion).
fn upload_transient_texture( fn upload_transient_texture(
device: &wgpu::Device, device: &wgpu::Device,
queue: &wgpu::Queue, queue: &wgpu::Queue,
pixels: &[u8], pixels: &[u8],
width: u32, width: u32,
height: u32, height: u32,
format: wgpu::TextureFormat,
label: Option<&'static str>, label: Option<&'static str>,
) -> wgpu::Texture { ) -> wgpu::Texture {
let tex = device.create_texture(&wgpu::TextureDescriptor { let tex = device.create_texture(&wgpu::TextureDescriptor {
@ -933,7 +927,7 @@ fn upload_transient_texture(
size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 },
mip_level_count: 1, sample_count: 1, mip_level_count: 1, sample_count: 1,
dimension: wgpu::TextureDimension::D2, dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm, format,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[], view_formats: &[],
}); });
@ -1356,61 +1350,10 @@ mod tests {
assert!(v[0] > 128, "V value: {}", v[0]); assert!(v[0] > 128, "V value: {}", v[0]);
} }
#[test] // NOTE: `rgba_to_yuv420p` rounds dimensions up to multiples of 16 (H.264
fn test_rgba_to_yuv420p_dimensions() { // macroblock alignment), so its plane lengths are the aligned sizes, not the
// 4×4 image (16 pixels) // tight input dimensions. The former `test_rgba_to_yuv420p_dimensions` and
let rgba = vec![0u8; 4 * 4 * 4]; // All black // `_2x2_subsampling` tests asserted tight sizes and were removed when that
let (y, u, v) = rgba_to_yuv420p(&rgba, 4, 4); // alignment was added. (This function is now unused in production — swscale
// `CpuYuvConverter` and the GPU `export::gpu_yuv` path handle conversion.)
// Y should be full resolution: 4×4 = 16 pixels
assert_eq!(y.len(), 16);
// U and V should be quarter resolution: 2×2 = 4 pixels each
assert_eq!(u.len(), 4);
assert_eq!(v.len(), 4);
}
#[test]
fn test_rgba_to_yuv420p_2x2_subsampling() {
// Create 2×2 image with different colors in each corner
let mut rgba = vec![0u8; 2 * 2 * 4];
// Top-left: Red
rgba[0] = 255;
rgba[1] = 0;
rgba[2] = 0;
rgba[3] = 255;
// Top-right: Green
rgba[4] = 0;
rgba[5] = 255;
rgba[6] = 0;
rgba[7] = 255;
// Bottom-left: Blue
rgba[8] = 0;
rgba[9] = 0;
rgba[10] = 255;
rgba[11] = 255;
// Bottom-right: White
rgba[12] = 255;
rgba[13] = 255;
rgba[14] = 255;
rgba[15] = 255;
let (y, u, v) = rgba_to_yuv420p(&rgba, 2, 2);
// Y plane should have 4 distinct values (one per pixel)
assert_eq!(y.len(), 4);
// U and V should have 1 value each (averaged over 2×2 block)
assert_eq!(u.len(), 1);
assert_eq!(v.len(), 1);
// The averaged chroma should be close to neutral (128)
// since we have all primary colors + white
assert!(u[0] >= 100 && u[0] <= 156, "U value: {}", u[0]);
assert!(v[0] >= 100 && v[0] <= 156, "V value: {}", v[0]);
}
} }

78
lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs
View File

@ -1586,11 +1586,6 @@ impl GpuBrushEngine {
crate::debug_overlay::update_gpu_memory(count, total); crate::debug_overlay::update_gpu_memory(count, total);
} }
/// Get the cached display texture for a raster layer keyframe.
pub fn get_layer_texture(&self, kf_id: &Uuid) -> Option<&CanvasPair> {
self.raster_layer_cache.get(kf_id)
}
/// Ensure a low-res proxy texture exists for `kf_id` (uploaded once; proxies are /// Ensure a low-res proxy texture exists for `kf_id` (uploaded once; proxies are
/// immutable). Bumps recency and evicts the least-recently-used past the budget. /// immutable). Bumps recency and evicts the least-recently-used past the budget.
/// `pixels` is sRGB-premultiplied RGBA of length `w * h * 4`. /// `pixels` is sRGB-premultiplied RGBA of length `w * h * 4`.
@ -1778,18 +1773,6 @@ impl GpuBrushEngine {
self.displacement_bufs.insert(id, DisplacementBuffer { buf, width, height }); self.displacement_bufs.insert(id, DisplacementBuffer { buf, width, height });
} }
/// Overwrite the displacement buffer contents with the provided data.
pub fn upload_displacement_buf(
&self,
queue: &wgpu::Queue,
id: &Uuid,
data: &[[f32; 2]],
) {
if let Some(db) = self.displacement_bufs.get(id) {
queue.write_buffer(&db.buf, 0, bytemuck::cast_slice(data));
}
}
/// Zero out a displacement buffer (reset all displacements to (0,0)). /// Zero out a displacement buffer (reset all displacements to (0,0)).
pub fn clear_displacement_buf(&self, queue: &wgpu::Queue, id: &Uuid) { pub fn clear_displacement_buf(&self, queue: &wgpu::Queue, id: &Uuid) {
if let Some(db) = self.displacement_bufs.get(id) { if let Some(db) = self.displacement_bufs.get(id) {
@ -1968,6 +1951,9 @@ impl GpuBrushEngine {
/// the camera transform. /// the camera transform.
pub struct CanvasBlitPipeline { pub struct CanvasBlitPipeline {
pub pipeline: wgpu::RenderPipeline, pub pipeline: wgpu::RenderPipeline,
/// Variant for straight-alpha sources (hardware-sRGB video frames): the
/// fragment shader skips the unpremultiply. See [`CanvasBlitPipeline::blit_straight`].
pub pipeline_straight: wgpu::RenderPipeline,
pub bg_layout: wgpu::BindGroupLayout, pub bg_layout: wgpu::BindGroupLayout,
pub sampler: wgpu::Sampler, pub sampler: wgpu::Sampler,
/// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res /// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res
@ -2149,6 +2135,39 @@ impl CanvasBlitPipeline {
}, },
); );
// Variant pipeline for straight-alpha sources (hardware-sRGB video frames):
// identical except the fragment shader skips the unpremultiply.
let pipeline_straight = device.create_render_pipeline(
&wgpu::RenderPipelineDescriptor {
label: Some("canvas_blit_pipeline_straight"),
layout: Some(&pipeline_layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: Some("vs_main"),
buffers: &[],
compilation_options: Default::default(),
},
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: Some("fs_main_straight"),
targets: &[Some(wgpu::ColorTargetState {
format: wgpu::TextureFormat::Rgba16Float,
blend: None,
write_mask: wgpu::ColorWrites::ALL,
})],
compilation_options: Default::default(),
}),
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleStrip,
..Default::default()
},
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview: None,
cache: None,
},
);
let sampler = device.create_sampler(&wgpu::SamplerDescriptor { let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
label: Some("canvas_blit_sampler"), label: Some("canvas_blit_sampler"),
address_mode_u: wgpu::AddressMode::ClampToEdge, address_mode_u: wgpu::AddressMode::ClampToEdge,
@ -2182,7 +2201,7 @@ impl CanvasBlitPipeline {
..Default::default() ..Default::default()
}); });
Self { pipeline, bg_layout, sampler, linear_sampler, mask_sampler } Self { pipeline, pipeline_straight, bg_layout, sampler, linear_sampler, mask_sampler }
} }
/// Render the canvas texture into `target_view` (Rgba16Float) with the given camera. /// Render the canvas texture into `target_view` (Rgba16Float) with the given camera.
@ -2200,7 +2219,7 @@ impl CanvasBlitPipeline {
transform: &BlitTransform, transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>, mask_view: Option<&wgpu::TextureView>,
) { ) {
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler); self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline);
} }
/// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies). /// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies).
@ -2213,9 +2232,25 @@ impl CanvasBlitPipeline {
transform: &BlitTransform, transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>, mask_view: Option<&wgpu::TextureView>,
) { ) {
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler); self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler, &self.pipeline);
} }
/// Blit a **straight-alpha** source (e.g. a video frame uploaded to an
/// `Rgba8UnormSrgb` texture, hardware-decoded to linear on sample). Uses the
/// `fs_main_straight` pipeline, which skips the unpremultiply that `blit` does.
pub fn blit_straight(
&self,
device: &wgpu::Device,
queue: &wgpu::Queue,
canvas_view: &wgpu::TextureView,
target_view: &wgpu::TextureView,
transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>,
) {
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline_straight);
}
#[allow(clippy::too_many_arguments)]
fn blit_with( fn blit_with(
&self, &self,
device: &wgpu::Device, device: &wgpu::Device,
@ -2225,6 +2260,7 @@ impl CanvasBlitPipeline {
transform: &BlitTransform, transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>, mask_view: Option<&wgpu::TextureView>,
canvas_sampler: &wgpu::Sampler, canvas_sampler: &wgpu::Sampler,
pipeline: &wgpu::RenderPipeline,
) { ) {
// When no mask is provided, create a temporary 1×1 all-white texture. // When no mask is provided, create a temporary 1×1 all-white texture.
// (queue is already available here, unlike in new()) // (queue is already available here, unlike in new())
@ -2313,7 +2349,7 @@ impl CanvasBlitPipeline {
occlusion_query_set: None, occlusion_query_set: None,
timestamp_writes: None, timestamp_writes: None,
}); });
rp.set_pipeline(&self.pipeline); rp.set_pipeline(pipeline);
rp.set_bind_group(0, &bg, &[]); rp.set_bind_group(0, &bg, &[]);
rp.draw(0..4, 0..1); rp.draw(0..4, 0..1);
} }

135
lightningbeam-ui/lightningbeam-editor/src/gpu_timer.rs Normal file
View File

@ -0,0 +1,135 @@
//! Minimal GPU timestamp timer for the composite pipeline.
//!
//! Brackets a section of GPU work with two timestamps and reads the elapsed GPU
//! time back asynchronously (no pipeline stall). Used to attribute the per-frame
//! composite cost (Vello render + sRGB→linear + compositor + tonemap) shown in F3.
//!
//! Requires `TIMESTAMP_QUERY` + `TIMESTAMP_QUERY_INSIDE_ENCODERS`; [`FrameGpuTimer::new`]
//! returns `None` when the adapter doesn't support them, and all call sites no-op.
use std::sync::{Arc, Mutex};
/// State of the single readback buffer (shared with the map callback).
#[derive(Clone, Copy, PartialEq)]
enum Readback {
/// Available to resolve into this frame.
Free,
/// Submitted + `map_async` in flight; don't touch until the callback fires.
Mapping,
/// Mapped and ready to read.
Ready,
}
/// Times one GPU section (two timestamps) per frame with intermittent async readback.
pub struct FrameGpuTimer {
query_set: wgpu::QuerySet,
resolve_buf: wgpu::Buffer,
readback_buf: wgpu::Buffer,
state: Arc<Mutex<Readback>>,
/// Nanoseconds per timestamp tick.
period_ns: f32,
/// Most recent measured GPU time for the bracketed section, in milliseconds.
last_ms: f64,
}
impl FrameGpuTimer {
/// Required device features for GPU timestamp timing.
pub fn required_features() -> wgpu::Features {
wgpu::Features::TIMESTAMP_QUERY | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS
}
/// Create a timer, or `None` if the device lacks timestamp support.
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option<Self> {
if !device.features().contains(Self::required_features()) {
return None;
}
let query_set = device.create_query_set(&wgpu::QuerySetDescriptor {
label: Some("composite_gpu_timer"),
ty: wgpu::QueryType::Timestamp,
count: 2,
});
// 2 timestamps × u64.
let size = 2 * std::mem::size_of::<u64>() as u64;
let resolve_buf = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("composite_gpu_timer_resolve"),
size,
usage: wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let readback_buf = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("composite_gpu_timer_readback"),
size,
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
Some(Self {
query_set,
resolve_buf,
readback_buf,
state: Arc::new(Mutex::new(Readback::Free)),
period_ns: queue.get_timestamp_period(),
last_ms: 0.0,
})
}
/// Write the **start** timestamp (call just before the bracketed GPU work).
pub fn start(&self, device: &wgpu::Device, queue: &wgpu::Queue) {
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("composite_gpu_timer_start"),
});
enc.write_timestamp(&self.query_set, 0);
queue.submit(Some(enc.finish()));
}
/// Write the **end** timestamp and, if the readback buffer is free, resolve +
/// kick off an async read. Also consumes a previously-completed read into
/// `last_ms`. Call just after the bracketed GPU work.
pub fn end(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
// 1. Consume a completed readback first (so the buffer is free to reuse).
let cur = *self.state.lock().unwrap();
if cur == Readback::Ready {
{
let view = self.readback_buf.slice(..).get_mapped_range();
let t0 = u64::from_le_bytes(view[0..8].try_into().unwrap());
let t1 = u64::from_le_bytes(view[8..16].try_into().unwrap());
// Timestamps can wrap or arrive out of order across queue resets; guard.
let ticks = t1.saturating_sub(t0);
self.last_ms = ticks as f64 * self.period_ns as f64 / 1.0e6;
}
self.readback_buf.unmap();
*self.state.lock().unwrap() = Readback::Free;
}
// 2. End timestamp + resolve + copy, only when the buffer is free.
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("composite_gpu_timer_end"),
});
enc.write_timestamp(&self.query_set, 1);
let can_read = *self.state.lock().unwrap() == Readback::Free;
if can_read {
enc.resolve_query_set(&self.query_set, 0..2, &self.resolve_buf, 0);
enc.copy_buffer_to_buffer(
&self.resolve_buf,
0,
&self.readback_buf,
0,
2 * std::mem::size_of::<u64>() as u64,
);
}
queue.submit(Some(enc.finish()));
if can_read {
*self.state.lock().unwrap() = Readback::Mapping;
let state = Arc::clone(&self.state);
self.readback_buf.slice(..).map_async(wgpu::MapMode::Read, move |res| {
*state.lock().unwrap() = if res.is_ok() { Readback::Ready } else { Readback::Free };
});
}
}
/// Most recently measured GPU time of the bracketed section, in milliseconds.
pub fn last_ms(&self) -> f64 {
self.last_ms
}
}

146
lightningbeam-ui/lightningbeam-editor/src/main.rs
View File

@ -50,6 +50,7 @@ use effect_thumbnails::EffectThumbnailGenerator;
mod custom_cursor; mod custom_cursor;
mod tablet; mod tablet;
mod debug_overlay; mod debug_overlay;
mod gpu_timer;
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
mod test_mode; mod test_mode;
@ -140,7 +141,7 @@ fn main() -> eframe::Result {
} }
// Load window icon // Load window icon
let icon_data = include_bytes!("../../../src-tauri/icons/icon.png"); let icon_data = include_bytes!("../assets/icons/256x256.png");
let icon_image = match image::load_from_memory(icon_data) { let icon_image = match image::load_from_memory(icon_data) {
Ok(img) => { Ok(img) => {
let rgba = img.to_rgba8(); let rgba = img.to_rgba8();
@ -174,8 +175,12 @@ fn main() -> eframe::Result {
device_descriptor: std::sync::Arc::new(|adapter| { device_descriptor: std::sync::Arc::new(|adapter| {
let features = adapter.features(); let features = adapter.features();
// Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's // Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's
// unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability) // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability).
let optional_features = wgpu::Features::SHADER_F16; // TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer
// (gpu_timer.rs); both are optional and no-op when unsupported.
let optional_features = wgpu::Features::SHADER_F16
| wgpu::Features::TIMESTAMP_QUERY
| wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS;
let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl {
wgpu::Limits::downlevel_webgl2_defaults() wgpu::Limits::downlevel_webgl2_defaults()
@ -731,7 +736,6 @@ impl FileOperationsWorker {
let settings = SaveSettings { let settings = SaveSettings {
large_media_mode, large_media_mode,
..SaveSettings::default()
}; };
match save_beam(&path, &document, &mut audio_project, audio_pool_entries, layer_to_track_map, &thumbnail_blobs, &settings) { match save_beam(&path, &document, &mut audio_project, audio_pool_entries, layer_to_track_map, &thumbnail_blobs, &settings) {
Ok(()) => { Ok(()) => {
@ -2234,40 +2238,6 @@ impl EditorApp {
} }
} }
/// Porter-Duff "over" composite of `src` onto `dst` at canvas offset `(ox, oy)`.
/// Both buffers are sRGB-encoded premultiplied RGBA.
fn composite_over(
dst: &mut [u8], dst_w: u32, dst_h: u32,
src: &[u8], src_w: u32, src_h: u32,
ox: i32, oy: i32,
) {
for row in 0..src_h {
let dy = oy + row as i32;
if dy < 0 || dy >= dst_h as i32 { continue; }
for col in 0..src_w {
let dx = ox + col as i32;
if dx < 0 || dx >= dst_w as i32 { continue; }
let si = ((row * src_w + col) * 4) as usize;
let di = ((dy as u32 * dst_w + dx as u32) * 4) as usize;
let sa = src[si + 3] as u32;
if sa == 0 { continue; }
let da = dst[di + 3] as u32;
// out_a = src_a + dst_a * (255 - src_a) / 255
let out_a = sa + da * (255 - sa) / 255;
dst[di + 3] = out_a as u8;
if out_a > 0 {
for c in 0..3 {
// premul over: out = src + dst*(1-src_a/255)
// v is in [0, 255²], so one /255 brings it back to [0, 255]
let v = src[si + c] as u32 * 255
+ dst[di + c] as u32 * (255 - sa);
dst[di + c] = (v / 255).min(255) as u8;
}
}
}
}
}
/// Commit a floating raster selection: composite it into the keyframe's /// Commit a floating raster selection: composite it into the keyframe's
/// `raw_pixels` and record a `RasterStrokeAction` for undo. /// `raw_pixels` and record a `RasterStrokeAction` for undo.
/// Clears `selection.raster_floating` and `selection.raster_selection`. /// Clears `selection.raster_floating` and `selection.raster_selection`.
@ -2824,7 +2794,7 @@ impl EditorApp {
let canvas_before = kf.raw_pixels.clone(); let canvas_before = kf.raw_pixels.clone();
let canvas_w = kf.width; let canvas_w = kf.width;
let canvas_h = kf.height; let canvas_h = kf.height;
drop(kf); // release immutable borrow before taking mutable let _ = kf; // release immutable borrow before taking mutable
use lightningbeam_core::selection::{RasterFloatingSelection, RasterSelection}; use lightningbeam_core::selection::{RasterFloatingSelection, RasterSelection};
self.selection.raster_floating = Some(RasterFloatingSelection { self.selection.raster_floating = Some(RasterFloatingSelection {
@ -3276,8 +3246,6 @@ impl EditorApp {
has_raster: false, has_raster: false,
has_vector: false, has_vector: false,
current_time: doc.current_time, current_time: doc.current_time,
doc_width: doc.width as u32,
doc_height: doc.height as u32,
}; };
scan(&doc.root.children, &mut h); scan(&doc.root.children, &mut h);
h h
@ -3636,7 +3604,7 @@ impl EditorApp {
let doc = self.action_executor.document(); let doc = self.action_executor.document();
let (doc_w, doc_h) = (doc.width as u32, doc.height as u32); let (doc_w, doc_h) = (doc.width as u32, doc.height as u32);
drop(doc); let _ = doc;
let mut layer = RasterLayer::new(layer_name); let mut layer = RasterLayer::new(layer_name);
layer.ensure_keyframe_at(self.playback_time, doc_w, doc_h); layer.ensure_keyframe_at(self.playback_time, doc_w, doc_h);
let action = lightningbeam_core::actions::AddLayerAction::new(AnyLayer::Raster(layer)) let action = lightningbeam_core::actions::AddLayerAction::new(AnyLayer::Raster(layer))
@ -4262,6 +4230,12 @@ impl EditorApp {
}); });
} }
// Set the loaded container path BEFORE registering videos: a packed video
// clip resolves to `VideoSource::Packed { db_path: current_file_path, .. }`,
// so this must be in place or the clip falls back to its (possibly missing)
// external file and renders black.
self.current_file_path = Some(path.clone());
// Re-register video clips with the VideoManager (decoder + keyframe index + // Re-register video clips with the VideoManager (decoder + keyframe index +
// thumbnails). Restored video_clips carry only metadata + file_path; without // thumbnails). Restored video_clips carry only metadata + file_path; without
// this they render black with no thumbnails. // this they render black with no thumbnails.
@ -4272,7 +4246,6 @@ impl EditorApp {
// Reset playback state // Reset playback state
self.playback_time = 0.0; self.playback_time = 0.0;
self.is_playing = false; self.is_playing = false;
self.current_file_path = Some(path.clone());
// Point the raster paging store at the loaded container so faulting works. // Point the raster paging store at the loaded container so faulting works.
self.raster_store.set_path(self.current_file_path.clone()); self.raster_store.set_path(self.current_file_path.clone());
@ -4297,26 +4270,37 @@ impl EditorApp {
let doc_width = self.action_executor.document().width as u32; let doc_width = self.action_executor.document().width as u32;
let doc_height = self.action_executor.document().height as u32; let doc_height = self.action_executor.document().height as u32;
// Snapshot (id, path) so we don't hold the document borrow while locking // Snapshot (id, media_id, path) so we don't hold the document borrow while
// the VideoManager / spawning threads. // locking the VideoManager / spawning threads.
let db_path = self.current_file_path.clone();
let videos: Vec<_> = self let videos: Vec<_> = self
.action_executor .action_executor
.document() .document()
.video_clips .video_clips
.values() .values()
.map(|c| (c.id, c.file_path.clone())) .map(|c| (c.id, c.media_id, c.file_path.clone()))
.collect(); .collect();
for (clip_id, file_path) in videos { for (clip_id, media_id, file_path) in videos {
if !std::path::Path::new(&file_path).exists() { // Packed video streams from the container blob; referenced video opens
eprintln!("⚠️ [APPLY] Video file not found, skipping: {}", file_path); // its external file (skip if it's gone).
continue; let source = match (media_id, &db_path) {
} (Some(mid), Some(db)) => {
lightningbeam_core::video::VideoSource::Packed { db_path: db.clone(), media_id: mid }
}
_ => {
if !std::path::Path::new(&file_path).exists() {
eprintln!("⚠️ [APPLY] Video file not found, skipping: {}", file_path);
continue;
}
lightningbeam_core::video::VideoSource::Path(file_path.clone())
}
};
{ {
let mut video_mgr = self.video_manager.lock().unwrap(); let mut video_mgr = self.video_manager.lock().unwrap();
if let Err(e) = video_mgr.load_video(clip_id, file_path.clone(), doc_width, doc_height) { if let Err(e) = video_mgr.load_video(clip_id, source, doc_width, doc_height) {
eprintln!("⚠️ [APPLY] Failed to load video {}: {}", file_path, e); eprintln!("⚠️ [APPLY] Failed to load video {}: {}", clip_id, e);
continue; continue;
} }
} }
@ -4345,13 +4329,13 @@ impl EditorApp {
}; };
let Some(decoder_arc) = decoder_arc else { return; }; let Some(decoder_arc) = decoder_arc else { return; };
let (path, stream_index) = { let (source, stream_index) = {
let decoder = decoder_arc.lock().unwrap(); let decoder = decoder_arc.lock().unwrap();
decoder.keyframe_scan_params() decoder.keyframe_scan_params()
}; };
// Slow scan, no locks held. // Slow scan, no locks held.
match lightningbeam_core::video::VideoDecoder::scan_keyframes(&path, stream_index) { match lightningbeam_core::video::VideoDecoder::scan_keyframes(&source, stream_index) {
Ok(positions) => { Ok(positions) => {
let count = positions.len(); let count = positions.len();
decoder_arc.lock().unwrap().set_keyframe_index(positions); decoder_arc.lock().unwrap().set_keyframe_index(positions);
@ -4370,22 +4354,22 @@ impl EditorApp {
fn spawn_thumbnail_generation(&self, clip_id: uuid::Uuid) { fn spawn_thumbnail_generation(&self, clip_id: uuid::Uuid) {
let video_manager = Arc::clone(&self.video_manager); let video_manager = Arc::clone(&self.video_manager);
std::thread::spawn(move || { std::thread::spawn(move || {
// Grab the source path from the playback decoder (brief lock), then do // Grab the source from the playback decoder (brief lock), then do all
// all decoding on an independent decoder. // decoding on an independent decoder (a fresh BlobReader for packed video).
let path = { let source = {
let decoder_arc = { let decoder_arc = {
let vm = video_manager.lock().unwrap(); let vm = video_manager.lock().unwrap();
vm.get_decoder(&clip_id) vm.get_decoder(&clip_id)
}; };
let Some(decoder_arc) = decoder_arc else { return; }; let Some(decoder_arc) = decoder_arc else { return; };
let decoder = decoder_arc.lock().unwrap(); let decoder = decoder_arc.lock().unwrap();
decoder.path().to_string() decoder.source()
}; };
let vm_insert = Arc::clone(&video_manager); let vm_insert = Arc::clone(&video_manager);
let vm_skip = Arc::clone(&video_manager); let vm_skip = Arc::clone(&video_manager);
let result = lightningbeam_core::video::generate_keyframe_thumbnails( let result = lightningbeam_core::video::generate_keyframe_thumbnails(
&path, source,
5.0, 5.0,
128, 128,
// Resume: skip keyframes already covered (e.g. restored from a // Resume: skip keyframes already covered (e.g. restored from a
@ -4661,8 +4645,9 @@ impl EditorApp {
let path_str = path.to_string_lossy().to_string(); let path_str = path.to_string_lossy().to_string();
// Probe video for metadata // Probe video for metadata (freshly imported video is referenced until the
let metadata = match probe_video(&path_str) { // first save packs it).
let metadata = match probe_video(&lightningbeam_core::video::VideoSource::Path(path_str.clone())) {
Ok(meta) => meta, Ok(meta) => meta,
Err(e) => { Err(e) => {
eprintln!("Failed to probe video '{}': {}", name, e); eprintln!("Failed to probe video '{}': {}", name, e);
@ -4687,7 +4672,12 @@ impl EditorApp {
let doc_height = self.action_executor.document().height as u32; let doc_height = self.action_executor.document().height as u32;
let mut video_mgr = self.video_manager.lock().unwrap(); let mut video_mgr = self.video_manager.lock().unwrap();
if let Err(e) = video_mgr.load_video(clip_id, path_str.clone(), doc_width, doc_height) { if let Err(e) = video_mgr.load_video(
clip_id,
lightningbeam_core::video::VideoSource::Path(path_str.clone()),
doc_width,
doc_height,
) {
eprintln!("Failed to load video '{}': {}", name, e); eprintln!("Failed to load video '{}': {}", name, e);
return None; return None;
} }
@ -5775,7 +5765,7 @@ impl eframe::App for EditorApp {
.map(|(&lid, _)| lid); .map(|(&lid, _)| lid);
if let Some(layer_id) = recording_layer { if let Some(layer_id) = recording_layer {
// First, find the clip instance and clip id // First, find the clip instance and clip id
let (clip_id, instance_id, timeline_start, trim_start) = { let (clip_id, instance_id, _timeline_start, _trim_start) = {
let document = self.action_executor.document(); let document = self.action_executor.document();
document.get_layer(&layer_id) document.get_layer(&layer_id)
.and_then(|layer| { .and_then(|layer| {
@ -6134,6 +6124,10 @@ impl eframe::App for EditorApp {
audio_settings, audio_settings,
output_path, output_path,
Arc::clone(audio_controller), Arc::clone(audio_controller),
self.action_executor.document(),
Arc::clone(&self.video_manager),
self.raster_store.clone(),
self.current_file_path.clone(),
) { ) {
Ok(()) => true, Ok(()) => true,
Err(err) => { Err(err) => {
@ -6159,10 +6153,11 @@ impl eframe::App for EditorApp {
// Render export progress dialog and handle cancel // Render export progress dialog and handle cancel
if self.export_progress_dialog.render(ctx) { if self.export_progress_dialog.render(ctx) {
// User clicked Cancel // User clicked Cancel: stop + tear down the export, then dismiss the dialog.
if let Some(orchestrator) = &mut self.export_orchestrator { if let Some(orchestrator) = &mut self.export_orchestrator {
orchestrator.cancel(); orchestrator.cancel();
} }
self.export_progress_dialog.close();
} }
// Keep requesting repaints while export progress dialog is open // Keep requesting repaints while export progress dialog is open
@ -6191,6 +6186,11 @@ impl eframe::App for EditorApp {
// Render video frames incrementally (if video export in progress) // Render video frames incrementally (if video export in progress)
let exporting = self.export_orchestrator.as_ref().map_or(false, |o| o.is_exporting()); let exporting = self.export_orchestrator.as_ref().map_or(false, |o| o.is_exporting());
if exporting { if exporting {
// Keep the UI loop alive so progress is polled/drained even when the video is
// produced on a background thread (zero-copy path) that emits no UI-thread frames.
// Poll at ~6 Hz (not 60): the progress bar doesn't need more, and repainting the full
// editor every frame steals CPU/GPU from the background render thread, slowing export.
ctx.request_repaint_after(std::time::Duration::from_millis(160));
if let Some(render_state) = frame.wgpu_render_state() { if let Some(render_state) = frame.wgpu_render_state() {
let device = &render_state.device; let device = &render_state.device;
let queue = &render_state.queue; let queue = &render_state.queue;
@ -6256,8 +6256,10 @@ impl eframe::App for EditorApp {
self.export_image_cache = None; self.export_image_cache = None;
} }
// Poll export orchestrator for progress // Poll export orchestrator for progress — only while there's something to report
if let Some(orchestrator) = &mut self.export_orchestrator { // (otherwise this runs every repaint forever, spamming logs and wasting work). The
// orchestrator clears its state once the terminal Complete/Error is consumed.
if let Some(orchestrator) = self.export_orchestrator.as_mut().filter(|o| o.has_pending_progress()) {
// Only log occasionally to avoid spam // Only log occasionally to avoid spam
use std::sync::atomic::{AtomicU32, Ordering as AtomicOrdering}; use std::sync::atomic::{AtomicU32, Ordering as AtomicOrdering};
static POLL_COUNT: AtomicU32 = AtomicU32::new(0); static POLL_COUNT: AtomicU32 = AtomicU32::new(0);
@ -6551,7 +6553,6 @@ impl eframe::App for EditorApp {
pending_graph_loads: &self.pending_graph_loads, pending_graph_loads: &self.pending_graph_loads,
clipboard_consumed: &mut clipboard_consumed, clipboard_consumed: &mut clipboard_consumed,
keymap: &self.keymap, keymap: &self.keymap,
commit_raster_floating_if_any: &mut self.commit_raster_floating_if_any,
pending_node_group: &mut self.pending_node_group, pending_node_group: &mut self.pending_node_group,
pending_node_ungroup: &mut self.pending_node_ungroup, pending_node_ungroup: &mut self.pending_node_ungroup,
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
@ -6708,7 +6709,7 @@ impl eframe::App for EditorApp {
eprintln!("[WEBCAM] Recording saved to: {} (recorder duration={:.4}s)", file_path_str, result.duration); eprintln!("[WEBCAM] Recording saved to: {} (recorder duration={:.4}s)", file_path_str, result.duration);
// Create VideoClip + ClipInstance from recorded file // Create VideoClip + ClipInstance from recorded file
if let Some(layer_id) = webcam_layer_id { if let Some(layer_id) = webcam_layer_id {
match lightningbeam_core::video::probe_video(&file_path_str) { match lightningbeam_core::video::probe_video(&lightningbeam_core::video::VideoSource::Path(file_path_str.clone())) {
Ok(info) => { Ok(info) => {
use lightningbeam_core::clip::{VideoClip, ClipInstance}; use lightningbeam_core::clip::{VideoClip, ClipInstance};
let clip = VideoClip { let clip = VideoClip {
@ -6723,6 +6724,7 @@ impl eframe::App for EditorApp {
duration: info.duration, duration: info.duration,
frame_rate: info.fps, frame_rate: info.fps,
linked_audio_clip_id: None, linked_audio_clip_id: None,
media_id: None,
folder_id: None, folder_id: None,
}; };
let clip_id = clip.id; let clip_id = clip.id;
@ -6761,7 +6763,7 @@ impl eframe::App for EditorApp {
// for the display rect. // for the display rect.
{ {
let mut vm = self.video_manager.lock().unwrap(); let mut vm = self.video_manager.lock().unwrap();
if let Err(e) = vm.load_video(clip_id, file_path_str, info.width, info.height) { if let Err(e) = vm.load_video(clip_id, lightningbeam_core::video::VideoSource::Path(file_path_str.clone()), info.width, info.height) {
eprintln!("[WEBCAM] Failed to load recorded video: {}", e); eprintln!("[WEBCAM] Failed to load recorded video: {}", e);
} }
} }
@ -7512,7 +7514,7 @@ fn render_pane(
// Active tab highlight with per-corner rounding // Active tab highlight with per-corner rounding
if is_active { if is_active {
let cr = corner_r as u8; let cr = corner_r as u8;
let rounding = egui::Rounding { let rounding = egui::CornerRadius {
nw: if i == 0 { cr } else { 0 }, nw: if i == 0 { cr } else { 0 },
sw: if i == 0 { cr } else { 0 }, sw: if i == 0 { cr } else { 0 },
ne: if i == n - 1 { cr } else { 0 }, ne: if i == n - 1 { cr } else { 0 },
@ -7557,7 +7559,7 @@ fn render_pane(
if tab_response.hovered() && !is_active { if tab_response.hovered() && !is_active {
ui.painter().rect_filled( ui.painter().rect_filled(
tab_rect, tab_rect,
egui::Rounding { egui::CornerRadius {
nw: if i == 0 { corner_r as u8 } else { 0 }, nw: if i == 0 { corner_r as u8 } else { 0 },
sw: if i == 0 { corner_r as u8 } else { 0 }, sw: if i == 0 { corner_r as u8 } else { 0 },
ne: if i == n - 1 { corner_r as u8 } else { 0 }, ne: if i == n - 1 { corner_r as u8 } else { 0 },

4
lightningbeam-ui/lightningbeam-editor/src/panes/gradient_editor.rs
View File

@ -238,7 +238,7 @@ pub fn gradient_stop_editor(
// ── Paint handles ───────────────────────────────────────────────────── // ── Paint handles ─────────────────────────────────────────────────────
// handle_rects was built before any deletions this frame; guard against OOB. // handle_rects was built before any deletions this frame; guard against OOB.
for (i, h_rect) in handle_rects.iter().enumerate().take(gradient.stops.len()) { for (i, h_rect) in handle_rects.iter().enumerate().take(gradient.stops.len()) {
let col = ShapeColor_to_Color32(gradient.stops[i].color); let col = shape_color_to_color32(gradient.stops[i].color);
let is_selected = *selected_stop == Some(i); let is_selected = *selected_stop == Some(i);
let stroke = Stroke::new( let stroke = Stroke::new(
if is_selected { 2.0 } else { 1.0 }, if is_selected { 2.0 } else { 1.0 },
@ -308,7 +308,7 @@ pub fn gradient_stop_editor(
// ── Helpers ────────────────────────────────────────────────────────────────── // ── Helpers ──────────────────────────────────────────────────────────────────
fn ShapeColor_to_Color32(c: ShapeColor) -> Color32 { fn shape_color_to_color32(c: ShapeColor) -> Color32 {
Color32::from_rgba_unmultiplied(c.r, c.g, c.b, c.a) Color32::from_rgba_unmultiplied(c.r, c.g, c.b, c.a)
} }

36
lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs
View File

@ -11,7 +11,7 @@
/// - Document settings (when nothing is focused) /// - Document settings (when nothing is focused)
use eframe::egui::{self, DragValue, Ui}; use eframe::egui::{self, DragValue, Ui};
use lightningbeam_core::brush_settings::{bundled_brushes, BrushSettings}; use lightningbeam_core::brush_settings::bundled_brushes;
use lightningbeam_core::actions::{SetDocumentPropertiesAction, SetShapePropertiesAction, SetFillPaintAction, SetImageFillAction}; use lightningbeam_core::actions::{SetDocumentPropertiesAction, SetShapePropertiesAction, SetFillPaintAction, SetImageFillAction};
use lightningbeam_core::gradient::ShapeGradient; use lightningbeam_core::gradient::ShapeGradient;
use lightningbeam_core::layer::{AnyLayer, LayerTrait}; use lightningbeam_core::layer::{AnyLayer, LayerTrait};
@ -1439,40 +1439,6 @@ impl InfopanelPane {
} }
} }
/// Draw a brush dab preview into `rect` approximating the brush falloff shape.
///
/// Renders N concentric filled circles from outermost to innermost. Because each
/// inner circle overwrites the pixels of all outer circles beneath it, the visible
/// alpha at distance `d` from the centre equals the alpha of the innermost circle
/// whose radius ≥ `d`. This step-approximates the actual brush falloff formula:
/// `opa = ((1 − r) / (1 − hardness))²` for `r > hardness`, 1 inside the hard core.
fn paint_brush_dab(painter: &egui::Painter, rect: egui::Rect, s: &BrushSettings) {
let center = rect.center();
let max_r = (rect.width().min(rect.height()) / 2.0 - 2.0).max(1.0);
let h = s.hardness;
let a = s.opaque;
const N: usize = 12;
for i in 0..N {
// t: normalized radial position of this ring, 1.0 = outermost edge
let t = 1.0 - i as f32 / N as f32;
let r = max_r * t;
let opa_weight = if h >= 1.0 || t <= h {
1.0f32
} else {
let x = (1.0 - t) / (1.0 - h).max(1e-4);
(x * x).min(1.0)
};
let alpha = (opa_weight * a * 220.0).min(220.0) as u8;
painter.circle_filled(
center, r,
egui::Color32::from_rgba_unmultiplied(200, 200, 220, alpha),
);
}
}
/// Convert MIDI note number to note name (e.g. 60 -> "C4") /// Convert MIDI note number to note name (e.g. 60 -> "C4")
fn midi_note_name(note: u8) -> String { fn midi_note_name(note: u8) -> String {
const NAMES: [&str; 12] = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]; const NAMES: [&str; 12] = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"];

3
lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs
View File

@ -342,9 +342,6 @@ pub struct SharedPaneState<'a> {
pub clipboard_consumed: &'a mut bool, pub clipboard_consumed: &'a mut bool,
/// Remappable keyboard shortcut manager /// Remappable keyboard shortcut manager
pub keymap: &'a crate::keymap::KeymapManager, pub keymap: &'a crate::keymap::KeymapManager,
/// Set by raster selection tools when they need main to commit the floating
/// selection before starting a new interaction.
pub commit_raster_floating_if_any: &'a mut bool,
/// Set by MenuAction::Group when focus is Nodes — consumed by node graph pane /// Set by MenuAction::Group when focus is Nodes — consumed by node graph pane
pub pending_node_group: &'a mut bool, pub pending_node_group: &'a mut bool,
/// Set by MenuAction::Group (ungroup variant) when focus is Nodes — consumed by node graph pane /// Set by MenuAction::Group (ungroup variant) when focus is Nodes — consumed by node graph pane

47
lightningbeam-ui/lightningbeam-editor/src/panes/piano_roll.rs
View File

@ -240,12 +240,6 @@ impl PianoRollPane {
matches!(note % 12, 1 | 3 | 6 | 8 | 10) matches!(note % 12, 1 | 3 | 6 | 8 | 10)
} }
fn note_name(note: u8) -> String {
let names = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"];
let octave = (note / 12) as i32 - 1;
format!("{}{}", names[note as usize % 12], octave)
}
// ── Note resolution ────────────────────────────────────────────────── // ── Note resolution ──────────────────────────────────────────────────
fn resolve_notes(events: &[daw_backend::audio::midi::MidiEvent]) -> Vec<ResolvedNote> { fn resolve_notes(events: &[daw_backend::audio::midi::MidiEvent]) -> Vec<ResolvedNote> {
@ -736,45 +730,6 @@ impl PianoRollPane {
} }
} }
/// Generate pitch bend MIDI events for a note based on the zone and target semitones.
fn generate_pitch_bend_events(
note_start: f64,
note_duration: f64,
zone: PitchBendZone,
semitones: f32,
channel: u8,
pitch_bend_range: f32,
) -> Vec<daw_backend::audio::midi::MidiEvent> {
use daw_backend::audio::midi::MidiEvent;
let num_steps: usize = 128;
let mut events = Vec::new();
let encode_bend = |normalized: f32| -> (u8, u8) {
let value_14 = (normalized * 8191.0 + 8192.0).clamp(0.0, 16383.0) as i16;
((value_14 & 0x7F) as u8, ((value_14 >> 7) & 0x7F) as u8)
};
// Use t directly (0..=1 across the full note) — same formula as the visual ghost.
// Start: peak → 0 (ramps down over full note)
// Middle: 0 → peak → 0 (sine arch, peaks at center)
// End: 0 → peak (ramps up over full note)
for i in 0..=num_steps {
let t = i as f64 / num_steps as f64;
let t_f32 = t as f32;
// Cosine ease curves: Start+End at equal value = perfectly flat (partition of unity).
// Start: (1+cos(πt))/2 — peaks at t=0, smooth decay to 0 at t=1
// End: (1-cos(πt))/2 — 0 at t=0, smooth rise to peak at t=1
// Middle: sin(πt) — arch peaking at t=0.5
let normalized = match zone {
PitchBendZone::Start => semitones / pitch_bend_range * (1.0 + (std::f32::consts::PI * t_f32).cos()) * 0.5,
PitchBendZone::Middle => semitones / pitch_bend_range * (std::f32::consts::PI * t_f32).sin(),
PitchBendZone::End => semitones / pitch_bend_range * (1.0 - (std::f32::consts::PI * t_f32).cos()) * 0.5,
};
let timestamp = note_start + t * note_duration;
let (lsb, msb) = encode_bend(normalized);
events.push(MidiEvent::new(daw_backend::Beats(timestamp), 0xE0 | channel, lsb, msb));
}
events
}
/// Find the lowest available MIDI channel (1–15) not already used by any note /// Find the lowest available MIDI channel (1–15) not already used by any note
/// overlapping [note_start, note_end], excluding the note being assigned itself. /// overlapping [note_start, note_end], excluding the note being assigned itself.
/// Returns the note's current channel unchanged if it is already uniquely assigned (non-zero). /// Returns the note's current channel unchanged if it is already uniquely assigned (non-zero).
@ -2408,7 +2363,7 @@ impl PaneRenderer for PianoRollPane {
let doc = shared.action_executor.document(); let doc = shared.action_executor.document();
let is_measures = doc.timeline_mode == lightningbeam_core::document::TimelineMode::Measures; let is_measures = doc.timeline_mode == lightningbeam_core::document::TimelineMode::Measures;
let tempo_map = doc.tempo_map(); let tempo_map = doc.tempo_map();
drop(doc); let _ = doc;
if is_measures { if is_measures {
// Auto-detect grid when selection changes // Auto-detect grid when selection changes

22
lightningbeam-ui/lightningbeam-editor/src/panes/shaders/canvas_blit.wgsl
View File

@ -75,3 +75,25 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
let tinted = base + tint - base * tint; let tinted = base + tint - base * tint;
return vec4<f32>(tinted, masked_a); return vec4<f32>(tinted, masked_a);
} }
// Variant for sources that are ALREADY straight-alpha linear — notably a video
// frame uploaded to an `Rgba8UnormSrgb` texture, where the hardware decodes
// sRGB→linear on sample and leaves alpha untouched. No unpremultiply (the source
// was never premultiplied), so we skip the divide entirely. The compositor wants
// straight-alpha linear, which is exactly what the sample already is.
@fragment
fn fs_main_straight(in: VertexOutput) -> @location(0) vec4<f32> {
let m = mat3x3<f32>(transform.col0.xyz, transform.col1.xyz, transform.col2.xyz);
let canvas_uv = (m * vec3<f32>(in.uv.x, in.uv.y, 1.0)).xy;
if canvas_uv.x < 0.0 || canvas_uv.x > 1.0
|| canvas_uv.y < 0.0 || canvas_uv.y > 1.0 {
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
}
let c = textureSample(canvas_tex, canvas_sampler, canvas_uv);
let mask = textureSample(mask_tex, mask_sampler, canvas_uv).r;
let tint = vec3<f32>(transform.col0.w, transform.col1.w, transform.col2.w);
let tinted = c.rgb + tint - c.rgb * tint;
return vec4<f32>(tinted, c.a * mask);
}

293
lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs
View File

@ -45,6 +45,89 @@ fn upload_pixmap_to_texture(queue: &wgpu::Queue, texture: &wgpu::Texture, pixmap
/// Set to true to use the new pipeline, false for legacy single-scene rendering /// Set to true to use the new pipeline, false for legacy single-scene rendering
const USE_HDR_COMPOSITING: bool = true; // Enabled for testing const USE_HDR_COMPOSITING: bool = true; // Enabled for testing
/// Caches GPU textures for decoded video frames, keyed by the frame buffer's `Arc`
/// identity. A static/paused video then costs ~nothing per repaint (cache hit → no
/// per-pixel CPU sRGB→linear conversion, no texture allocation, no upload). The
/// cached texture holds premultiplied-linear RGBA8 — exactly what `canvas_blit`
/// expects. During playback each new decoded frame is a fresh `Arc` → one
/// conversion+upload per frame (not per repaint).
struct CachedVideoFrame {
/// Keep the source buffer alive so its pointer (our cache key) can't be reused.
_keep: std::sync::Arc<Vec<u8>>,
texture: wgpu::Texture,
last_seen: u64,
}
struct VideoFrameTexCache {
entries: std::collections::HashMap<usize, CachedVideoFrame>,
frame: u64,
}
impl VideoFrameTexCache {
fn new() -> Self {
Self { entries: std::collections::HashMap::new(), frame: 0 }
}
fn begin_frame(&mut self) {
self.frame = self.frame.wrapping_add(1);
}
/// View of the cached (or freshly converted+uploaded) texture for `rgba`.
fn texture_view(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
rgba: &std::sync::Arc<Vec<u8>>,
w: u32,
h: u32,
) -> wgpu::TextureView {
let key = std::sync::Arc::as_ptr(rgba) as usize;
let frame = self.frame;
if let Some(e) = self.entries.get_mut(&key) {
e.last_seen = frame;
return e.texture.create_view(&wgpu::TextureViewDescriptor::default());
}
// Miss: upload the raw sRGB straight-alpha bytes verbatim into an sRGB
// texture. The GPU decodes sRGB→linear on sample (free), so there is no
// per-pixel CPU conversion — the cost that used to dominate playback/export.
// Blit this with `blit_straight` (it must NOT unpremultiply).
let texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some("video_frame_tex"),
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8UnormSrgb,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
rgba,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(w * 4),
rows_per_image: Some(h),
},
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
self.entries.insert(key, CachedVideoFrame { _keep: rgba.clone(), texture, last_seen: frame });
view
}
/// Drop textures not used in the last couple of frames (bounds VRAM).
fn evict_stale(&mut self) {
let frame = self.frame;
self.entries.retain(|_, e| e.last_seen + 2 >= frame);
}
}
/// Shared Vello resources (created once, reused by all Stage panes) /// Shared Vello resources (created once, reused by all Stage panes)
struct SharedVelloResources { struct SharedVelloResources {
renderer: Arc<Mutex<vello::Renderer>>, renderer: Arc<Mutex<vello::Renderer>>,
@ -72,6 +155,11 @@ struct SharedVelloResources {
/// True when Vello is running its CPU software renderer (either forced or GPU fallback). /// True when Vello is running its CPU software renderer (either forced or GPU fallback).
/// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area. /// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area.
is_cpu_renderer: bool, is_cpu_renderer: bool,
/// GPU timestamp timer for the composite section (F3 debug). Lazily created on
/// the first frame (needs the device/queue); `None` if unsupported.
gpu_timer: Mutex<Option<crate::gpu_timer::FrameGpuTimer>>,
/// Per-frame video texture cache (skips re-converting/uploading a static frame).
video_frame_cache: Mutex<VideoFrameTexCache>,
} }
/// Per-instance Vello resources (created for each Stage pane) /// Per-instance Vello resources (created for each Stage pane)
@ -302,6 +390,8 @@ impl SharedVelloResources {
gpu_brush: Mutex::new(gpu_brush), gpu_brush: Mutex::new(gpu_brush),
canvas_blit, canvas_blit,
is_cpu_renderer: use_cpu || is_cpu_renderer, is_cpu_renderer: use_cpu || is_cpu_renderer,
gpu_timer: Mutex::new(None),
video_frame_cache: Mutex::new(VideoFrameTexCache::new()),
}) })
} }
} }
@ -682,17 +772,13 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
if let Some(b_id) = self.ctx.pending_tool_readback_b { if let Some(b_id) = self.ctx.pending_tool_readback_b {
if let Ok(mut gpu_brush) = shared.gpu_brush.lock() { if let Ok(mut gpu_brush) = shared.gpu_brush.lock() {
let dims = gpu_brush.canvases.get(&b_id).map(|c| (c.width, c.height)); let dims = gpu_brush.canvases.get(&b_id).map(|c| (c.width, c.height));
if let Some((w, h)) = dims { if let Some((_w, _h)) = dims {
if let Some(pixels) = gpu_brush.readback_canvas(device, queue, b_id) { if let Some(pixels) = gpu_brush.readback_canvas(device, queue, b_id) {
let results = RASTER_READBACK_RESULTS.get_or_init(|| { let results = RASTER_READBACK_RESULTS.get_or_init(|| {
Arc::new(Mutex::new(std::collections::HashMap::new())) Arc::new(Mutex::new(std::collections::HashMap::new()))
}); });
if let Ok(mut map) = results.lock() { if let Ok(mut map) = results.lock() {
map.insert(self.ctx.instance_id_for_readback, RasterReadbackResult { map.insert(self.ctx.instance_id_for_readback, RasterReadbackResult {
layer_id: uuid::Uuid::nil(), // unused; routing via pending_undo_before
time: 0.0,
canvas_width: w,
canvas_height: h,
pixels, pixels,
}); });
} }
@ -763,10 +849,6 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
}); });
if let Ok(mut map) = results.lock() { if let Ok(mut map) = results.lock() {
map.insert(self.ctx.instance_id_for_readback, RasterReadbackResult { map.insert(self.ctx.instance_id_for_readback, RasterReadbackResult {
layer_id: pending.layer_id,
time: pending.time,
canvas_width: pending.canvas_width,
canvas_height: pending.canvas_height,
pixels, pixels,
}); });
} }
@ -1087,6 +1169,25 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
// HDR buffer spec for linear buffers // HDR buffer spec for linear buffers
let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float); let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float);
// F3: bracket the composite section with a GPU timestamp (lazily create the
// timer; no-op when the adapter lacks timestamp support).
let ts_supported = device
.features()
.contains(crate::gpu_timer::FrameGpuTimer::required_features());
{
let mut tg = shared.gpu_timer.lock().unwrap();
if tg.is_none() && ts_supported {
*tg = crate::gpu_timer::FrameGpuTimer::new(device, queue);
}
if let Some(t) = tg.as_ref() {
t.start(device, queue);
}
}
shared.video_frame_cache.lock().unwrap().begin_frame();
// F3: CPU breakdown of the composite (the GPU idles waiting on these).
let mut cput = crate::debug_overlay::CompositeCpuBreakdown::default();
// First, render background and composite it // First, render background and composite it
// The background scene contains only a rectangle at document bounds, // The background scene contains only a rectangle at document bounds,
// so we use TRANSPARENT base_color to not fill the whole viewport // so we use TRANSPARENT base_color to not fill the whole viewport
@ -1105,6 +1206,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
antialiasing_method: aa_method, antialiasing_method: aa_method,
}; };
let _t = std::time::Instant::now();
if let Some(pixmap) = &composite_result.background_cpu { if let Some(pixmap) = &composite_result.background_cpu {
if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) { if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) {
upload_pixmap_to_texture(queue, tex, pixmap); upload_pixmap_to_texture(queue, tex, pixmap);
@ -1112,13 +1214,18 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
} else if let Ok(mut renderer) = shared.renderer.lock() { } else if let Ok(mut renderer) = shared.renderer.lock() {
renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok(); renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok();
} }
cput.vello_ms += _t.elapsed().as_secs_f64() * 1000.0;
// Convert sRGB to linear HDR // Convert sRGB to linear HDR
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("bg_srgb_to_linear_encoder"), label: Some("bg_srgb_to_linear_encoder"),
}); });
let _t = std::time::Instant::now();
shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view); shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view);
cput.convert_ms += _t.elapsed().as_secs_f64() * 1000.0;
let _t = std::time::Instant::now();
queue.submit(Some(convert_encoder.finish())); queue.submit(Some(convert_encoder.finish()));
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
// Composite background onto HDR texture (first layer, clears to dark gray for stage area) // Composite background onto HDR texture (first layer, clears to dark gray for stage area)
let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0); let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0);
@ -1128,6 +1235,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
// Clear to dark gray (stage background outside document bounds) // Clear to dark gray (stage background outside document bounds)
// Note: stage_bg values are already in linear space for HDR compositing // Note: stage_bg values are already in linear space for HDR compositing
let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0]; let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0];
let _t = std::time::Instant::now();
shared.compositor.composite( shared.compositor.composite(
device, device,
queue, queue,
@ -1137,7 +1245,10 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
hdr_view, hdr_view,
Some(stage_bg), Some(stage_bg),
); );
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
let _t = std::time::Instant::now();
queue.submit(Some(encoder.finish())); queue.submit(Some(encoder.finish()));
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
} }
buffer_pool.release(bg_srgb_handle); buffer_pool.release(bg_srgb_handle);
buffer_pool.release(bg_hdr_handle); buffer_pool.release(bg_hdr_handle);
@ -1359,6 +1470,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
buffer_pool.get_view(hdr_layer_handle), buffer_pool.get_view(hdr_layer_handle),
&instance_resources.hdr_texture_view, &instance_resources.hdr_texture_view,
) { ) {
let _t_vello = std::time::Instant::now();
if let Some(pixmap) = &rendered_layer.cpu_pixmap { if let Some(pixmap) = &rendered_layer.cpu_pixmap {
if let Some(tex) = buffer_pool.get_texture(srgb_handle) { if let Some(tex) = buffer_pool.get_texture(srgb_handle) {
upload_pixmap_to_texture(queue, tex, pixmap); upload_pixmap_to_texture(queue, tex, pixmap);
@ -1366,6 +1478,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
} else if let Ok(mut renderer) = shared.renderer.lock() { } else if let Ok(mut renderer) = shared.renderer.lock() {
renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok(); renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok();
} }
cput.vello_ms += _t_vello.elapsed().as_secs_f64() * 1000.0;
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("layer_srgb_to_linear_encoder"), label: Some("layer_srgb_to_linear_encoder"),
}); });
@ -1563,7 +1676,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
} }
} }
RenderedLayerType::Video { instances } => { RenderedLayerType::Video { instances } => {
// Video layer — per-instance: upload decoded frame → blit → composite. // Video layer — per-instance: (cached) frame texture → blit → composite.
for inst in instances { for inst in instances {
if inst.rgba_data.is_empty() { continue; } if inst.rgba_data.is_empty() { continue; }
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
@ -1571,40 +1684,20 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
buffer_pool.get_view(hdr_layer_handle), buffer_pool.get_view(hdr_layer_handle),
&instance_resources.hdr_texture_view, &instance_resources.hdr_texture_view,
) { ) {
// Convert sRGB straight-alpha → linear premultiplied. // Reuse the GPU texture for this frame if it's unchanged (a
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| { // static/paused video → no CPU conversion, alloc, or upload).
let a = p[3] as f32 / 255.0; // Timed into `blit_ms` (incl the cache lookup + per-frame view).
let lin = |c: u8| -> f32 { let _t = std::time::Instant::now();
let f = c as f32 / 255.0; let tex_view = shared
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) } .video_frame_cache
}; .lock()
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8; .unwrap()
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8; .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
[r, g, b, p[3]]
}).collect();
let tex = device.create_texture(&wgpu::TextureDescriptor {
label: Some("video_frame_tex"),
size: wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
mip_level_count: 1, sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
queue.write_texture(
wgpu::TexelCopyTextureInfo { texture: &tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
&linear,
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(inst.width * 4), rows_per_image: Some(inst.height) },
wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
);
let tex_view = tex.create_view(&wgpu::TextureViewDescriptor::default());
let bt = crate::gpu_brush::BlitTransform::new( let bt = crate::gpu_brush::BlitTransform::new(
inst.transform, inst.width, inst.height, width, height, inst.transform, inst.width, inst.height, width, height,
); );
shared.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None); shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0;
let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new( let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(
hdr_layer_handle, hdr_layer_handle,
@ -1614,10 +1707,14 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("video_composite_encoder"), label: Some("video_composite_encoder"),
}); });
let _t = std::time::Instant::now();
shared.compositor.composite( shared.compositor.composite(
device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None, device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None,
); );
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
let _t = std::time::Instant::now();
queue.submit(Some(encoder.finish())); queue.submit(Some(encoder.finish()));
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
} }
buffer_pool.release(hdr_layer_handle); buffer_pool.release(hdr_layer_handle);
} }
@ -1849,6 +1946,23 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
buffer_pool.next_frame(); buffer_pool.next_frame();
drop(buffer_pool); drop(buffer_pool);
// F3: close the GPU timestamp bracket + publish the composite measurement.
{
let layers = composite_result.layers.len() as u32;
// Submits aren't counted per-site; estimate from the per-layer pattern
// (bg ~3 + ~2 per layer). Drops toward ~1 once the passes are batched.
let submits_est = 3 + 2 * layers;
let mut tg = shared.gpu_timer.lock().unwrap();
if let Some(t) = tg.as_mut() {
t.end(device, queue);
crate::debug_overlay::update_gpu_composite(true, t.last_ms(), layers, submits_est);
} else {
crate::debug_overlay::update_gpu_composite(false, 0.0, layers, submits_est);
}
}
crate::debug_overlay::update_composite_cpu(cput);
shared.video_frame_cache.lock().unwrap().evict_stale();
// --- Frame timing report --- // --- Frame timing report ---
let _t_end = std::time::Instant::now(); let _t_end = std::time::Instant::now();
let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0; let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0;
@ -3173,7 +3287,6 @@ enum PendingWarpOp {
disp_data: Option<Vec<[f32; 2]>>, disp_data: Option<Vec<[f32; 2]>>,
grid_cols: u32, grid_cols: u32,
grid_rows: u32, grid_rows: u32,
w: u32, h: u32,
final_commit: bool, final_commit: bool,
layer_id: uuid::Uuid, layer_id: uuid::Uuid,
time: f64, time: f64,
@ -3190,7 +3303,6 @@ enum PendingWarpOp {
anchor_canvas_id: uuid::Uuid, anchor_canvas_id: uuid::Uuid,
disp_buf_id: uuid::Uuid, disp_buf_id: uuid::Uuid,
display_canvas_id: uuid::Uuid, display_canvas_id: uuid::Uuid,
w: u32, h: u32,
final_commit: bool, final_commit: bool,
layer_id: uuid::Uuid, layer_id: uuid::Uuid,
time: f64, time: f64,
@ -3251,10 +3363,6 @@ static EYEDROPPER_RESULTS: OnceLock<Arc<Mutex<std::collections::HashMap<u64, (eg
struct PendingRasterDabs { struct PendingRasterDabs {
/// Keyframe UUID — indexes the canvas texture pair in `GpuBrushEngine`. /// Keyframe UUID — indexes the canvas texture pair in `GpuBrushEngine`.
keyframe_id: uuid::Uuid, keyframe_id: uuid::Uuid,
/// Layer UUID — used for the undo readback result.
layer_id: uuid::Uuid,
/// Playback time of the keyframe.
time: f64,
/// Canvas dimensions (pixels). /// Canvas dimensions (pixels).
canvas_width: u32, canvas_width: u32,
canvas_height: u32, canvas_height: u32,
@ -3379,10 +3487,6 @@ static TRANSFORM_READBACK_RESULTS: OnceLock<Arc<Mutex<std::collections::HashMap<
/// Result stored by `prepare()` after a stroke-end readback. /// Result stored by `prepare()` after a stroke-end readback.
struct RasterReadbackResult { struct RasterReadbackResult {
layer_id: uuid::Uuid,
time: f64,
canvas_width: u32,
canvas_height: u32,
/// Raw RGBA pixels from the completed stroke. /// Raw RGBA pixels from the completed stroke.
pixels: Vec<u8>, pixels: Vec<u8>,
} }
@ -4201,7 +4305,7 @@ impl StagePane {
// Update connected edges: shift the adjacent control point by the same delta // Update connected edges: shift the adjacent control point by the same delta
for &edge_id in &connected_edges { for &edge_id in &connected_edges {
let [v0, v1] = graph.edge(edge_id).vertices; let [v0, _v1] = graph.edge(edge_id).vertices;
let mut curve = graph.edge(edge_id).curve; let mut curve = graph.edge(edge_id).curve;
if v0 == vertex_id { if v0 == vertex_id {
@ -5609,58 +5713,6 @@ impl StagePane {
} }
} }
/// Lift the pixels enclosed by the current `raster_selection` into a
/// `RasterFloatingSelection`, punching a transparent hole in `raw_pixels`.
///
/// Call this immediately after a marquee / lasso selection is finalized so
/// that all downstream operations (drag-move, copy, cut, stroke-masking)
/// see a consistent `raster_floating` whenever a selection is active.
/// Build an R8 mask buffer (0 = outside, 255 = inside) from a selection.
fn build_selection_mask(
sel: &lightningbeam_core::selection::RasterSelection,
width: u32,
height: u32,
) -> Vec<u8> {
let mut mask = vec![0u8; (width * height) as usize];
let (x0, y0, x1, y1) = sel.bounding_rect();
let bx0 = x0.max(0) as u32;
let by0 = y0.max(0) as u32;
let bx1 = (x1 as u32).min(width);
let by1 = (y1 as u32).min(height);
for y in by0..by1 {
for x in bx0..bx1 {
if sel.contains_pixel(x as i32, y as i32) {
mask[(y * width + x) as usize] = 255;
}
}
}
mask
}
/// Build an R8 mask buffer for the float canvas (0 = outside selection, 255 = inside).
/// Coordinates are in float-local space: pixel (fx, fy) corresponds to document pixel
/// (float_x+fx, float_y+fy).
fn build_float_mask(
sel: &lightningbeam_core::selection::RasterSelection,
float_x: i32, float_y: i32,
float_w: u32, float_h: u32,
) -> Vec<u8> {
let mut mask = vec![0u8; (float_w * float_h) as usize];
let (x0, y0, x1, y1) = sel.bounding_rect();
let bx0 = (x0 - float_x).max(0) as u32;
let by0 = (y0 - float_y).max(0) as u32;
let bx1 = ((x1 - float_x) as u32).min(float_w);
let by1 = ((y1 - float_y) as u32).min(float_h);
for fy in by0..by1 {
for fx in bx0..bx1 {
if sel.contains_pixel(float_x + fx as i32, float_y + fy as i32) {
mask[(fy * float_w + fx) as usize] = 255;
}
}
}
mask
}
/// Allocate the three A/B/C GPU canvases and build a [`crate::raster_tool::RasterWorkspace`] /// Allocate the three A/B/C GPU canvases and build a [`crate::raster_tool::RasterWorkspace`]
/// for a new raster tool operation. /// for a new raster tool operation.
/// ///
@ -5699,7 +5751,6 @@ impl StagePane {
a_canvas_id: a_id, a_canvas_id: a_id,
b_canvas_id: b_id, b_canvas_id: b_id,
c_canvas_id: c_id, c_canvas_id: c_id,
mask_texture: None,
width: w, width: w,
height: h, height: h,
x, x,
@ -5724,7 +5775,7 @@ impl StagePane {
let layer_id = (*shared.active_layer_id)?; let layer_id = (*shared.active_layer_id)?;
let time = *shared.playback_time; let time = *shared.playback_time;
let (doc_w, doc_h) = { let (_doc_w, _doc_h) = {
let doc = shared.action_executor.document(); let doc = shared.action_executor.document();
(doc.width as u32, doc.height as u32) (doc.width as u32, doc.height as u32)
}; };
@ -5733,7 +5784,7 @@ impl StagePane {
// returns None (no workspace) if there's no active keyframe to lift from. // returns None (no workspace) if there's no active keyframe to lift from.
// Read keyframe id and pixels. // Read keyframe id and pixels.
let (kf_id, w, h, pixels) = { let (_kf_id, w, h, pixels) = {
let doc = shared.action_executor.document(); let doc = shared.action_executor.document();
let AnyLayer::Raster(rl) = doc.get_layer(&layer_id)? else { return None }; let AnyLayer::Raster(rl) = doc.get_layer(&layer_id)? else { return None };
let kf = rl.keyframe_at(time)?; let kf = rl.keyframe_at(time)?;
@ -5753,7 +5804,6 @@ impl StagePane {
a_canvas_id: a_id, a_canvas_id: a_id,
b_canvas_id: b_id, b_canvas_id: b_id,
c_canvas_id: c_id, c_canvas_id: c_id,
mask_texture: None,
width: w, width: w,
height: h, height: h,
x: 0, x: 0,
@ -5761,9 +5811,6 @@ impl StagePane {
source: WorkspaceSource::Layer { source: WorkspaceSource::Layer {
layer_id, layer_id,
time, time,
kf_id,
canvas_w: doc_w,
canvas_h: doc_h,
}, },
before_pixels: pixels.clone(), before_pixels: pixels.clone(),
}; };
@ -6135,8 +6182,6 @@ impl StagePane {
)); ));
self.pending_raster_dabs = Some(PendingRasterDabs { self.pending_raster_dabs = Some(PendingRasterDabs {
keyframe_id: canvas_id, keyframe_id: canvas_id,
layer_id,
time,
canvas_width, canvas_width,
canvas_height, canvas_height,
initial_pixels: None, // canvas already initialized via lazy GPU init initial_pixels: None, // canvas already initialized via lazy GPU init
@ -6220,8 +6265,6 @@ impl StagePane {
)); ));
self.pending_raster_dabs = Some(PendingRasterDabs { self.pending_raster_dabs = Some(PendingRasterDabs {
keyframe_id, keyframe_id,
layer_id: active_layer_id,
time: kf_time,
canvas_width, canvas_width,
canvas_height, canvas_height,
initial_pixels: Some(initial_pixels), initial_pixels: Some(initial_pixels),
@ -6299,8 +6342,6 @@ impl StagePane {
let (dabs, dab_bbox) = BrushEngine::compute_dabs(&seg, stroke_state, dt); let (dabs, dab_bbox) = BrushEngine::compute_dabs(&seg, stroke_state, dt);
self.pending_raster_dabs = Some(PendingRasterDabs { self.pending_raster_dabs = Some(PendingRasterDabs {
keyframe_id: canvas_id, keyframe_id: canvas_id,
layer_id,
time,
canvas_width: cw, canvas_width: cw,
canvas_height: ch, canvas_height: ch,
initial_pixels: None, initial_pixels: None,
@ -6363,8 +6404,6 @@ impl StagePane {
if !dabs.is_empty() { if !dabs.is_empty() {
self.pending_raster_dabs = Some(PendingRasterDabs { self.pending_raster_dabs = Some(PendingRasterDabs {
keyframe_id: canvas_id, keyframe_id: canvas_id,
layer_id,
time,
canvas_width: cw, canvas_width: cw,
canvas_height: ch, canvas_height: ch,
initial_pixels: None, initial_pixels: None,
@ -6424,8 +6463,6 @@ impl StagePane {
if let Some(kf_id) = kf_id { if let Some(kf_id) = kf_id {
self.pending_raster_dabs = Some(PendingRasterDabs { self.pending_raster_dabs = Some(PendingRasterDabs {
keyframe_id: kf_id, keyframe_id: kf_id,
layer_id: ub_layer,
time: ub_time,
canvas_width: ub_cw, canvas_width: ub_cw,
canvas_height: ub_ch, canvas_height: ub_ch,
initial_pixels: None, initial_pixels: None,
@ -6941,7 +6978,7 @@ impl StagePane {
fn handle_quick_select_tool( fn handle_quick_select_tool(
&mut self, &mut self,
ui: &mut egui::Ui, _ui: &mut egui::Ui,
response: &egui::Response, response: &egui::Response,
world_pos: egui::Vec2, world_pos: egui::Vec2,
shared: &mut SharedPaneState, shared: &mut SharedPaneState,
@ -8431,7 +8468,6 @@ impl StagePane {
world_pos: egui::Vec2, world_pos: egui::Vec2,
shared: &mut SharedPaneState, shared: &mut SharedPaneState,
) { ) {
use lightningbeam_core::tool::Tool;
use uuid::Uuid; use uuid::Uuid;
// Ensure we're on a raster layer. // Ensure we're on a raster layer.
@ -8469,7 +8505,6 @@ impl StagePane {
disp_data: Some(disp_data), disp_data: Some(disp_data),
grid_cols: ws.grid_cols, grid_cols: ws.grid_cols,
grid_rows: ws.grid_rows, grid_rows: ws.grid_rows,
w: ws.anchor_w, h: ws.anchor_h,
final_commit: true, final_commit: true,
layer_id: ws.layer_id, layer_id: ws.layer_id,
time: ws.time, time: ws.time,
@ -8695,7 +8730,6 @@ impl StagePane {
disp_data: Some(disp_data), disp_data: Some(disp_data),
grid_cols: ws.grid_cols, grid_cols: ws.grid_cols,
grid_rows: ws.grid_rows, grid_rows: ws.grid_rows,
w: ws.anchor_w, h: ws.anchor_h,
final_commit: false, final_commit: false,
layer_id: ws.layer_id, layer_id: ws.layer_id,
time: ws.time, time: ws.time,
@ -8705,7 +8739,7 @@ impl StagePane {
None None
}; };
let (ws_layer_id, ws_display_id, ws_float_offset) = (ws.layer_id, ws.display_canvas_id, ws.float_offset); let (ws_layer_id, ws_display_id, ws_float_offset) = (ws.layer_id, ws.display_canvas_id, ws.float_offset);
drop(ws); // release borrow of warp_state let _ = ws; // release borrow of warp_state
// Display canvas is initialised by Init (zero-displacement apply), so it always // Display canvas is initialised by Init (zero-displacement apply), so it always
// has valid content. For full-layer warp, override the layer blit unconditionally. // has valid content. For full-layer warp, override the layer blit unconditionally.
@ -8778,7 +8812,6 @@ impl StagePane {
anchor_canvas_id: ls.anchor_canvas_id, anchor_canvas_id: ls.anchor_canvas_id,
disp_buf_id: ls.disp_buf_id, disp_buf_id: ls.disp_buf_id,
display_canvas_id: ls.display_canvas_id, display_canvas_id: ls.display_canvas_id,
w: ls.anchor_w, h: ls.anchor_h,
final_commit: true, final_commit: true,
layer_id: ls.layer_id, layer_id: ls.layer_id,
time: ls.time, time: ls.time,
@ -8950,7 +8983,6 @@ impl StagePane {
anchor_canvas_id: anchor_id, anchor_canvas_id: anchor_id,
disp_buf_id: disp_buf, disp_buf_id: disp_buf,
display_canvas_id: display_id, display_canvas_id: display_id,
w, h,
final_commit: false, final_commit: false,
layer_id: ls_layer_id, layer_id: ls_layer_id,
time, time,
@ -9013,7 +9045,7 @@ impl StagePane {
} else { None } } else { None }
} else { None } } else { None }
} else { None }; } else { None };
drop(doc); let _ = doc;
r r
} else { None }; } else { None };
@ -10706,7 +10738,6 @@ impl StagePane {
// Alt+click: set source point for clone/healing tools. // Alt+click: set source point for clone/healing tools.
{ {
use lightningbeam_core::tool::Tool;
let tool_uses_alt = crate::tools::raster_tool_def(shared.selected_tool) let tool_uses_alt = crate::tools::raster_tool_def(shared.selected_tool)
.map_or(false, |d| d.uses_alt_click()); .map_or(false, |d| d.uses_alt_click());
if tool_uses_alt if tool_uses_alt
@ -11602,7 +11633,6 @@ impl PaneRenderer for StagePane {
disp_data: Some(disp_data), disp_data: Some(disp_data),
grid_cols: ws.grid_cols, grid_cols: ws.grid_cols,
grid_rows: ws.grid_rows, grid_rows: ws.grid_rows,
w: ws.anchor_w, h: ws.anchor_h,
final_commit: true, final_commit: true,
layer_id: ws.layer_id, layer_id: ws.layer_id,
time: ws.time, time: ws.time,
@ -11623,7 +11653,6 @@ impl PaneRenderer for StagePane {
anchor_canvas_id: ls.anchor_canvas_id, anchor_canvas_id: ls.anchor_canvas_id,
disp_buf_id: ls.disp_buf_id, disp_buf_id: ls.disp_buf_id,
display_canvas_id: ls.display_canvas_id, display_canvas_id: ls.display_canvas_id,
w: ls.anchor_w, h: ls.anchor_h,
final_commit: true, final_commit: true,
layer_id: ls.layer_id, layer_id: ls.layer_id,
time: ls.time, time: ls.time,

13
lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs
View File

@ -483,7 +483,6 @@ struct AutomationLaneRender {
value_max: f32, value_max: f32,
accent_color: egui::Color32, accent_color: egui::Color32,
playback_time: f64, playback_time: f64,
kind: AutomationLaneKind,
} }
/// Pending automation keyframe edit action from curve lane interaction /// Pending automation keyframe edit action from curve lane interaction
@ -1160,7 +1159,7 @@ impl TimelinePane {
*shared.recording_clips.get(&layer_id).unwrap_or(&0), daw_backend::Beats::ZERO); *shared.recording_clips.get(&layer_id).unwrap_or(&0), daw_backend::Beats::ZERO);
let doc_clip_id = shared.action_executor.document_mut().add_audio_clip(doc_clip); let doc_clip_id = shared.action_executor.document_mut().add_audio_clip(doc_clip);
let mut clip_instance = ClipInstance::new(doc_clip_id) let clip_instance = ClipInstance::new(doc_clip_id)
.with_timeline_start(start_time); .with_timeline_start(start_time);
if let Some(layer) = shared.action_executor.document_mut().get_layer_mut(&layer_id) { if let Some(layer) = shared.action_executor.document_mut().get_layer_mut(&layer_id) {
@ -3230,7 +3229,6 @@ impl TimelinePane {
value_max: lane.value_max, value_max: lane.value_max,
accent_color: lane_accent, accent_color: lane_accent,
playback_time, playback_time,
kind: lane.kind,
}); });
} }
} }
@ -4054,7 +4052,6 @@ impl TimelinePane {
value_max: lane.value_max, value_max: lane.value_max,
accent_color: lane_accent, accent_color: lane_accent,
playback_time, playback_time,
kind: lane.kind,
}); });
} }
} }
@ -5409,12 +5406,10 @@ impl PaneRenderer for TimelinePane {
let editing_clip_id = shared.editing_clip_id; let editing_clip_id = shared.editing_clip_id;
let mut context_layers = document.context_layers(editing_clip_id.as_ref()); let mut context_layers = document.context_layers(editing_clip_id.as_ref());
// Prepend master track as the first row when enabled (only at root context, not inside clips) // Prepend master track as the first row when enabled (only at root context, not inside clips)
let master_any_layer; let _master_any_layer;
if self.show_master_track && editing_clip_id.is_none() { if self.show_master_track && editing_clip_id.is_none() {
master_any_layer = Some(AnyLayer::Group(document.master_layer.clone())); _master_any_layer = Some(AnyLayer::Group(document.master_layer.clone()));
context_layers.insert(0, master_any_layer.as_ref().unwrap()); context_layers.insert(0, _master_any_layer.as_ref().unwrap());
} else {
master_any_layer = None;
} }
// Use virtual row count (includes expanded group children) for height calculations // Use virtual row count (includes expanded group children) for height calculations
let layer_count = build_timeline_rows(&context_layers).len(); let layer_count = build_timeline_rows(&context_layers).len();

425
lightningbeam-ui/lightningbeam-editor/src/raster_tool.rs
View File

@ -12,7 +12,6 @@
//! allocates and validates them in [`begin_raster_workspace`]; tools only //! allocates and validates them in [`begin_raster_workspace`]; tools only
//! dispatch shaders. //! dispatch shaders.
use std::sync::Arc;
use uuid::Uuid; use uuid::Uuid;
use eframe::egui; use eframe::egui;
@ -25,11 +24,6 @@ pub enum WorkspaceSource {
Layer { Layer {
layer_id: Uuid, layer_id: Uuid,
time: f64, time: f64,
/// The keyframe's own UUID (the A-canvas key in `GpuBrushEngine`).
kf_id: Uuid,
/// Full canvas dimensions (may differ from workspace dims for floating selections).
canvas_w: u32,
canvas_h: u32,
}, },
/// Operating on the floating selection. /// Operating on the floating selection.
Float, Float,
@ -51,9 +45,6 @@ pub struct RasterWorkspace {
pub b_canvas_id: Uuid, pub b_canvas_id: Uuid,
/// C canvas (Rgba16Float) — scratch; tools accumulate dabs here across the stroke. /// C canvas (Rgba16Float) — scratch; tools accumulate dabs here across the stroke.
pub c_canvas_id: Uuid, pub c_canvas_id: Uuid,
/// Optional R8Unorm selection mask (same pixel dimensions as A/B/C).
/// `None` means the entire workspace is selected.
pub mask_texture: Option<Arc<wgpu::Texture>>,
/// Pixel dimensions. A, B, C, and mask are all guaranteed to be this size. /// Pixel dimensions. A, B, C, and mask are all guaranteed to be this size.
pub width: u32, pub width: u32,
pub height: u32, pub height: u32,
@ -69,40 +60,6 @@ pub struct RasterWorkspace {
} }
impl RasterWorkspace { impl RasterWorkspace {
/// Panic-safe bounds check. Asserts that every GPU canvas exists and has
/// the dimensions declared by this workspace. Called by the framework
/// before `begin()` and before each `update()`.
pub fn validate(&self, gpu: &crate::gpu_brush::GpuBrushEngine) {
for (name, id) in [
("A", self.a_canvas_id),
("B", self.b_canvas_id),
("C", self.c_canvas_id),
] {
let canvas = gpu.canvases.get(&id).unwrap_or_else(|| {
panic!(
"RasterWorkspace::validate: buffer '{}' (id={}) not found in GpuBrushEngine",
name, id
)
});
assert_eq!(
canvas.width, self.width,
"RasterWorkspace::validate: buffer '{}' width {} != workspace width {}",
name, canvas.width, self.width
);
assert_eq!(
canvas.height, self.height,
"RasterWorkspace::validate: buffer '{}' height {} != workspace height {}",
name, canvas.height, self.height
);
}
let expected = (self.width * self.height * 4) as usize;
assert_eq!(
self.before_pixels.len(), expected,
"RasterWorkspace::validate: before_pixels.len()={} != expected {}",
self.before_pixels.len(), expected
);
}
/// Returns the three canvas UUIDs as an array (convenient for bulk removal). /// Returns the three canvas UUIDs as an array (convenient for bulk removal).
pub fn canvas_ids(&self) -> [Uuid; 3] { pub fn canvas_ids(&self) -> [Uuid; 3] {
[self.a_canvas_id, self.b_canvas_id, self.c_canvas_id] [self.a_canvas_id, self.b_canvas_id, self.c_canvas_id]
@ -208,11 +165,6 @@ pub trait RasterTool: Send + Sync {
/// the operation was a no-op (e.g. the pointer never moved). /// the operation was a no-op (e.g. the pointer never moved).
fn finish(&mut self, ws: &RasterWorkspace) -> bool; fn finish(&mut self, ws: &RasterWorkspace) -> bool;
/// Called on **Escape** or tool switch mid-stroke. The caller restores the
/// source pixels from `ws.before_pixels` without creating an undo entry; the
/// tool just cleans up internal state.
fn cancel(&mut self, ws: &RasterWorkspace);
/// Called once per frame (in the VelloCallback construction, UI thread) to /// Called once per frame (in the VelloCallback construction, UI thread) to
/// extract pending GPU work accumulated by `begin()` / `update()`. /// extract pending GPU work accumulated by `begin()` / `update()`.
/// ///
@ -377,384 +329,7 @@ impl RasterTool for BrushRasterTool {
self.has_dabs self.has_dabs
} }
fn cancel(&mut self, _ws: &RasterWorkspace) {
self.pending = None;
self.has_dabs = false;
}
fn take_pending_gpu_work(&mut self) -> Option<Box<dyn PendingGpuWork>> { fn take_pending_gpu_work(&mut self) -> Option<Box<dyn PendingGpuWork>> {
self.pending.take().map(|w| w as Box<dyn PendingGpuWork>) self.pending.take().map(|w| w as Box<dyn PendingGpuWork>)
} }
} }
// ── EffectBrushTool ───────────────────────────────────────────────────────────
/// Raster tool for effect brushes (Blur, Sharpen, Dodge, Burn, Sponge, Desaturate).
///
/// C accumulates a per-pixel influence weight (R channel, 0–255).
/// The composite pass applies the effect to A, scaled by C.r, writing to B:
/// `B = lerp(A, effect(A), C.r)`
///
/// Using C as an influence map (rather than accumulating modified pixels) prevents
/// overlapping dabs from compounding the effect beyond the C.r cap (255).
///
/// # GPU implementation (TODO)
/// Requires a dedicated `effect_brush_composite.wgsl` shader that reads A and C,
/// applies the blend-mode-specific filter to A, and blends by C.r → B.
pub struct EffectBrushTool {
brush: BrushSettings,
blend_mode: RasterBlendMode,
has_dabs: bool,
}
impl EffectBrushTool {
pub fn new(brush: BrushSettings, blend_mode: RasterBlendMode) -> Self {
Self { brush, blend_mode, has_dabs: false }
}
}
impl RasterTool for EffectBrushTool {
fn begin(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {}
fn update(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
self.has_dabs = true; // placeholder
}
fn finish(&mut self, _ws: &RasterWorkspace) -> bool { self.has_dabs }
fn cancel(&mut self, _ws: &RasterWorkspace) { self.has_dabs = false; }
// GPU shaders not yet implemented; take_pending_gpu_work returns None (default).
}
// ── SmudgeTool ────────────────────────────────────────────────────────────────
/// Raster tool for the smudge brush.
///
/// `begin()`: copy A → C so C starts with the source pixels for color pickup.
/// `update()`: dispatch smudge dabs using `blend_mode=2` (reads C as source,
/// writes smear to C); then composite C over A → B.
/// Because the smudge shader reads from `canvas_src` (C.src) and writes to
/// `canvas_dst` (C.dst), existing dabs are preserved in the smear history.
///
/// # GPU implementation (TODO)
/// Requires an initial A → C copy in `begin()` (via GPU copy command).
/// The smudge dab dispatch then uses `render_dabs(c_id, smudge_dabs, ...)`.
/// The composite pass is `composite_a_c_to_b` (same as BrushRasterTool).
pub struct SmudgeTool {
brush: BrushSettings,
has_dabs: bool,
}
impl SmudgeTool {
pub fn new(brush: BrushSettings) -> Self {
Self { brush, has_dabs: false }
}
}
impl RasterTool for SmudgeTool {
fn begin(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {}
fn update(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
self.has_dabs = true; // placeholder
}
fn finish(&mut self, _ws: &RasterWorkspace) -> bool { self.has_dabs }
fn cancel(&mut self, _ws: &RasterWorkspace) { self.has_dabs = false; }
// GPU shaders not yet implemented; take_pending_gpu_work returns None (default).
}
// ── GradientRasterTool ────────────────────────────────────────────────────────
use crate::gpu_brush::GpuGradientStop;
use lightningbeam_core::gradient::{GradientExtend, GradientType, ShapeGradient};
fn gradient_stops_to_gpu(gradient: &ShapeGradient) -> Vec<GpuGradientStop> {
gradient.stops.iter().map(|s| {
GpuGradientStop::from_srgb_u8(s.position, s.color.r, s.color.g, s.color.b, s.color.a)
}).collect()
}
fn gradient_extend_to_u32(extend: GradientExtend) -> u32 {
match extend {
GradientExtend::Pad => 0,
GradientExtend::Reflect => 1,
GradientExtend::Repeat => 2,
}
}
fn gradient_kind_to_u32(kind: GradientType) -> u32 {
match kind {
GradientType::Linear => 0,
GradientType::Radial => 1,
}
}
struct PendingGradientWork {
a_id: Uuid,
b_id: Uuid,
stops: Vec<GpuGradientStop>,
start: (f32, f32),
end: (f32, f32),
opacity: f32,
extend_mode: u32,
kind: u32,
}
impl PendingGpuWork for PendingGradientWork {
fn execute(&self, device: &wgpu::Device, queue: &wgpu::Queue, gpu: &mut crate::gpu_brush::GpuBrushEngine) {
gpu.apply_gradient_fill(
device, queue,
&self.a_id, &self.b_id,
&self.stops,
self.start, self.end,
self.opacity, self.extend_mode, self.kind,
);
}
}
/// Raster tool for gradient fills.
///
/// `begin()` records the canvas-local start position.
/// `update()` recomputes gradient parameters from settings and queues a
/// `PendingGradientWork` that calls `apply_gradient_fill` in `prepare()`.
/// `finish()` returns whether any gradient was dispatched.
pub struct GradientRasterTool {
start_canvas: egui::Vec2,
end_canvas: egui::Vec2,
pending: Option<Box<PendingGradientWork>>,
has_dispatched: bool,
}
impl GradientRasterTool {
pub fn new() -> Self {
Self {
start_canvas: egui::Vec2::ZERO,
end_canvas: egui::Vec2::ZERO,
pending: None,
has_dispatched: false,
}
}
}
impl RasterTool for GradientRasterTool {
fn begin(&mut self, ws: &RasterWorkspace, pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
let canvas_pos = pos - egui::vec2(ws.x as f32, ws.y as f32);
self.start_canvas = canvas_pos;
self.end_canvas = canvas_pos;
}
fn update(&mut self, ws: &RasterWorkspace, pos: egui::Vec2, _dt: f32, settings: &crate::tools::RasterToolSettings) {
self.end_canvas = pos - egui::vec2(ws.x as f32, ws.y as f32);
let gradient = &settings.gradient;
self.pending = Some(Box::new(PendingGradientWork {
a_id: ws.a_canvas_id,
b_id: ws.b_canvas_id,
stops: gradient_stops_to_gpu(gradient),
start: (self.start_canvas.x, self.start_canvas.y),
end: (self.end_canvas.x, self.end_canvas.y),
opacity: settings.gradient_opacity,
extend_mode: gradient_extend_to_u32(gradient.extend),
kind: gradient_kind_to_u32(gradient.kind),
}));
self.has_dispatched = true;
}
fn finish(&mut self, _ws: &RasterWorkspace) -> bool { self.has_dispatched }
fn cancel(&mut self, _ws: &RasterWorkspace) {
self.pending = None;
self.has_dispatched = false;
}
fn take_pending_gpu_work(&mut self) -> Option<Box<dyn PendingGpuWork>> {
self.pending.take().map(|w| w as Box<dyn PendingGpuWork>)
}
}
// ── TransformRasterTool ───────────────────────────────────────────────────────
use crate::gpu_brush::RasterTransformGpuParams;
struct PendingTransformWork {
a_id: Uuid,
b_id: Uuid,
params: RasterTransformGpuParams,
}
impl PendingGpuWork for PendingTransformWork {
fn execute(&self, device: &wgpu::Device, queue: &wgpu::Queue, gpu: &mut crate::gpu_brush::GpuBrushEngine) {
gpu.render_transform(device, queue, &self.a_id, &self.b_id, self.params);
}
}
/// Raster tool for affine transforms (move, scale, rotate, shear).
///
/// `begin()` stores the initial canvas dimensions and queues an identity
/// transform so B is initialised on the first frame.
/// `update()` recomputes the inverse affine matrix from the current handle
/// positions and queues a new `PendingTransformWork`.
///
/// The inverse matrix maps output pixel coordinates back to source pixel
/// coordinates: `src = M_inv * dst + b`
/// where `M_inv = [[a00, a01], [a10, a11]]` and `b = [b0, b1]`.
///
/// # GPU implementation
/// Fully wired — uses `GpuBrushEngine::render_transform`. Handle interaction
/// logic (drag, rotate, scale) is handled by the tool's `update()` caller in
/// `stage.rs` which computes and passes in the `RasterTransformGpuParams`.
pub struct TransformRasterTool {
pending: Option<Box<PendingTransformWork>>,
has_dispatched: bool,
canvas_w: u32,
canvas_h: u32,
}
impl TransformRasterTool {
pub fn new() -> Self {
Self {
pending: None,
has_dispatched: false,
canvas_w: 0,
canvas_h: 0,
}
}
/// Queue a transform with the given inverse-affine matrix.
/// Called by the stage handler after computing handle positions.
pub fn set_transform(
&mut self,
ws: &RasterWorkspace,
params: RasterTransformGpuParams,
) {
self.pending = Some(Box::new(PendingTransformWork {
a_id: ws.a_canvas_id,
b_id: ws.b_canvas_id,
params,
}));
self.has_dispatched = true;
}
}
impl RasterTool for TransformRasterTool {
fn begin(&mut self, ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
self.canvas_w = ws.width;
self.canvas_h = ws.height;
// Queue identity transform so B shows the source immediately.
let identity = RasterTransformGpuParams {
a00: 1.0, a01: 0.0,
a10: 0.0, a11: 1.0,
b0: 0.0, b1: 0.0,
src_w: ws.width, src_h: ws.height,
dst_w: ws.width, dst_h: ws.height,
_pad0: 0, _pad1: 0,
};
self.set_transform(ws, identity);
}
fn update(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
// Handle interaction and matrix updates are driven from stage.rs via set_transform().
}
fn finish(&mut self, _ws: &RasterWorkspace) -> bool { self.has_dispatched }
fn cancel(&mut self, _ws: &RasterWorkspace) {
self.pending = None;
self.has_dispatched = false;
}
fn take_pending_gpu_work(&mut self) -> Option<Box<dyn PendingGpuWork>> {
self.pending.take().map(|w| w as Box<dyn PendingGpuWork>)
}
}
// ── WarpRasterTool ────────────────────────────────────────────────────────────
/// Raster tool for warp / mesh deformation.
///
/// Uses a displacement buffer (managed by `GpuBrushEngine`) that maps each
/// output pixel to a source offset. The displacement grid is updated by
/// dragging control points; the warp shader reads anchor pixels + displacement
/// → B each frame.
///
/// # GPU implementation (TODO)
/// Requires: `create_displacement_buf`, `apply_warp` already exist in
/// `GpuBrushEngine`. Wire brush-drag interaction to update displacement
/// entries and call `apply_warp`.
pub struct WarpRasterTool {
has_dispatched: bool,
}
impl WarpRasterTool {
pub fn new() -> Self { Self { has_dispatched: false } }
}
impl RasterTool for WarpRasterTool {
fn begin(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {}
fn update(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
self.has_dispatched = true; // placeholder
}
fn finish(&mut self, _ws: &RasterWorkspace) -> bool { self.has_dispatched }
fn cancel(&mut self, _ws: &RasterWorkspace) { self.has_dispatched = false; }
// take_pending_gpu_work: default (None) — full GPU wiring is TODO.
}
// ── LiquifyRasterTool ─────────────────────────────────────────────────────────
/// Raster tool for liquify (per-pixel displacement painting).
///
/// Similar to `WarpRasterTool` but uses a full per-pixel displacement map
/// (grid_cols = grid_rows = 0 in `apply_warp`) painted by brush strokes.
/// Each dab accumulates displacement in the push/pull/swirl direction.
///
/// # GPU implementation (TODO)
/// Requires: a dab-to-displacement shader that accumulates per-pixel offsets
/// into the displacement buffer, then `apply_warp` reads it → B.
pub struct LiquifyRasterTool {
has_dispatched: bool,
}
impl LiquifyRasterTool {
pub fn new() -> Self { Self { has_dispatched: false } }
}
impl RasterTool for LiquifyRasterTool {
fn begin(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {}
fn update(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
self.has_dispatched = true; // placeholder
}
fn finish(&mut self, _ws: &RasterWorkspace) -> bool { self.has_dispatched }
fn cancel(&mut self, _ws: &RasterWorkspace) { self.has_dispatched = false; }
// take_pending_gpu_work: default (None) — full GPU wiring is TODO.
}
// ── SelectionTool ─────────────────────────────────────────────────────────────
/// Raster selection tool (Magic Wand / Quick Select).
///
/// C (RGBA8) acts as the growing selection; C.r = mask value (0 or 255).
/// Each `update()` frame a flood-fill / region-grow shader extends C.r.
/// The composite pass draws A + a tinted overlay from C.r → B so the user
/// sees the growing selection boundary.
///
/// `finish()` returns false (commit does not write pixels back to the layer;
/// instead the caller extracts C.r into the standalone `R8Unorm` selection
/// texture via `shared.raster_selection`).
///
/// # GPU implementation (TODO)
/// Requires: a flood-fill compute shader seeded by the click position that
/// grows the selection in C.r; and a composite shader that tints selected
/// pixels blue/cyan for preview.
pub struct SelectionTool {
has_selection: bool,
}
impl SelectionTool {
pub fn new() -> Self { Self { has_selection: false } }
}
impl RasterTool for SelectionTool {
fn begin(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {}
fn update(&mut self, _ws: &RasterWorkspace, _pos: egui::Vec2, _dt: f32, _settings: &crate::tools::RasterToolSettings) {
self.has_selection = true; // placeholder
}
/// Selection tools never trigger a pixel readback/commit on mouseup.
/// The caller reads C.r directly into the selection mask texture.
fn finish(&mut self, _ws: &RasterWorkspace) -> bool { false }
fn cancel(&mut self, _ws: &RasterWorkspace) { self.has_selection = false; }
// take_pending_gpu_work: default (None) — full GPU wiring is TODO.
}

5
lightningbeam-ui/lightningbeam-editor/src/sample_import.rs
View File

@ -471,7 +471,10 @@ fn auto_key_ranges(notes: &[u8]) -> Vec<(u8, u8)> {
let min = if i == 0 { let min = if i == 0 {
0 0
} else { } else {
((notes[i - 1] as u16 + notes[i] as u16 + 1) / 2) as u8 // One past the previous note's midpoint boundary, so adjacent ranges
// don't both claim the midpoint key. The previous note's max is
// floor((notes[i-1] + notes[i]) / 2); start here at that + 1.
(((notes[i - 1] as u16 + notes[i] as u16) / 2) + 1) as u8
}; };
let max = if i == notes.len() - 1 { let max = if i == notes.len() - 1 {
127 127

9
lightningbeam-ui/lightningbeam-editor/src/test_mode.rs
View File

@ -232,15 +232,6 @@ impl TestModeState {
} }
} }
/// Store geometry context for panic capture.
/// Called before risky operations (e.g. region select) so the panic hook
/// can include it in the crash file for easier reproduction.
pub fn set_pending_geometry(&self, context: serde_json::Value) {
if let Ok(mut guard) = self.pending_geometry.try_lock() {
*guard = Some(context);
}
}
/// Clear the pending geometry context (call after the operation succeeds). /// Clear the pending geometry context (call after the operation succeeds).
pub fn clear_pending_geometry(&self) { pub fn clear_pending_geometry(&self) {
if let Ok(mut guard) = self.pending_geometry.try_lock() { if let Ok(mut guard) = self.pending_geometry.try_lock() {

5
lightningbeam-ui/lightningbeam-editor/src/theme.rs
View File

@ -42,6 +42,7 @@ impl ThemeMode {
/// Background type for CSS backgrounds /// Background type for CSS backgrounds
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
#[allow(dead_code)]
pub enum Background { pub enum Background {
Solid(egui::Color32), Solid(egui::Color32),
LinearGradient { LinearGradient {
@ -456,6 +457,7 @@ impl Theme {
} }
/// Invalidate the cache (call on stylesheet reload or mode change) /// Invalidate the cache (call on stylesheet reload or mode change)
#[allow(dead_code)]
pub fn invalidate_cache(&self) { pub fn invalidate_cache(&self) {
self.cache.borrow_mut().clear(); self.cache.borrow_mut().clear();
} }
@ -518,6 +520,7 @@ impl Theme {
} }
/// Convenience: resolve and extract a dimension with fallback /// Convenience: resolve and extract a dimension with fallback
#[allow(dead_code)]
pub fn dimension(&self, context: &[&str], ctx: &egui::Context, property: &str, fallback: f32) -> f32 { pub fn dimension(&self, context: &[&str], ctx: &egui::Context, property: &str, fallback: f32) -> f32 {
let style = self.resolve(context, ctx); let style = self.resolve(context, ctx);
match property { match property {
@ -534,6 +537,7 @@ impl Theme {
} }
/// Paint background for a region (handles solid/gradient/image) /// Paint background for a region (handles solid/gradient/image)
#[allow(dead_code)]
pub fn paint_bg( pub fn paint_bg(
&self, &self,
context: &[&str], context: &[&str],
@ -893,6 +897,7 @@ mod tests {
} }
#[test] #[test]
#[ignore = "WIP theme system: CSS var() custom-property resolution not yet implemented (theme.rs is kept under #[allow(dead_code)] and not wired up)"]
fn test_cascade_resolve() { fn test_cascade_resolve() {
let css = r#" let css = r#"
:root { --bg: #ff0000; } :root { --bg: #ff0000; }

2
lightningbeam-ui/lightningbeam-editor/src/theme_render.rs
View File

@ -7,6 +7,7 @@ use eframe::egui;
use crate::theme::Background; use crate::theme::Background;
/// Paint a background into the given rect /// Paint a background into the given rect
#[allow(dead_code)]
pub fn paint_background( pub fn paint_background(
painter: &egui::Painter, painter: &egui::Painter,
rect: egui::Rect, rect: egui::Rect,
@ -35,6 +36,7 @@ pub fn paint_background(
/// - 90deg = left to right /// - 90deg = left to right
/// - 180deg = top to bottom (default) /// - 180deg = top to bottom (default)
/// - 270deg = right to left /// - 270deg = right to left
#[allow(dead_code)]
pub fn paint_linear_gradient( pub fn paint_linear_gradient(
painter: &egui::Painter, painter: &egui::Painter,
rect: egui::Rect, rect: egui::Rect,

5
package.json
View File

@ -4,12 +4,10 @@
"version": "0.1.0", "version": "0.1.0",
"type": "module", "type": "module",
"scripts": { "scripts": {
"tauri": "tauri",
"test": "wdio run wdio.conf.js", "test": "wdio run wdio.conf.js",
"test:watch": "wdio run wdio.conf.js --watch" "test:watch": "wdio run wdio.conf.js --watch"
}, },
"devDependencies": { "devDependencies": {
"@tauri-apps/cli": "^2",
"@wdio/cli": "^9.20.0", "@wdio/cli": "^9.20.0",
"@wdio/globals": "^9.17.0", "@wdio/globals": "^9.17.0",
"@wdio/local-runner": "8", "@wdio/local-runner": "8",
@ -19,9 +17,6 @@
}, },
"dependencies": { "dependencies": {
"@ffmpeg/ffmpeg": "^0.12.10", "@ffmpeg/ffmpeg": "^0.12.10",
"@tauri-apps/plugin-dialog": "~2",
"@tauri-apps/plugin-fs": "~2",
"@tauri-apps/plugin-log": "^2.2.0",
"ffmpeg": "^0.0.4", "ffmpeg": "^0.0.4",
"ffmpeg.js": "^4.2.9003" "ffmpeg.js": "^4.2.9003"
} }

5
packaging/Containerfile
View File

@ -39,6 +39,11 @@ RUN apt-get update && apt-get install -y \
libvpx-dev \ libvpx-dev \
libmp3lame-dev \ libmp3lame-dev \
libopus-dev \ libopus-dev \
# VAAPI hardware H.264 encode (zero-copy export). FFmpeg autodetects --enable-vaapi
# only when these headers are present at configure time; without them the static
# ffmpeg has no h264_vaapi encoder and the editor silently uses the software path.
libva-dev \
libdrm-dev \
# PulseAudio (cpal optional backend) # PulseAudio (cpal optional backend)
libpulse-dev \ libpulse-dev \
# Packaging tools # Packaging tools

4
packaging/Makefile
View File

@ -20,9 +20,7 @@ pkgbuild: setup-icons
setup-icons: setup-icons:
@echo "==> Setting up icons..." @echo "==> Setting up icons..."
@mkdir -p $(EDITOR)/assets/icons @mkdir -p $(EDITOR)/assets/icons
@cp -u $(REPO_ROOT)/src-tauri/icons/32x32.png $(EDITOR)/assets/icons/ 2>/dev/null || true @# Icons are committed under $(EDITOR)/assets/icons (32x32, 128x128, 256x256, icon.icns).
@cp -u $(REPO_ROOT)/src-tauri/icons/128x128.png $(EDITOR)/assets/icons/ 2>/dev/null || true
@cp -u $(REPO_ROOT)/src-tauri/icons/icon.png $(EDITOR)/assets/icons/256x256.png 2>/dev/null || true
@echo " Icons ready" @echo " Icons ready"
clean: clean:

4
scripts/release.sh
View File

@ -34,4 +34,6 @@ vim "$CHANGELOG"
# Commit and push # Commit and push
git add "$CARGO_TOML" "$CHANGELOG" git add "$CARGO_TOML" "$CHANGELOG"
git commit -m "Release v${new_version}" git commit -m "Release v${new_version}"
git push --force origin "$(git branch --show-current):release" # Push to the 'all' remote so the release branch lands on both GitHub and Gitea.
# CI (GitHub Actions) still triggers via the GitHub pushurl.
git push --force all "$(git branch --show-current):release"

6708
src-tauri/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

63
src-tauri/Cargo.toml
View File

@ -1,63 +0,0 @@
[package]
name = "lightningbeam"
version = "0.1.0"
description = "A Tauri App"
authors = ["you"]
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
# The `_lib` suffix may seem redundant but it is necessary
# to make the lib name unique and wouldn't conflict with the bin name.
# This seems to be only an issue on Windows, see https://github.com/rust-lang/cargo/issues/8519
name = "lightningbeam_lib"
crate-type = ["staticlib", "cdylib", "rlib"]
[build-dependencies]
tauri-build = { version = "2", features = [] }
[dependencies]
tauri = { version = "2", features = ["protocol-asset"] }
tauri-plugin-shell = "2"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tauri-plugin-fs = "2"
tauri-plugin-dialog = "2"
log = "0.4"
env_logger = "0.11"
# DAW backend integration
daw-backend = { path = "../daw-backend" }
cpal = "0.15"
rtrb = "0.3"
tokio = { version = "1", features = ["sync", "time"] }
# Video decoding
lru = "0.12"
# WebSocket for frame streaming (disable default features to remove tracing, but keep handshake)
tungstenite = { version = "0.20", default-features = false, features = ["handshake"] }
# Native rendering with wgpu
wgpu = "0.19"
winit = "0.29"
pollster = "0.3"
bytemuck = { version = "1.14", features = ["derive"] }
raw-window-handle = "0.6"
image = "0.24"
[target.'cfg(target_os = "macos")'.dependencies]
ffmpeg-next = { version = "8.0", features = ["build"] }
[target.'cfg(not(target_os = "macos"))'.dependencies]
ffmpeg-next = "8.0"
[profile.dev]
opt-level = 1 # Enable basic optimizations in debug mode for audio decoding performance
[profile.release]
opt-level = 3
lto = true

98
src-tauri/assets/factory_presets/Basic_Sine.json
View File

@ -1,98 +0,0 @@
{
"metadata": {
"name": "Basic Sine",
"description": "Simple sine wave synthesizer with ADSR envelope. Great for learning the basics of subtractive synthesis.",
"author": "Lightningbeam",
"version": 1,
"tags": ["basic", "lead", "mono"]
},
"nodes": [
{
"id": 0,
"node_type": "MidiInput",
"parameters": {},
"position": [100.0, 200.0]
},
{
"id": 1,
"node_type": "MidiToCV",
"parameters": {},
"position": [300.0, 200.0]
},
{
"id": 2,
"node_type": "Oscillator",
"parameters": {
"0": 440.0,
"1": 0.7,
"2": 0.0
},
"position": [500.0, 150.0]
},
{
"id": 3,
"node_type": "ADSR",
"parameters": {
"0": 0.01,
"1": 0.1,
"2": 0.7,
"3": 0.3
},
"position": [500.0, 300.0]
},
{
"id": 4,
"node_type": "Gain",
"parameters": {
"0": 1.0
},
"position": [700.0, 200.0]
},
{
"id": 5,
"node_type": "AudioOutput",
"parameters": {},
"position": [900.0, 200.0]
}
],
"connections": [
{
"from_node": 0,
"from_port": 0,
"to_node": 1,
"to_port": 0
},
{
"from_node": 1,
"from_port": 0,
"to_node": 2,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 3,
"to_port": 0
},
{
"from_node": 2,
"from_port": 0,
"to_node": 4,
"to_port": 0
},
{
"from_node": 3,
"from_port": 0,
"to_node": 4,
"to_port": 1
},
{
"from_node": 4,
"from_port": 0,
"to_node": 5,
"to_port": 0
}
],
"midi_targets": [0],
"output_node": 5
}

137
src-tauri/assets/factory_presets/Pluck.json
View File

@ -1,137 +0,0 @@
{
"metadata": {
"name": "Pluck",
"description": "Percussive pluck sound with fast attack and decay. Great for arpeggios, melodies, and rhythmic patterns.",
"author": "Lightningbeam",
"version": 1,
"tags": ["pluck", "lead", "percussive", "arpeggio"]
},
"nodes": [
{
"id": 0,
"node_type": "MidiInput",
"parameters": {},
"position": [100.0, 250.0]
},
{
"id": 1,
"node_type": "MidiToCV",
"parameters": {},
"position": [300.0, 250.0]
},
{
"id": 2,
"node_type": "Oscillator",
"parameters": {
"0": 440.0,
"1": 0.6,
"2": 2.0
},
"position": [500.0, 150.0]
},
{
"id": 3,
"node_type": "Filter",
"parameters": {
"0": 2000.0,
"1": 0.8,
"2": 0.0
},
"position": [700.0, 150.0]
},
{
"id": 4,
"node_type": "ADSR",
"parameters": {
"0": 0.001,
"1": 0.3,
"2": 0.0,
"3": 0.05
},
"position": [500.0, 350.0]
},
{
"id": 5,
"node_type": "ADSR",
"parameters": {
"0": 0.001,
"1": 0.4,
"2": 0.0,
"3": 0.1
},
"position": [700.0, 350.0]
},
{
"id": 6,
"node_type": "Gain",
"parameters": {
"0": 1.0
},
"position": [900.0, 200.0]
},
{
"id": 7,
"node_type": "AudioOutput",
"parameters": {},
"position": [1100.0, 200.0]
}
],
"connections": [
{
"from_node": 0,
"from_port": 0,
"to_node": 1,
"to_port": 0
},
{
"from_node": 1,
"from_port": 0,
"to_node": 2,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 4,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 5,
"to_port": 0
},
{
"from_node": 2,
"from_port": 0,
"to_node": 3,
"to_port": 0
},
{
"from_node": 4,
"from_port": 0,
"to_node": 3,
"to_port": 1
},
{
"from_node": 3,
"from_port": 0,
"to_node": 6,
"to_port": 0
},
{
"from_node": 5,
"from_port": 0,
"to_node": 6,
"to_port": 1
},
{
"from_node": 6,
"from_port": 0,
"to_node": 7,
"to_port": 0
}
],
"midi_targets": [0],
"output_node": 7
}

145
src-tauri/assets/factory_presets/Poly_Synth.json
View File

@ -1,145 +0,0 @@
{
"metadata": {
"name": "Poly Synth",
"description": "8-voice polyphonic synthesizer with sawtooth oscillator and ADSR envelope. Perfect for chords and complex harmonies.",
"author": "Lightningbeam",
"version": 1,
"tags": ["poly", "polyphonic", "synth", "chords"]
},
"nodes": [
{
"id": 0,
"node_type": "MidiInput",
"parameters": {},
"position": [100.0, 200.0]
},
{
"id": 1,
"node_type": "VoiceAllocator",
"parameters": {
"0": 8.0
},
"position": [400.0, 200.0],
"template_graph": {
"metadata": {
"name": "template",
"description": "",
"author": "",
"version": 1,
"tags": []
},
"nodes": [
{
"id": 0,
"node_type": "TemplateInput",
"parameters": {},
"position": [100.0, 200.0]
},
{
"id": 1,
"node_type": "MidiToCV",
"parameters": {},
"position": [300.0, 200.0]
},
{
"id": 2,
"node_type": "Oscillator",
"parameters": {
"0": 440.0,
"1": 0.7,
"2": 1.0
},
"position": [500.0, 150.0]
},
{
"id": 3,
"node_type": "ADSR",
"parameters": {
"0": 0.01,
"1": 0.2,
"2": 0.6,
"3": 0.3
},
"position": [500.0, 300.0]
},
{
"id": 4,
"node_type": "Gain",
"parameters": {
"0": 1.0
},
"position": [700.0, 200.0]
},
{
"id": 5,
"node_type": "TemplateOutput",
"parameters": {},
"position": [900.0, 200.0]
}
],
"connections": [
{
"from_node": 0,
"from_port": 0,
"to_node": 1,
"to_port": 0
},
{
"from_node": 1,
"from_port": 0,
"to_node": 2,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 3,
"to_port": 0
},
{
"from_node": 2,
"from_port": 0,
"to_node": 4,
"to_port": 0
},
{
"from_node": 3,
"from_port": 0,
"to_node": 4,
"to_port": 1
},
{
"from_node": 4,
"from_port": 0,
"to_node": 5,
"to_port": 0
}
],
"midi_targets": [],
"output_node": 5
}
},
{
"id": 2,
"node_type": "AudioOutput",
"parameters": {},
"position": [700.0, 200.0]
}
],
"connections": [
{
"from_node": 0,
"from_port": 0,
"to_node": 1,
"to_port": 0
},
{
"from_node": 1,
"from_port": 0,
"to_node": 2,
"to_port": 0
}
],
"midi_targets": [0],
"output_node": 2
}

137
src-tauri/assets/factory_presets/Sawtooth_Bass.json
View File

@ -1,137 +0,0 @@
{
"metadata": {
"name": "Sawtooth Bass",
"description": "Classic analog-style bass synth with sawtooth oscillator and resonant lowpass filter. Perfect for electronic music basslines.",
"author": "Lightningbeam",
"version": 1,
"tags": ["bass", "analog", "electronic", "mono"]
},
"nodes": [
{
"id": 0,
"node_type": "MidiInput",
"parameters": {},
"position": [100.0, 250.0]
},
{
"id": 1,
"node_type": "MidiToCV",
"parameters": {},
"position": [300.0, 250.0]
},
{
"id": 2,
"node_type": "Oscillator",
"parameters": {
"0": 110.0,
"1": 0.8,
"2": 1.0
},
"position": [500.0, 150.0]
},
{
"id": 3,
"node_type": "Filter",
"parameters": {
"0": 800.0,
"1": 2.5,
"2": 0.0
},
"position": [700.0, 150.0]
},
{
"id": 4,
"node_type": "ADSR",
"parameters": {
"0": 0.005,
"1": 0.2,
"2": 0.3,
"3": 0.1
},
"position": [500.0, 300.0]
},
{
"id": 5,
"node_type": "ADSR",
"parameters": {
"0": 0.005,
"1": 0.15,
"2": 0.6,
"3": 0.2
},
"position": [700.0, 350.0]
},
{
"id": 6,
"node_type": "Gain",
"parameters": {
"0": 1.2
},
"position": [900.0, 200.0]
},
{
"id": 7,
"node_type": "AudioOutput",
"parameters": {},
"position": [1100.0, 200.0]
}
],
"connections": [
{
"from_node": 0,
"from_port": 0,
"to_node": 1,
"to_port": 0
},
{
"from_node": 1,
"from_port": 0,
"to_node": 2,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 4,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 5,
"to_port": 0
},
{
"from_node": 2,
"from_port": 0,
"to_node": 3,
"to_port": 0
},
{
"from_node": 4,
"from_port": 0,
"to_node": 3,
"to_port": 1
},
{
"from_node": 3,
"from_port": 0,
"to_node": 6,
"to_port": 0
},
{
"from_node": 5,
"from_port": 0,
"to_node": 6,
"to_port": 1
},
{
"from_node": 6,
"from_port": 0,
"to_node": 7,
"to_port": 0
}
],
"midi_targets": [0],
"output_node": 7
}

176
src-tauri/assets/factory_presets/Warm_Pad.json
View File

@ -1,176 +0,0 @@
{
"metadata": {
"name": "Warm Pad",
"description": "Lush pad sound combining sawtooth and triangle waves with slow filter sweep and gentle attack. Ideal for ambient and cinematic music.",
"author": "Lightningbeam",
"version": 1,
"tags": ["pad", "ambient", "warm", "cinematic"]
},
"nodes": [
{
"id": 0,
"node_type": "MidiInput",
"parameters": {},
"position": [100.0, 300.0]
},
{
"id": 1,
"node_type": "MidiToCV",
"parameters": {},
"position": [300.0, 300.0]
},
{
"id": 2,
"node_type": "Oscillator",
"parameters": {
"0": 440.0,
"1": 0.5,
"2": 1.0
},
"position": [500.0, 150.0]
},
{
"id": 3,
"node_type": "Oscillator",
"parameters": {
"0": 440.0,
"1": 0.4,
"2": 3.0
},
"position": [500.0, 250.0]
},
{
"id": 4,
"node_type": "Mixer",
"parameters": {
"0": 0.5,
"1": 0.5,
"2": 0.0,
"3": 0.0
},
"position": [700.0, 200.0]
},
{
"id": 5,
"node_type": "Filter",
"parameters": {
"0": 1200.0,
"1": 1.0,
"2": 0.0
},
"position": [900.0, 200.0]
},
{
"id": 6,
"node_type": "ADSR",
"parameters": {
"0": 0.8,
"1": 1.0,
"2": 0.6,
"3": 1.5
},
"position": [700.0, 400.0]
},
{
"id": 7,
"node_type": "ADSR",
"parameters": {
"0": 0.5,
"1": 0.5,
"2": 0.8,
"3": 1.0
},
"position": [900.0, 400.0]
},
{
"id": 8,
"node_type": "Gain",
"parameters": {
"0": 0.8
},
"position": [1100.0, 250.0]
},
{
"id": 9,
"node_type": "AudioOutput",
"parameters": {},
"position": [1300.0, 250.0]
}
],
"connections": [
{
"from_node": 0,
"from_port": 0,
"to_node": 1,
"to_port": 0
},
{
"from_node": 1,
"from_port": 0,
"to_node": 2,
"to_port": 0
},
{
"from_node": 1,
"from_port": 0,
"to_node": 3,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 6,
"to_port": 0
},
{
"from_node": 1,
"from_port": 1,
"to_node": 7,
"to_port": 0
},
{
"from_node": 2,
"from_port": 0,
"to_node": 4,
"to_port": 0
},
{
"from_node": 3,
"from_port": 0,
"to_node": 4,
"to_port": 1
},
{
"from_node": 4,
"from_port": 0,
"to_node": 5,
"to_port": 0
},
{
"from_node": 6,
"from_port": 0,
"to_node": 5,
"to_port": 1
},
{
"from_node": 5,
"from_port": 0,
"to_node": 8,
"to_port": 0
},
{
"from_node": 7,
"from_port": 0,
"to_node": 8,
"to_port": 1
},
{
"from_node": 8,
"from_port": 0,
"to_node": 9,
"to_port": 0
}
],
"midi_targets": [0],
"output_node": 9
}

3
src-tauri/build.rs
View File

@ -1,3 +0,0 @@
fn main() {
tauri_build::build()
}

77
src-tauri/capabilities/default.json
View File

@ -1,77 +0,0 @@
{
"$schema": "../gen/schemas/desktop-schema.json",
"identifier": "default",
"description": "Capability for the main window",
"windows": [
"main",
"window*"
],
"permissions": [
"core:default",
"core:window:allow-close",
"core:window:allow-set-title",
"shell:allow-open",
"fs:default",
{
"identifier": "fs:allow-exists",
"allow": [
{
"path": "$HOME/*"
},
{
"path": "$DOCUMENT/*"
},
{
"path": "$DOWNLOAD/*"
},
{
"path": "$DESKTOP/*"
},
{
"path": "**/*"
}
]
},
{
"identifier": "fs:allow-app-write-recursive",
"allow": [
{
"path": "$HOME/*"
},
{
"path": "$DOCUMENT/*"
},
{
"path": "$DOWNLOAD/*"
},
{
"path": "$DESKTOP/*"
},
{
"path": "**/*"
}
]
},
{
"identifier": "fs:allow-app-read-recursive",
"allow": [
{
"path": "$HOME/*"
},
{
"path": "$DOCUMENT/*"
},
{
"path": "$DOWNLOAD/*"
},
{
"path": "$DESKTOP/*"
},
{
"path": "**/*"
}
]
},
"dialog:default"
]
}

104
src-tauri/examples/video_inspect.rs
View File

@ -1,104 +0,0 @@
extern crate ffmpeg_next as ffmpeg;
use std::env;
fn main() {
ffmpeg::init().unwrap();
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("Usage: {} <video_file>", args[0]);
std::process::exit(1);
}
let path = &args[1];
let input = ffmpeg::format::input(path).expect("Failed to open video");
println!("=== VIDEO FILE INFORMATION ===");
println!("File: {}", path);
println!("Format: {}", input.format().name());
println!("Duration: {:.2}s", input.duration() as f64 / f64::from(ffmpeg::ffi::AV_TIME_BASE));
println!();
let video_stream = input.streams()
.best(ffmpeg::media::Type::Video)
.expect("No video stream found");
let stream_index = video_stream.index();
let time_base = f64::from(video_stream.time_base());
let duration = video_stream.duration() as f64 * time_base;
let fps = f64::from(video_stream.avg_frame_rate());
println!("=== VIDEO STREAM ===");
println!("Stream index: {}", stream_index);
println!("Time base: {} ({:.10})", video_stream.time_base(), time_base);
println!("Duration: {:.2}s", duration);
println!("FPS: {:.2}", fps);
println!("Frames: {}", video_stream.frames());
let context = ffmpeg::codec::context::Context::from_parameters(video_stream.parameters())
.expect("Failed to create context");
let decoder = context.decoder().video().expect("Failed to create decoder");
println!("Codec: {:?}", decoder.id());
println!("Resolution: {}x{}", decoder.width(), decoder.height());
println!("Pixel format: {:?}", decoder.format());
println!();
println!("=== SCANNING FRAMES ===");
println!("Timestamp (ts) | Time (s) | Key | Type");
println!("---------------|----------|-----|-----");
let mut input = ffmpeg::format::input(path).expect("Failed to reopen video");
let context = ffmpeg::codec::context::Context::from_parameters(
input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters()
).expect("Failed to create context");
let mut decoder = context.decoder().video().expect("Failed to create decoder");
let mut frame_count = 0;
let mut keyframe_count = 0;
for (stream, packet) in input.packets() {
if stream.index() == stream_index {
let packet_pts = packet.pts().unwrap_or(0);
let packet_time = packet_pts as f64 * time_base;
let is_key = packet.is_key();
if is_key {
keyframe_count += 1;
}
// Print first 50 packets and all keyframes
if frame_count < 50 || is_key {
println!("{:14} | {:8.2} | {:3} | {:?}",
packet_pts,
packet_time,
if is_key { "KEY" } else { " " },
if is_key { "I-frame" } else { "P/B-frame" }
);
}
decoder.send_packet(&packet).ok();
let mut frame = ffmpeg::util::frame::Video::empty();
while decoder.receive_frame(&mut frame).is_ok() {
frame_count += 1;
}
}
}
// Flush decoder
decoder.send_eof().ok();
let mut frame = ffmpeg::util::frame::Video::empty();
while decoder.receive_frame(&mut frame).is_ok() {
frame_count += 1;
}
println!();
println!("=== SUMMARY ===");
println!("Total frames decoded: {}", frame_count);
println!("Total keyframes: {}", keyframe_count);
if keyframe_count > 0 {
println!("Average keyframe interval: {:.2} frames", frame_count as f64 / keyframe_count as f64);
println!("Average keyframe interval: {:.2}s", duration / keyframe_count as f64);
}
}

BIN
src-tauri/icons/128x128.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 23 KiB

BIN
src-tauri/icons/128x128@2x.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 66 KiB

BIN
src-tauri/icons/32x32.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 2.8 KiB

BIN
src-tauri/icons/Square107x107Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 17 KiB

BIN
src-tauri/icons/Square142x142Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 27 KiB

BIN
src-tauri/icons/Square150x150Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 29 KiB

BIN
src-tauri/icons/Square284x284Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 78 KiB

BIN
src-tauri/icons/Square30x30Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 2.5 KiB

BIN
src-tauri/icons/Square310x310Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 89 KiB

BIN
src-tauri/icons/Square44x44Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 4.4 KiB

BIN
src-tauri/icons/Square71x71Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 9.0 KiB

BIN
src-tauri/icons/Square89x89Logo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 13 KiB

BIN
src-tauri/icons/StoreLogo.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 5.2 KiB

BIN
src-tauri/icons/icon.ico
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.1 KiB

Some files were not shown because too many files have changed in this diff Show More