From 7525a604f29a842bdf1fe22d3c4cc6fe421b5467 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Thu, 25 Jun 2026 22:44:14 -0400 Subject: [PATCH 01/31] video: reuse the swscale context + gate per-frame decode tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_frame rebuilt the RGBA swscale context on every decoded frame and printed [Video Timing] lines unconditionally. A stream's frames share one input format/size, so build the scaler once (keyed on format+dims, rebuilt only if they change) and reuse it; gate the per-frame traces behind LB_VIDEO_DEBUG. Cuts export wall time ~10% on a 1080p video clip (the scaler rebuild was the bulk of the per-frame "scale" cost; it's now ~0ms). SwsContext is !Send, so the cached scaler is wrapped in a SendScaler — sound because a VideoDecoder is only ever touched under the VideoManager mutex (same invariant as its decoder/input). --- .../lightningbeam-core/src/video.rs | 72 ++++++++++++++----- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 801132e..92cd745 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -97,6 +97,22 @@ pub struct VideoDecoder { decoder: Option, last_decoded_ts: i64, // Track the last decoded frame timestamp keyframe_positions: Vec, // Index of keyframe timestamps for fast seeking + /// Reused RGBA scaler, keyed by the input (format, width, height). Building an swscale + /// context is not free; a stream's frames share one format/size, so build it once. + scaler: Option<(ffmpeg::format::Pixel, u32, u32, SendScaler)>, +} + +/// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only +/// ever accessed under the `VideoManager` mutex — never concurrently — so moving it between +/// threads is sound. The decoder/input fields rely on the same invariant. +struct SendScaler(ffmpeg::software::scaling::context::Context); +unsafe impl Send for SendScaler {} + +/// Per-frame video decode tracing, gated behind `LB_VIDEO_DEBUG` (checked once). Off by +/// default — at export frame rates these prints are a lot of locked stderr writes. +fn video_debug() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("LB_VIDEO_DEBUG").is_ok()) } impl VideoDecoder { @@ -183,6 +199,7 @@ impl VideoDecoder { decoder: None, last_decoded_ts: -1, keyframe_positions, + scaler: None, }) } @@ -270,7 +287,9 @@ impl VideoDecoder { // Check cache if let Some(cached_frame) = self.frame_cache.get(&frame_ts) { - eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + if video_debug() { + eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + } return Ok(cached_frame.clone()); } @@ -292,8 +311,10 @@ impl VideoDecoder { let keyframe_seconds = keyframe_ts_stream as f64 * self.time_base; let keyframe_ts_av = (keyframe_seconds * 1_000_000.0) as i64; // AV_TIME_BASE = 1000000 - eprintln!("[Video Seek] Target: {} | Keyframe(stream): {} | Keyframe(AV): {} | Index size: {}", - frame_ts, keyframe_ts_stream, keyframe_ts_av, self.keyframe_positions.len()); + if video_debug() { + eprintln!("[Video Seek] Target: {} | Keyframe(stream): {} | Keyframe(AV): {} | Index size: {}", + frame_ts, keyframe_ts_stream, keyframe_ts_av, self.keyframe_positions.len()); + } // Reopen input (a fresh BlobReader for packed sources). let mut owned = self.source.open() @@ -306,7 +327,9 @@ impl VideoDecoder { input.seek(keyframe_ts_av, keyframe_ts_av..(keyframe_ts_av + 1)) .map_err(|e| format!("Seek failed: {}", e))?; - eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis()); + if video_debug() { + eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis()); + } let context_decoder = ffmpeg::codec::context::Context::from_parameters( input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() @@ -354,16 +377,27 @@ impl VideoDecoder { if is_better { let t_scale_start = Instant::now(); - // Convert to RGBA and scale to output size - let mut scaler = ffmpeg::software::scaling::context::Context::get( - frame.format(), - frame.width(), - frame.height(), - ffmpeg::format::Pixel::RGBA, - self.output_width, - self.output_height, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ).map_err(|e| e.to_string())?; + // Reuse the RGBA scaler across frames; rebuild only if the input + // format/size changes (it won't within a stream). + let need_new = match &self.scaler { + Some((fmt, w, h, _)) => { + *fmt != frame.format() || *w != frame.width() || *h != frame.height() + } + None => true, + }; + if need_new { + let ctx = ffmpeg::software::scaling::context::Context::get( + frame.format(), + frame.width(), + frame.height(), + ffmpeg::format::Pixel::RGBA, + self.output_width, + self.output_height, + ffmpeg::software::scaling::flag::Flags::BILINEAR, + ).map_err(|e| e.to_string())?; + self.scaler = Some((frame.format(), frame.width(), frame.height(), SendScaler(ctx))); + } + let scaler = &mut self.scaler.as_mut().unwrap().3.0; let mut rgb_frame = ffmpeg::util::frame::Video::empty(); scaler.run(&frame, &mut rgb_frame) @@ -392,10 +426,12 @@ impl VideoDecoder { if current_frame_ts >= frame_ts { // Found our frame, cache and return it if let Some(data) = best_frame_data { - let total_time = t_start.elapsed().as_millis(); - let decode_time = t_decode_start.elapsed().as_millis(); - eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", - timestamp, decode_count, decode_time, scale_time_ms, total_time); + if video_debug() { + let total_time = t_start.elapsed().as_millis(); + let decode_time = t_decode_start.elapsed().as_millis(); + eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", + timestamp, decode_count, decode_time, scale_time_ms, total_time); + } self.frame_cache.put(frame_ts, data.clone()); return Ok(data); } From aa7d3a3bf4c5d6e5312fc96d5a696023ad7a9cf5 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Thu, 25 Jun 2026 23:30:44 -0400 Subject: [PATCH 02/31] export: cache the static background + bilinear video, add render profiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The export render bucket was dominated by re-rendering the static document background through Vello every frame and by nearest-sampling the video on upscale. - Background cache: render the (static) background through Vello once, snapshot the composited HDR accumulator, and restore it with a single texture copy on every later frame instead of a Vello render + sRGB-convert + composite (+2 submits). Invalidated on resize. background-render 3.6ms -> 0.56ms (1080p), 7.5ms -> ~0.5ms (4K). - blit_straight now uses the bilinear sampler — video frames are scaled to the output size, and nearest made that blocky. Fixes export and live preview. - LB_RENDER_PROFILE: gated per-frame timing split (build/decode vs composite/ upload vs srgb, and background vs layers) used to find all of the above. Kept as a debug aid for the remaining decode stages. Net: export render ~12.8ms -> ~7.4ms/frame on a 1080p video clip. --- lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md | 102 +++++++++++ .../src/export/video_exporter.rs | 167 +++++++++++++++--- .../lightningbeam-editor/src/gpu_brush.rs | 4 +- 3 files changed, 250 insertions(+), 23 deletions(-) create mode 100644 lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md diff --git a/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md b/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md new file mode 100644 index 0000000..0586549 --- /dev/null +++ b/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md @@ -0,0 +1,102 @@ +# GPU-resident video decode + dynamic decode resolution + +## Context + +Profiling the zero-copy H.264 export (single Group[Video, Audio] clip, `LB_RENDER_PROFILE=1`) +broke the per-frame CPU "render" bucket down as: + +| Cost (ms/frame) | 1080p | 4K | What it is | +|-------------------------|-------|-------|-----------------------------------------------------| +| decode | 3.1 | 19.0 | software ffmpeg decode (`video.rs::get_frame`) | +| background re-render | 3.6 | 7.5 | static background pushed through Vello *every frame* | +| video upload + blit | 4.1 | 4.2 | per-frame transient texture alloc + `write_texture` | +| srgb | 0.4 | 0.4 | linear→sRGB pass | + +The video correctly takes the GPU Video-instance path (not Vello-baked) — `LB_LAYER_DEBUG=1` +shows `Video (1 instance)`. So the cost is **the video frame itself**: software decode, then an +8 MB `write_texture` upload of the decoded RGBA every frame. At 4K, software decode (19 ms) +dominates everything. + +### Two correctness problems found alongside the perf issue + +1. **Decode resolution is frozen to document size at import.** `load_video(clip, src, doc_w, doc_h)` + (`main.rs:4302`) sizes the decoder's swscale output to the document, capped to never upscale + (`video.rs:149`). Export *reuses that decoder*, so exporting **above** document resolution yields + video that was decoded to ≤document res and then GPU-**up**scaled — real source detail thrown away. +2. **It can't follow the consumer or a document resize.** Preview wants small/fast frames; export + wants full res; changing the document size should re-target the decode. None of that works with a + size frozen at import. + +## Goal + +Decouple **decode resolution** from import/document size: the renderer requests a frame *at a target +resolution*, and the decode path produces it. Hardware-decode H.264 (and later HEVC/AV1) into a GPU +surface and keep it GPU-resident through composite into the encoder — no CPU frame copy in either +direction. Software decode stays a **first-class** path (codecs/platforms without HW support), decoding +at the requested target res. This fixes the 4K decode wall, the 8 MB upload, *and* the resolution bugs. + +## Design principles + +- **Decode native, scale to the consumer's target.** + - *Hardware path:* decode into a native VAAPI surface → import as a wgpu texture (reuse the + `gpu-video-encoder` `dmabuf.rs` / `vk_device.rs` plumbing, read direction) → the GPU blit that + already composites the Video instance scales native→target for free. Handles any target res and + document resizes inherently; the cached frame is a native GPU texture. + - *Software path:* decode native → `swscale` to the requested target (the reusable scaler is keyed + on input format/size **and** output size — rebuilt when the target changes). Preview requests + preview res (cheap); export requests export res (full quality). +- **`VideoManager::get_frame` takes a target `(w, h)`** instead of relying on a frozen output size. + The frame cache is keyed to handle multiple live targets (preview + export) — either cache native + frames and scale on demand, or key by `(clip, ts, target)`; decide in Stage 2 by measuring cache + hit/scale tradeoff. +- **Software is not optional.** Hardware decode is an acceleration of the same `get_frame` contract, + selected per source when the codec/driver supports it; everything falls back to software cleanly. + +## Approach (staged; each stage compiles + is independently useful) + +### Stage 0 — independent quick wins (not blocked on decode) +- **Cache the static background** (`composite_document_to_hdr`): render once, reuse via a persistent + HDR texture (copy-in each frame) instead of a full Vello render + 2 passes/submits every frame. + Recovers ~3.6 ms (1080p) / ~7.5 ms (4K) per frame on *every* export. (In flight.) + +### Stage 1 — software: decode at the requested target res (testable; fixes the quality bug now) +- Change `VideoManager::get_frame(clip, ts)` → `get_frame(clip, ts, target_w, target_h)`; thread the + target from the renderer (preview = current doc/preview res, export = export res). Cap at native. +- Rework `VideoDecoder` so output size is per-request, not frozen at construction; cache the swscale + context per output size (already cached per stream — extend the key). Adjust the frame cache key. +- Result: software exports are full-quality at any export res, and document resizes re-target decode. + No hardware needed; this is the correctness fix for the codecs HW can't handle anyway. + +### Stage 2 — hardware decode primitive (headless-testable here, like the 8 encode tests) +- In `gpu-video-encoder` (rename → `gpu-video-codec`): `h264_vaapi`-style **decode** → VAAPI surface → + export DMA-BUF → import as a wgpu texture. Hardware test: decode a known file, verify dims/contents. + +### Stage 3 — wire hardware decode into `get_frame` (blind; user-verifies) +- When the source codec/driver is HW-decodable, `get_frame` returns a **GPU texture** (native res) + instead of `Arc>`; the compositor uses it directly (no `write_texture`), GPU-scaling to the + target. For the zero-copy export the frame never leaves the GPU: **decode → composite → encode** on + one device. Software path is the fallback for everything else. + +## Critical files +- `lightningbeam-core/src/video.rs` — `VideoDecoder` (per-request output size, scaler cache), + `VideoManager::get_frame` (target param, cache key). +- `lightningbeam-core/src/renderer.rs` — pass the render target res into the video-instance build. +- `lightningbeam-editor/src/export/video_exporter.rs` — background cache (Stage 0); consume a GPU + texture instead of uploading RGBA (Stage 3). +- `gpu-video-encoder/` (→ `gpu-video-codec`) — `dmabuf.rs`/`vk_device.rs` reused for the decode import. + +## Risks +- **Codec coverage** — only some codecs are HW-decodable per GPU/driver; software must stay correct + and well-tested. Selection must probe support per source, not assume. +- **Cache memory** — native-res GPU textures (esp. 4K) are large; the frame cache budget needs revisiting. +- **Colorspace/format** — VAAPI decode surfaces are NV12/tiled; the existing import handles NV12, but + 10-bit/HDR sources (P010) need format handling. +- **Preview vs export sharing** — two live targets (preview res + export res) from the same source; the + cache/scaler design must serve both without thrashing. + +## Verification +- Stage 0/1: visual — export above document res is now full-quality (not upscaled); profile shows + background ≈ 0 and (Stage 1) software export correct at the chosen res. +- Stage 2: headless hardware test in `gpu-video-codec` (decode → wgpu texture, ffprobe/byte checks). +- Stage 3 (user): 1080p + 4K H.264 export — decode/upload buckets collapse; software fallback for a + non-HW codec (e.g. ProRes) still produces correct full-res output. diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 86bc8d3..efa7990 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -83,6 +83,11 @@ pub struct ExportGpuResources { pub canvas_blit: crate::gpu_brush::CanvasBlitPipeline, /// Per-keyframe GPU texture cache for raster layers during export. pub raster_cache: std::collections::HashMap, + /// Cached HDR accumulator state after the (static) background is composited in. The document + /// background doesn't change across an export, so it's rendered once and restored with a cheap + /// texture copy each frame instead of a full Vello render + 2 passes/submits. `None` until the + /// first frame; invalidated on resize. + cached_bg_hdr: Option, } impl ExportGpuResources { @@ -108,7 +113,8 @@ impl ExportGpuResources { format: HDR_FORMAT, usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_SRC, + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::COPY_DST, // restore cached background each frame view_formats: &[], }); let hdr_texture_view = hdr_texture.create_view(&wgpu::TextureViewDescriptor::default()); @@ -261,6 +267,7 @@ impl ExportGpuResources { linear_to_srgb_sampler, canvas_blit, raster_cache: std::collections::HashMap::new(), + cached_bg_hdr: None, } } @@ -279,10 +286,12 @@ impl ExportGpuResources { format: HDR_FORMAT, usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_SRC, + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::COPY_DST, view_formats: &[], }); self.hdr_texture_view = self.hdr_texture.create_view(&wgpu::TextureViewDescriptor::default()); + self.cached_bg_hdr = None; // dimensions changed — rebuild the background cache } } @@ -748,29 +757,74 @@ fn composite_document_to_hdr( antialiasing_method: vello::AaConfig::Area, }; - // --- Background --- - let bg_srgb = gpu_resources.buffer_pool.acquire(device, layer_spec); - let bg_hdr = gpu_resources.buffer_pool.acquire(device, hdr_spec); - if let (Some(bg_srgb_view), Some(bg_hdr_view)) = ( - gpu_resources.buffer_pool.get_view(bg_srgb), - gpu_resources.buffer_pool.get_view(bg_hdr), - ) { - renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &layer_render_params) - .map_err(|e| format!("Failed to render background: {e}"))?; - let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_srgb_to_linear") }); - gpu_resources.srgb_to_linear.convert(device, &mut enc, bg_srgb_view, bg_hdr_view); + let prof = render_profile_enabled(); + let t_c0 = std::time::Instant::now(); + + // --- Background (cached) --- + // The document background is static across an export, so render it through Vello exactly once + // (into the accumulator) and snapshot the result; every later frame restores it with a single + // GPU texture copy instead of a Vello render + sRGB-convert + composite (+2 submits). + let bg_cached = matches!( + &gpu_resources.cached_bg_hdr, + Some(t) if t.width() == width && t.height() == height + ); + let copy_size = wgpu::Extent3d { width, height, depth_or_array_layers: 1 }; + if bg_cached { + // Restore the cached background into the accumulator. + let cached = gpu_resources.cached_bg_hdr.as_ref().unwrap(); + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_restore") }); + enc.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { texture: cached, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyTextureInfo { texture: &gpu_resources.hdr_texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + copy_size, + ); queue.submit(Some(enc.finish())); - let bg_layer = CompositorLayer::normal(bg_hdr, 1.0); - let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_composite") }); - // When transparency is allowed, start from transparent black so the background's - // native alpha is preserved. Otherwise force an opaque black underlay. - let clear = if allow_transparency { [0.0, 0.0, 0.0, 0.0] } else { [0.0, 0.0, 0.0, 1.0] }; - gpu_resources.compositor.composite(device, queue, &mut enc, &[bg_layer], - &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, Some(clear)); + } else { + // First frame (or after a resize): full background render into the accumulator. + let bg_srgb = gpu_resources.buffer_pool.acquire(device, layer_spec); + let bg_hdr = gpu_resources.buffer_pool.acquire(device, hdr_spec); + if let (Some(bg_srgb_view), Some(bg_hdr_view)) = ( + gpu_resources.buffer_pool.get_view(bg_srgb), + gpu_resources.buffer_pool.get_view(bg_hdr), + ) { + renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &layer_render_params) + .map_err(|e| format!("Failed to render background: {e}"))?; + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_srgb_to_linear") }); + gpu_resources.srgb_to_linear.convert(device, &mut enc, bg_srgb_view, bg_hdr_view); + queue.submit(Some(enc.finish())); + let bg_layer = CompositorLayer::normal(bg_hdr, 1.0); + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_composite") }); + // When transparency is allowed, start from transparent black so the background's + // native alpha is preserved. Otherwise force an opaque black underlay. + let clear = if allow_transparency { [0.0, 0.0, 0.0, 0.0] } else { [0.0, 0.0, 0.0, 1.0] }; + gpu_resources.compositor.composite(device, queue, &mut enc, &[bg_layer], + &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, Some(clear)); + queue.submit(Some(enc.finish())); + } + gpu_resources.buffer_pool.release(bg_srgb); + gpu_resources.buffer_pool.release(bg_hdr); + + // Snapshot the composited background for reuse on subsequent frames. + let cached = device.create_texture(&wgpu::TextureDescriptor { + label: Some("export_cached_bg_hdr"), + size: copy_size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: HDR_FORMAT, + usage: wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_snapshot") }); + enc.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { texture: &gpu_resources.hdr_texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyTextureInfo { texture: &cached, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + copy_size, + ); queue.submit(Some(enc.finish())); + gpu_resources.cached_bg_hdr = Some(cached); } - gpu_resources.buffer_pool.release(bg_srgb); - gpu_resources.buffer_pool.release(bg_hdr); + let t_bg = std::time::Instant::now(); // --- Layers --- for rendered_layer in &composite_result.layers { @@ -906,10 +960,33 @@ fn composite_document_to_hdr( } } + if prof { + record_composite_profile(t_bg.duration_since(t_c0), t_bg.elapsed()); + } + gpu_resources.buffer_pool.next_frame(); Ok(()) } +/// Split of `composite_document_to_hdr`: static-background re-render vs. the layer loop +/// (video upload + blits). Prints a running average every 200 frames under LB_RENDER_PROFILE. +fn record_composite_profile(background: std::time::Duration, layers: std::time::Duration) { + use std::sync::atomic::{AtomicU64, Ordering}; + static BG_US: AtomicU64 = AtomicU64::new(0); + static LAYERS_US: AtomicU64 = AtomicU64::new(0); + static N: AtomicU64 = AtomicU64::new(0); + BG_US.fetch_add(background.as_micros() as u64, Ordering::Relaxed); + LAYERS_US.fetch_add(layers.as_micros() as u64, Ordering::Relaxed); + let n = N.fetch_add(1, Ordering::Relaxed) + 1; + if n % 200 == 0 { + println!( + "📊 [COMPOSITE PROFILE] {n} frames avg: background-render {:.2}ms | layers(video upload+blit) {:.2}ms", + BG_US.load(Ordering::Relaxed) as f64 / n as f64 / 1000.0, + LAYERS_US.load(Ordering::Relaxed) as f64 / n as f64 / 1000.0, + ); + } +} + /// Upload `pixels` to a transient GPU texture (TEXTURE_BINDING | COPY_DST) in the /// given format. Use `Rgba8UnormSrgb` to upload raw sRGB bytes and let the GPU /// decode to linear on sample (no CPU conversion). @@ -1224,6 +1301,12 @@ pub fn render_frame_to_gpu_rgba( ) -> Result { use vello::kurbo::Affine; + // One-shot profiling of the render-bucket split (LB_RENDER_PROFILE=1): how much of the + // per-frame CPU "render" is document build (incl. video decode) vs. composite-command + // recording (incl. the frame texture upload) vs. the sRGB pass. Prints a running average. + let prof = render_profile_enabled(); + let t0 = std::time::Instant::now(); + // Set document time to the frame timestamp document.current_time = timestamp; @@ -1256,8 +1339,10 @@ pub fn render_frame_to_gpu_rgba( floating_selection, false, // No checkerboard in export ); + let t_build = std::time::Instant::now(); composite_document_to_hdr(&composite_result, document, device, queue, renderer, gpu_resources, width, height, allow_transparency)?; + let t_composite = std::time::Instant::now(); // Convert HDR to sRGB (linear → sRGB), render directly to external RGBA texture let output_view = rgba_texture_view; @@ -1302,11 +1387,49 @@ pub fn render_frame_to_gpu_rgba( render_pass.draw(0..4, 0..1); } + if prof { + record_render_profile( + t_build.duration_since(t0), + t_composite.duration_since(t_build), + t_composite.elapsed(), + ); + } + // Return encoder for caller to submit (ReadbackPipeline will handle submission and async readback) // Frame is already rendered to external RGBA texture, no GPU YUV conversion needed Ok(encoder) } +/// `LB_RENDER_PROFILE` gate, checked once. +fn render_profile_enabled() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("LB_RENDER_PROFILE").is_ok()) +} + +/// Accumulate the per-frame render split and print a running average every 200 frames. +/// `build` = document build incl. video decode; `composite` = composite-command recording +/// incl. the frame texture upload; `srgb` = the linear→sRGB pass. +fn record_render_profile(build: std::time::Duration, composite: std::time::Duration, srgb: std::time::Duration) { + use std::sync::atomic::{AtomicU64, Ordering}; + static BUILD_US: AtomicU64 = AtomicU64::new(0); + static COMPOSITE_US: AtomicU64 = AtomicU64::new(0); + static SRGB_US: AtomicU64 = AtomicU64::new(0); + static N: AtomicU64 = AtomicU64::new(0); + BUILD_US.fetch_add(build.as_micros() as u64, Ordering::Relaxed); + COMPOSITE_US.fetch_add(composite.as_micros() as u64, Ordering::Relaxed); + SRGB_US.fetch_add(srgb.as_micros() as u64, Ordering::Relaxed); + let n = N.fetch_add(1, Ordering::Relaxed) + 1; + if n % 200 == 0 { + let (b, c, s) = (BUILD_US.load(Ordering::Relaxed), COMPOSITE_US.load(Ordering::Relaxed), SRGB_US.load(Ordering::Relaxed)); + println!( + "📊 [RENDER PROFILE] {n} frames avg: build(+decode) {:.2}ms | composite(+upload) {:.2}ms | srgb {:.2}ms", + b as f64 / n as f64 / 1000.0, + c as f64 / n as f64 / 1000.0, + s as f64 / n as f64 / 1000.0, + ); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs index 5cf8421..80586d4 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs @@ -2238,6 +2238,8 @@ impl CanvasBlitPipeline { /// Blit a **straight-alpha** source (e.g. a video frame uploaded to an /// `Rgba8UnormSrgb` texture, hardware-decoded to linear on sample). Uses the /// `fs_main_straight` pipeline, which skips the unpremultiply that `blit` does. + /// Bilinear-sampled: video frames are scaled to the output size (document→export, or any + /// non-1:1 transform), and nearest sampling makes that look blocky. pub fn blit_straight( &self, device: &wgpu::Device, @@ -2247,7 +2249,7 @@ impl CanvasBlitPipeline { transform: &BlitTransform, mask_view: Option<&wgpu::TextureView>, ) { - self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline_straight); + self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler, &self.pipeline_straight); } #[allow(clippy::too_many_arguments)] From f1fba186c18cacc875983d150e9819d927a71d11 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 00:36:15 -0400 Subject: [PATCH 03/31] video: decode at the consumer's target resolution (Stage 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decoder's output size was frozen to the document size at import, and export reused that decoder — so exporting above document res upscaled the video (real source detail discarded) and a document resize never re-targeted the decode. Decode size is now chosen per get_frame call: VideoDecoder::get_frame and VideoManager::get_frame take a target (w, h), capped to native (never upscale), with the swscale context and frame caches keyed on the output size so preview (preview res) and an in-progress export (export res) don't collide. The renderer derives the target from the document->output base_transform, so export decodes at export res (full detail) and the canvas at preview res. Thumbnails/asset library pass small targets. --- .../lightningbeam-core/src/renderer.rs | 22 +++- .../lightningbeam-core/src/video.rs | 113 +++++++++--------- .../src/panes/asset_library.rs | 3 +- 3 files changed, 79 insertions(+), 59 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index 4ee00bd..c367a15 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -366,6 +366,20 @@ pub struct CompositeRenderResult { /// and effects in the GPU compositor. /// /// Layers are returned in bottom-to-top order for compositing. +/// Decode-target resolution for video clips = the output (export/preview) resolution, derived from +/// the document→output `base_transform` scale. The decoder caps this to the source's native size, +/// so it means "decode at the size we'll actually display, never upscaling": full detail when +/// exporting above document res (instead of upscaling a document-res frame), and cheap small frames +/// for the canvas. Stable per render pass, so it doesn't thrash the decoder's scaler/cache. +fn video_decode_target(document: &Document, base_transform: Affine) -> (u32, u32) { + let c = base_transform.as_coeffs(); // [a, b, c, d, e, f] + let sx = (c[0] * c[0] + c[1] * c[1]).sqrt(); + let sy = (c[2] * c[2] + c[3] * c[3]).sqrt(); + let w = (document.width * sx).ceil().max(1.0) as u32; + let h = (document.height * sy).ceil().max(1.0) as u32; + (w, h) +} + pub fn render_document_for_compositing( document: &Document, base_transform: Affine, @@ -545,12 +559,13 @@ pub fn render_layer_isolated( let layer_opacity = layer.opacity(); let mut video_mgr = video_manager.lock().unwrap(); let mut instances = Vec::new(); + let (target_w, target_h) = video_decode_target(document, base_transform); let tempo_map = document.tempo_map(); for clip_instance in &video_layer.clip_instances { let Some(video_clip) = document.video_clips.get(&clip_instance.clip_id) else { continue }; let Some(clip_time) = clip_instance.remap_time(time, video_clip.duration, tempo_map) else { continue }; - let Some(frame) = video_mgr.get_frame(&clip_instance.clip_id, clip_time) else { continue }; + let Some(frame) = video_mgr.get_frame(&clip_instance.clip_id, clip_time, target_w, target_h) else { continue }; // Evaluate animated transform properties. let anim = &video_layer.layer.animation_data; @@ -1095,8 +1110,9 @@ fn render_video_layer( continue; // Clip instance not active at this time }; - // Get video frame from VideoManager - let Some(frame) = video_manager.get_frame(&clip_instance.clip_id, clip_time) else { + // Get video frame from VideoManager at the output (export/preview) resolution. + let (target_w, target_h) = video_decode_target(document, base_transform); + let Some(frame) = video_manager.get_frame(&clip_instance.clip_id, clip_time, target_w, target_h) else { continue; // Frame not available }; diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 92cd745..e2beb5e 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -84,22 +84,23 @@ pub struct VideoMetadata { /// Video decoder with LRU frame caching pub struct VideoDecoder { source: VideoSource, - _width: u32, // Original video width - _height: u32, // Original video height - output_width: u32, // Scaled output width - output_height: u32, // Scaled output height + native_width: u32, // Original (decoded) video width + native_height: u32, // Original (decoded) video height fps: f64, _duration: f64, time_base: f64, stream_index: usize, - frame_cache: LruCache>, // timestamp -> RGBA data + // Decoded RGBA keyed by (frame timestamp, output width, output height): the same source + // frame may be requested at different sizes (preview res vs export res). + frame_cache: LruCache<(i64, u32, u32), Vec>, input: Option, decoder: Option, last_decoded_ts: i64, // Track the last decoded frame timestamp keyframe_positions: Vec, // Index of keyframe timestamps for fast seeking - /// Reused RGBA scaler, keyed by the input (format, width, height). Building an swscale - /// context is not free; a stream's frames share one format/size, so build it once. - scaler: Option<(ffmpeg::format::Pixel, u32, u32, SendScaler)>, + /// Reused RGBA scaler, keyed by `(input format, input w, input h, output w, output h)`. + /// Building an swscale context isn't free; a stream's frames share one input format/size and a + /// consumer keeps one output size, so it's built once and rebuilt only when either changes. + scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>, } /// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only @@ -145,14 +146,12 @@ impl VideoDecoder { let height = decoder.height(); let time_base = f64::from(video_stream.time_base()); - // Calculate output dimensions (scale down if larger than max) - let (output_width, output_height) = if let (Some(max_w), Some(max_h)) = (max_width, max_height) { - // Calculate scale to fit within max dimensions while preserving aspect ratio - let scale = (max_w as f32 / width as f32).min(max_h as f32 / height as f32).min(1.0); - ((width as f32 * scale) as u32, (height as f32 * scale) as u32) - } else { - (width, height) - }; + // Output dimensions are now chosen per `get_frame` call (the caller's target res, capped to + // native) rather than frozen here — so the same clip can be decoded at preview res for the + // canvas and at full export res, and exporting above document res no longer upscales. + // `max_width`/`max_height` are retained as an upper bound for callers that want a fixed cap + // (e.g. thumbnails pass their thumb width per call instead). + let _ = (max_width, max_height); // Try to get duration from stream, fallback to container let duration = if video_stream.duration() > 0 { @@ -184,10 +183,8 @@ impl VideoDecoder { Ok(Self { source, - _width: width, - _height: height, - output_width, - output_height, + native_width: width, + native_height: height, fps, _duration: duration, time_base, @@ -218,19 +215,18 @@ impl VideoDecoder { self.keyframe_positions = positions; } - /// Get the output width (scaled dimensions) - pub fn get_output_width(&self) -> u32 { - self.output_width + /// The output size for a requested target: the target capped to native resolution, preserving + /// aspect ratio (never upscale beyond native — there's no detail to invent). + fn capped_output(&self, target_w: u32, target_h: u32) -> (u32, u32) { + let (nw, nh) = (self.native_width as f32, self.native_height as f32); + if nw <= 0.0 || nh <= 0.0 { return (self.native_width.max(1), self.native_height.max(1)); } + let scale = (target_w as f32 / nw).min(target_h as f32 / nh).min(1.0); + (((nw * scale) as u32).max(1), ((nh * scale) as u32).max(1)) } - /// Get the output height (scaled dimensions) - pub fn get_output_height(&self) -> u32 { - self.output_height - } - - /// Decode a frame at the specified timestamp (public wrapper) + /// Decode a frame at the specified timestamp, at native resolution (public wrapper). pub fn decode_frame(&mut self, timestamp: f64) -> Result, String> { - self.get_frame(timestamp) + self.get_frame(timestamp, self.native_width, self.native_height).map(|(d, _, _)| d) } /// Build an index of all keyframe positions in the video by scanning packets @@ -273,10 +269,14 @@ impl VideoDecoder { } /// Get a decoded frame at the specified timestamp - fn get_frame(&mut self, timestamp: f64) -> Result, String> { + /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns the + /// RGBA bytes and the actual output dimensions. + fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32) -> Result<(Vec, u32, u32), String> { use std::time::Instant; let t_start = Instant::now(); + let (out_w, out_h) = self.capped_output(target_w, target_h); + // Round timestamp to nearest frame boundary to improve cache hits // This ensures that timestamps like 1.0001s and 0.9999s both map to frame 1.0s let frame_duration = 1.0 / self.fps; @@ -284,13 +284,14 @@ impl VideoDecoder { // Convert timestamp to frame timestamp let frame_ts = (rounded_timestamp / self.time_base) as i64; + let cache_key = (frame_ts, out_w, out_h); // Check cache - if let Some(cached_frame) = self.frame_cache.get(&frame_ts) { + if let Some(cached_frame) = self.frame_cache.get(&cache_key) { if video_debug() { eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); } - return Ok(cached_frame.clone()); + return Ok((cached_frame.clone(), out_w, out_h)); } // Determine if we need to seek @@ -378,10 +379,11 @@ impl VideoDecoder { let t_scale_start = Instant::now(); // Reuse the RGBA scaler across frames; rebuild only if the input - // format/size changes (it won't within a stream). + // format/size or the requested output size changes. let need_new = match &self.scaler { - Some((fmt, w, h, _)) => { + Some((fmt, w, h, ow, oh, _)) => { *fmt != frame.format() || *w != frame.width() || *h != frame.height() + || *ow != out_w || *oh != out_h } None => true, }; @@ -391,21 +393,21 @@ impl VideoDecoder { frame.width(), frame.height(), ffmpeg::format::Pixel::RGBA, - self.output_width, - self.output_height, + out_w, + out_h, ffmpeg::software::scaling::flag::Flags::BILINEAR, ).map_err(|e| e.to_string())?; - self.scaler = Some((frame.format(), frame.width(), frame.height(), SendScaler(ctx))); + self.scaler = Some((frame.format(), frame.width(), frame.height(), out_w, out_h, SendScaler(ctx))); } - let scaler = &mut self.scaler.as_mut().unwrap().3.0; + let scaler = &mut self.scaler.as_mut().unwrap().5.0; let mut rgb_frame = ffmpeg::util::frame::Video::empty(); scaler.run(&frame, &mut rgb_frame) .map_err(|e| e.to_string())?; // Remove stride padding to create tightly packed RGBA data - let width = self.output_width as usize; - let height = self.output_height as usize; + let width = out_w as usize; + let height = out_h as usize; let stride = rgb_frame.stride(0); let row_size = width * 4; // RGBA = 4 bytes per pixel let source_data = rgb_frame.data(0); @@ -432,8 +434,8 @@ impl VideoDecoder { eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", timestamp, decode_count, decode_time, scale_time_ms, total_time); } - self.frame_cache.put(frame_ts, data.clone()); - return Ok(data); + self.frame_cache.put(cache_key, data.clone()); + return Ok((data, out_w, out_h)); } break; } @@ -491,7 +493,8 @@ pub fn generate_keyframe_thumbnails( if should_skip(ks) { continue; } - if let Ok(rgba) = decoder.get_frame(ks) { + // Decode at the thumbnail width (large height so width is the constraint), capped to native. + if let Ok((rgba, _, _)) = decoder.get_frame(ks, thumb_width, 100_000) { on_thumb(ks, Arc::new(rgba)); } } @@ -569,7 +572,7 @@ pub struct VideoManager { /// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which /// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600) /// so playback of arbitrarily long video never grows unbounded. - frame_cache: LruCache<(Uuid, i64), Arc>, + frame_cache: LruCache<(Uuid, i64, u32, u32), Arc>, /// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len), /// kept in sync on insert/evict/remove so eviction is O(1) per frame. frame_cache_bytes: usize, @@ -647,10 +650,12 @@ impl VideoManager { /// /// Returns None if the clip is not loaded or decoding fails. /// Frames are cached for performance. - pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64) -> Option> { - // Convert timestamp to milliseconds for cache key + pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { + // Convert timestamp to milliseconds for cache key. The target size is part of the key: the + // canvas (preview res) and an in-progress export (export res) request the same clip/time at + // different sizes, and must not collide. let timestamp_ms = (timestamp * 1000.0) as i64; - let cache_key = (*clip_id, timestamp_ms); + let cache_key = (*clip_id, timestamp_ms, target_w, target_h); // Check frame cache first if let Some(cached_frame) = self.frame_cache.get(&cache_key) { @@ -662,10 +667,8 @@ impl VideoManager { let decoder_arc = Arc::clone(self.decoders.get(clip_id)?); let mut decoder = decoder_arc.lock().ok()?; - // Decode the frame - let rgba_data = decoder.get_frame(timestamp).ok()?; - let width = decoder.output_width; - let height = decoder.output_height; + // Decode the frame at the requested target (capped to native by the decoder). + let (rgba_data, width, height) = decoder.get_frame(timestamp, target_w, target_h).ok()?; drop(decoder); // release the lock before touching `self` // Create VideoFrame and cache it @@ -683,7 +686,7 @@ impl VideoManager { /// Insert a frame into the byte-budgeted cache, evicting least-recently-used /// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`]. - fn cache_frame(&mut self, key: (Uuid, i64), frame: Arc) { + fn cache_frame(&mut self, key: (Uuid, i64, u32, u32), frame: Arc) { let bytes = frame.rgba_data.len(); if let Some(old) = self.frame_cache.put(key, frame) { self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len()); @@ -799,10 +802,10 @@ impl VideoManager { // Remove all cached frames for this clip (LruCache has no retain; collect // matching keys, then pop each, keeping the byte total in sync). - let keys: Vec<(Uuid, i64)> = self + let keys: Vec<(Uuid, i64, u32, u32)> = self .frame_cache .iter() - .filter(|((id, _), _)| id == clip_id) + .filter(|((id, _, _, _), _)| id == clip_id) .map(|(k, _)| *k) .collect(); for key in keys { diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs index 3e3cd41..a76fc80 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs @@ -316,7 +316,8 @@ fn generate_video_thumbnail( let frame = { let mut video_mgr = video_manager.lock().ok()?; - video_mgr.get_frame(clip_id, timestamp)? + // Small frame for the asset thumbnail (capped to native, aspect preserved). + video_mgr.get_frame(clip_id, timestamp, THUMBNAIL_SIZE, THUMBNAIL_SIZE)? }; let src_width = frame.width as usize; From 9411145ce9fedd7b96531358ae22cae5d6fbd097 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 00:36:44 -0400 Subject: [PATCH 04/31] export: H.264 color range toggle (fix dark/oversaturated output) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The zero-copy VAAPI encoder emitted full-range BT.709 NV12 but wrote no color tags, so players assumed limited range and stretched it — the H.264 output looked dark and oversaturated (preview and VP9/software were fine). - Rgba2Nv12 takes `full_range` and applies the matching Y/chroma scale+offset (limited 16-235 / full 0-255) via a uniform; the encoder sets color_range + bt709 colorspace/primaries/transfer tags to match. ffprobe-confirmed. - New ColorRange { Limited, Full } on VideoExportSettings (default Limited, the universally-correct choice; serde(default) so saved settings still load), surfaced as a "Color range" dropdown in advanced export settings for H.264. The swscale software fallback still emits Limited regardless of the toggle (Full only affects the VAAPI zero-copy path). --- .../gpu-video-encoder/src/encoder.rs | 15 +++- .../gpu-video-encoder/src/render_nv12.rs | 80 ++++++++++++++----- .../gpu-video-encoder/tests/zerocopy.rs | 2 +- .../tests/zerocopy_encode.rs | 2 +- .../lightningbeam-core/src/export.rs | 28 +++++++ .../lightningbeam-editor/src/export/dialog.rs | 16 +++- .../lightningbeam-editor/src/export/mod.rs | 1 + 7 files changed, 122 insertions(+), 22 deletions(-) diff --git a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs index 1e42a82..e334bac 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs @@ -42,15 +42,17 @@ unsafe impl Send for ZeroCopyEncoder {} impl ZeroCopyEncoder { /// Build a zero-copy `h264_vaapi` encoder writing to `output_path` (container inferred /// from the extension, e.g. `.mp4`). `Err` if VAAPI/the device is unavailable. + #[allow(clippy::too_many_arguments)] pub fn new( width: u32, height: u32, framerate: i32, bitrate_kbps: u32, output_path: &Path, + full_range: bool, ) -> Result { let drm = vk_device::create()?; - let renderer = Rgba2Nv12::new(&drm.device); + let renderer = Rgba2Nv12::new(&drm.device, full_range); unsafe { let mut hw_device = crate::vaapi::create_device()?; let name = CString::new("h264_vaapi").unwrap(); @@ -66,6 +68,17 @@ impl ZeroCopyEncoder { (*enc).framerate = ff::AVRational { num: framerate, den: 1 }; (*enc).pix_fmt = ff::AVPixelFormat::AV_PIX_FMT_VAAPI; (*enc).bit_rate = (bitrate_kbps as i64) * 1000; + // Color signalling for the H.264 VUI. The Rgba2Nv12 shader produces BT.709 luma/chroma + // in the matching range; without these tags players assume limited range and a + // full-range stream looks dark + oversaturated. + (*enc).color_range = if full_range { + ff::AVColorRange::AVCOL_RANGE_JPEG + } else { + ff::AVColorRange::AVCOL_RANGE_MPEG + }; + (*enc).colorspace = ff::AVColorSpace::AVCOL_SPC_BT709; + (*enc).color_primaries = ff::AVColorPrimaries::AVCOL_PRI_BT709; + (*enc).color_trc = ff::AVColorTransferCharacteristic::AVCOL_TRC_BT709; let frames_ref = ff::av_hwframe_ctx_alloc(hw_device); { diff --git a/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs b/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs index 7e3ccab..025435a 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs @@ -2,29 +2,61 @@ //! surface's plane textures (R8 Y, RG8 UV). Render targets (not compute storage) so it //! works with the DMA-BUF-imported plane images, which aren't storage-writable. //! -//! BT.709 full-range, matching `nv12::cpu_reference` and the encoder's color tags. +//! BT.709 matrix; the Y/chroma scale+offset are chosen by `full_range` — limited (TV, 16–235) +//! is the H.264 convention most players assume, full (PC, 0–255) needs the matching color tag. /// Converts a bound RGBA texture into a Y plane (R8) and a UV plane (RG8) via two passes. pub struct Rgba2Nv12 { y_pipeline: wgpu::RenderPipeline, uv_pipeline: wgpu::RenderPipeline, bgl: wgpu::BindGroupLayout, + params_buf: wgpu::Buffer, } impl Rgba2Nv12 { - pub fn new(device: &wgpu::Device) -> Self { + /// `full_range`: true → full/PC range (Y 0–255, no scaling); false → limited/TV range + /// (Y 16–235, chroma 16–240), the H.264 default. The encoder must tag the stream to match. + pub fn new(device: &wgpu::Device, full_range: bool) -> Self { + // (y_offset, y_scale, chroma_offset, chroma_scale). The shader applies these to the + // BT.709 luma/chroma dot products. + let params: [f32; 4] = if full_range { + [0.0, 1.0, 128.0 / 255.0, 1.0] + } else { + [16.0 / 255.0, 219.0 / 255.0, 128.0 / 255.0, 224.0 / 255.0] + }; + let params_bytes: Vec = params.iter().flat_map(|f| f.to_le_bytes()).collect(); + let params_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("rgba2nv12_params"), + size: params_bytes.len() as u64, + usage: wgpu::BufferUsages::UNIFORM, + mapped_at_creation: true, + }); + params_buf.slice(..).get_mapped_range_mut().copy_from_slice(¶ms_bytes); + params_buf.unmap(); let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("rgba2nv12_bgl"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: false }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, }, - count: None, - }], + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], }); let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("rgba2nv12_pl"), @@ -65,6 +97,7 @@ impl Rgba2Nv12 { y_pipeline: mk("y_fs", wgpu::TextureFormat::R8Unorm), uv_pipeline: mk("uv_fs", wgpu::TextureFormat::Rg8Unorm), bgl, + params_buf, } } @@ -80,10 +113,16 @@ impl Rgba2Nv12 { let bg = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("rgba2nv12_bg"), layout: &self.bgl, - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(rgba_view), - }], + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(rgba_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: self.params_buf.as_entire_binding(), + }, + ], }); for (pipeline, view) in [(&self.y_pipeline, y_view), (&self.uv_pipeline, uv_view)] { let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { @@ -110,6 +149,8 @@ impl Rgba2Nv12 { const SHADER: &str = r#" @group(0) @binding(0) var input_rgba: texture_2d; +// (y_offset, y_scale, chroma_offset, chroma_scale) — selects limited vs full range. +@group(0) @binding(1) var params: vec4; // Fullscreen triangle. @vertex @@ -127,7 +168,8 @@ fn load(p: vec2) -> vec3 { @fragment fn y_fs(@builtin(position) pos: vec4) -> @location(0) vec4 { let c = load(vec2(i32(pos.x), i32(pos.y))); - let y = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b; + let luma = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b; + let y = params.x + params.y * luma; return vec4(y, 0.0, 0.0, 1.0); } @@ -138,8 +180,10 @@ fn uv_fs(@builtin(position) pos: vec4) -> @location(0) vec4 { let sy = 2 * i32(pos.y); let a = (load(vec2(sx, sy)) + load(vec2(sx + 1, sy)) + load(vec2(sx, sy + 1)) + load(vec2(sx + 1, sy + 1))) * 0.25; - let u = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b + 0.5; - let v = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b + 0.5; + let cb = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b; + let cr = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b; + let u = params.z + params.w * cb; + let v = params.z + params.w * cr; return vec4(u, v, 0.0, 1.0); } "#; diff --git a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs index dd2152c..2234091 100644 --- a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs +++ b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs @@ -54,7 +54,7 @@ fn zerocopy_real_frame_render() { wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, ); - let conv = render_nv12::Rgba2Nv12::new(&drm.device); + let conv = render_nv12::Rgba2Nv12::new(&drm.device, true); let src_view = src.create_view(&Default::default()); let y_view = imported.y().create_view(&Default::default()); let uv_view = imported.uv().create_view(&Default::default()); diff --git a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs index 69e8ca9..749bca8 100644 --- a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs +++ b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs @@ -10,7 +10,7 @@ fn zerocopy_encode_h264() { let (w, h) = (640u32, 480u32); let out = std::env::temp_dir().join("gpu_video_encoder_zerocopy.mp4"); let _ = std::fs::remove_file(&out); - let mut enc = match ZeroCopyEncoder::new(w, h, 30, 4000, &out) { + let mut enc = match ZeroCopyEncoder::new(w, h, 30, 4000, &out, false) { Ok(e) => e, Err(e) => { eprintln!("[zc-encode] unavailable, skipping: {e}"); diff --git a/lightningbeam-ui/lightningbeam-core/src/export.rs b/lightningbeam-ui/lightningbeam-core/src/export.rs index 4bef658..3d90c88 100644 --- a/lightningbeam-ui/lightningbeam-core/src/export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/export.rs @@ -265,6 +265,29 @@ impl VideoCodec { } } +/// YUV color range for the encoded video (currently H.264). Limited/TV (16–235) is what nearly +/// every player assumes; Full/PC (0–255) keeps a bit more precision but only looks right in players +/// that honor the full-range tag — so Limited is the safe default. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ColorRange { + Limited, + Full, +} + +impl Default for ColorRange { + fn default() -> Self { ColorRange::Limited } +} + +impl ColorRange { + pub fn is_full(&self) -> bool { matches!(self, ColorRange::Full) } + pub fn name(&self) -> &'static str { + match self { + ColorRange::Limited => "Limited (TV, 16–235)", + ColorRange::Full => "Full (PC, 0–255)", + } + } +} + /// Video quality presets #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum VideoQuality { @@ -322,6 +345,10 @@ pub struct VideoExportSettings { /// Video quality pub quality: VideoQuality, + /// YUV color range (H.264 only; ignored by other codecs). + #[serde(default)] + pub color_range: ColorRange, + /// Audio settings (None = no audio) pub audio: Option, @@ -340,6 +367,7 @@ impl Default for VideoExportSettings { height: None, framerate: 60.0, quality: VideoQuality::High, + color_range: ColorRange::Limited, audio: Some(AudioExportSettings::high_quality_aac()), start_time: 0.0, end_time: 60.0, diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs index 11eb248..2016a59 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs @@ -6,7 +6,7 @@ use eframe::egui; use lightningbeam_core::export::{ AudioExportSettings, AudioFormat, ImageExportSettings, ImageFormat, - VideoExportSettings, VideoCodec, VideoQuality, + VideoExportSettings, VideoCodec, VideoQuality, ColorRange, }; use std::path::PathBuf; @@ -527,6 +527,20 @@ impl ExportDialog { }); }); + // Color range applies to H.264 (the VAAPI zero-copy encoder honors it). Limited/TV is the + // compatible default; Full/PC only looks right in players that read the full-range tag. + if matches!(self.video_settings.codec, VideoCodec::H264) { + ui.horizontal(|ui| { + ui.label("Color range:"); + egui::ComboBox::from_id_salt("video_color_range") + .selected_text(self.video_settings.color_range.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.video_settings.color_range, ColorRange::Limited, ColorRange::Limited.name()); + ui.selectable_value(&mut self.video_settings.color_range, ColorRange::Full, ColorRange::Full.name()); + }); + }); + } + ui.checkbox(&mut self.include_audio, "Include Audio"); ui.add_space(8.0); diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 62381bc..b89ca54 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -899,6 +899,7 @@ impl ExportOrchestrator { framerate.round() as i32, settings.quality.bitrate_kbps(), output_path, + settings.color_range.is_full(), ) { Ok(e) => e, Err(e) => { From 255e16434e935ee5d26151b2b786560d9df79974 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 00:46:53 -0400 Subject: [PATCH 05/31] =?UTF-8?q?gpu-video-encoder:=20VAAPI=20hardware=20d?= =?UTF-8?q?ecode=20=E2=86=92=20wgpu=20texture=20(Stage=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Headless decode primitive, the mirror of the encoder: VaapiDecoder opens a file, hardware-decodes H.264 into VAAPI NV12 surfaces (hw_device_ctx + a get_format callback selecting AV_PIX_FMT_VAAPI), maps each surface to a DRM-PRIME DMA-BUF, and imports it as two wgpu plane textures via the existing dmabuf::import_raw — the same path the encoder uses, in the read direction. Frames stay GPU-resident (no CPU copy). Validated by a round-trip test: encode solid gray with ZeroCopyEncoder, decode it back, read the Y plane (mean ≈ 128). All 9 crate tests pass on the container's Intel GPU. Next (Stage 3): wire this into VideoManager/get_frame so the compositor consumes a GPU texture directly (no write_texture upload), with software decode fallback. Co-Authored-By: Claude Opus 4.8 --- .../gpu-video-encoder/src/decoder.rs | 216 ++++++++++++++++++ lightningbeam-ui/gpu-video-encoder/src/lib.rs | 4 + .../tests/decode_roundtrip.rs | 91 ++++++++ 3 files changed, 311 insertions(+) create mode 100644 lightningbeam-ui/gpu-video-encoder/src/decoder.rs create mode 100644 lightningbeam-ui/gpu-video-encoder/tests/decode_roundtrip.rs diff --git a/lightningbeam-ui/gpu-video-encoder/src/decoder.rs b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs new file mode 100644 index 0000000..f53c251 --- /dev/null +++ b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs @@ -0,0 +1,216 @@ +//! VAAPI hardware video decode → wgpu textures. The mirror of [`crate::encoder`]: the codec +//! decodes into a VAAPI NV12 surface, which is mapped to a DRM-PRIME DMA-BUF and imported as two +//! wgpu plane textures via [`crate::dmabuf::import_raw`] — the exact same path the encoder uses, +//! in the read direction. Stays GPU-resident: no CPU frame copy. + +use crate::dmabuf::{self, ImportedNv12, Nv12DmaBuf}; +use crate::vk_device::{self, DrmDevice}; +use ffmpeg_sys_next as ff; +use std::ffi::CString; +use std::path::Path; +use std::ptr; + +#[inline] +fn averror(e: i32) -> i32 { + -e +} + +/// `get_format` callback: pick VAAPI surfaces so the decoder outputs hardware frames. With +/// `hw_device_ctx` set, FFmpeg auto-allocates the matching frames context. +unsafe extern "C" fn get_vaapi_format( + _ctx: *mut ff::AVCodecContext, + mut fmts: *const ff::AVPixelFormat, +) -> ff::AVPixelFormat { + while *fmts != ff::AVPixelFormat::AV_PIX_FMT_NONE { + if *fmts == ff::AVPixelFormat::AV_PIX_FMT_VAAPI { + return ff::AVPixelFormat::AV_PIX_FMT_VAAPI; + } + fmts = fmts.add(1); + } + ff::AVPixelFormat::AV_PIX_FMT_NONE +} + +/// Hardware decoder for a single video file/stream. Frames come back as importable NV12 textures +/// on [`Self::device`]. +pub struct VaapiDecoder { + drm: DrmDevice, + hw_device: *mut ff::AVBufferRef, + fmt: *mut ff::AVFormatContext, + dec: *mut ff::AVCodecContext, + pkt: *mut ff::AVPacket, + frame: *mut ff::AVFrame, + stream_index: i32, + flushing: bool, +} + +// Owns its FFmpeg/Vulkan handles exclusively; only moved, never shared (same as the encoder). +unsafe impl Send for VaapiDecoder {} + +impl VaapiDecoder { + /// Open `input_path` and set up VAAPI hardware decoding of its best video stream. + pub fn new(input_path: &Path) -> Result { + let drm = vk_device::create()?; + unsafe { + let mut hw_device = crate::vaapi::create_device()?; + let cleanup_hw = |hw: *mut ff::AVBufferRef| { + let mut h = hw; + ff::av_buffer_unref(&mut h); + }; + + let path_c = CString::new(input_path.to_string_lossy().as_ref()).unwrap(); + let mut fmt: *mut ff::AVFormatContext = ptr::null_mut(); + if ff::avformat_open_input(&mut fmt, path_c.as_ptr(), ptr::null_mut(), ptr::null_mut()) < 0 { + cleanup_hw(hw_device); + return Err(format!("avformat_open_input {input_path:?} failed")); + } + if ff::avformat_find_stream_info(fmt, ptr::null_mut()) < 0 { + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("avformat_find_stream_info failed".into()); + } + + let mut decoder: *const ff::AVCodec = ptr::null(); + let stream_index = ff::av_find_best_stream( + fmt, + ff::AVMediaType::AVMEDIA_TYPE_VIDEO, + -1, + -1, + &mut decoder, + 0, + ); + if stream_index < 0 || decoder.is_null() { + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("no decodable video stream".into()); + } + + let dec = ff::avcodec_alloc_context3(decoder); + let stream = *(*fmt).streams.add(stream_index as usize); + if ff::avcodec_parameters_to_context(dec, (*stream).codecpar) < 0 { + ff::avcodec_free_context(&mut (dec as *mut _)); + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("avcodec_parameters_to_context failed".into()); + } + (*dec).hw_device_ctx = ff::av_buffer_ref(hw_device); + (*dec).get_format = Some(get_vaapi_format); + + if ff::avcodec_open2(dec, decoder, ptr::null_mut()) < 0 { + ff::avcodec_free_context(&mut (dec as *mut _)); + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("avcodec_open2 (vaapi decode) failed".into()); + } + + // `mut` only to satisfy the move into the struct; the binding above is consumed. + let _ = &mut hw_device; + Ok(Self { + drm, + hw_device, + fmt, + dec, + pkt: ff::av_packet_alloc(), + frame: ff::av_frame_alloc(), + stream_index, + flushing: false, + }) + } + } + + /// The wgpu device the decoded textures live on (the DMA-BUF-import device). + pub fn device(&self) -> &wgpu::Device { + &self.drm.device + } + pub fn queue(&self) -> &wgpu::Queue { + &self.drm.queue + } + + /// Decode the next frame and import it as NV12 plane textures, or `Ok(None)` at end of stream. + pub fn next_frame(&mut self) -> Result, String> { + unsafe { + loop { + let r = ff::avcodec_receive_frame(self.dec, self.frame); + if r == 0 { + let imported = self.map_current(); + ff::av_frame_unref(self.frame); + return imported.map(Some); + } + if r == ff::AVERROR_EOF { + return Ok(None); + } + if r != averror(libc::EAGAIN) { + return Err(format!("avcodec_receive_frame failed: {r}")); + } + if self.flushing { + return Ok(None); // already drained the flush + } + + // Decoder wants more input: pump one packet (or signal EOF to flush). + let rp = ff::av_read_frame(self.fmt, self.pkt); + if rp < 0 { + self.flushing = true; + ff::avcodec_send_packet(self.dec, ptr::null()); + continue; + } + if (*self.pkt).stream_index == self.stream_index { + let rs = ff::avcodec_send_packet(self.dec, self.pkt); + ff::av_packet_unref(self.pkt); + if rs < 0 && rs != averror(libc::EAGAIN) { + return Err(format!("avcodec_send_packet failed: {rs}")); + } + } else { + ff::av_packet_unref(self.pkt); + } + } + } + } + + /// Map the just-decoded VAAPI surface (`self.frame`) to a DRM-PRIME DMA-BUF and import it. + unsafe fn map_current(&self) -> Result { + let drm_f = ff::av_frame_alloc(); + (*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + let flags = ff::AV_HWFRAME_MAP_DIRECT as i32 | ff::AV_HWFRAME_MAP_READ as i32; + if ff::av_hwframe_map(drm_f, self.frame, flags) < 0 { + ff::av_frame_free(&mut (drm_f as *mut _)); + return Err("av_hwframe_map failed".into()); + } + let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor; + let obj = &(*desc).objects[0]; + let width = (*self.frame).width as u32; + let height = (*self.frame).height as u32; + // NV12: Y then UV — either as two layers (one plane each) or one layer with two planes. + let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 { + (&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0]) + } else { + (&(*desc).layers[0].planes[0], &(*desc).layers[0].planes[1]) + }; + let buf = Nv12DmaBuf { + fd: obj.fd, + size: obj.size as u64, + modifier: obj.format_modifier, + width, + height, + y_offset: y_pl.offset as u64, + y_pitch: y_pl.pitch as u64, + uv_offset: uv_pl.offset as u64, + uv_pitch: uv_pl.pitch as u64, + }; + let imported = dmabuf::import_raw(&self.drm, &buf); + ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan + imported + } +} + +impl Drop for VaapiDecoder { + fn drop(&mut self) { + unsafe { + ff::av_frame_free(&mut (self.frame as *mut _)); + ff::av_packet_free(&mut (self.pkt as *mut _)); + ff::avcodec_free_context(&mut (self.dec as *mut _)); + if !self.fmt.is_null() { + ff::avformat_close_input(&mut self.fmt); + } + ff::av_buffer_unref(&mut self.hw_device); + } + } +} diff --git a/lightningbeam-ui/gpu-video-encoder/src/lib.rs b/lightningbeam-ui/gpu-video-encoder/src/lib.rs index 0d76fe4..b1c5c3b 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/lib.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/lib.rs @@ -30,6 +30,10 @@ pub mod dmabuf; #[cfg(target_os = "linux")] pub mod encoder; +/// VAAPI hardware decode → wgpu textures (Linux). +#[cfg(target_os = "linux")] +pub mod decoder; + #[cfg(test)] mod probe_tests { /// Confirm a headless GPU adapter is reachable (Vulkan on Linux/Intel). This gates diff --git a/lightningbeam-ui/gpu-video-encoder/tests/decode_roundtrip.rs b/lightningbeam-ui/gpu-video-encoder/tests/decode_roundtrip.rs new file mode 100644 index 0000000..81f9dc8 --- /dev/null +++ b/lightningbeam-ui/gpu-video-encoder/tests/decode_roundtrip.rs @@ -0,0 +1,91 @@ +//! Round-trip: encode solid frames with the zero-copy encoder, then hardware-decode them back +//! into a wgpu texture and read the Y plane. Verifies the VAAPI decode → DMA-BUF → wgpu import +//! path produces real pixels on the GPU. Skips when VAAPI is unavailable. + +#![cfg(target_os = "linux")] + +use gpu_video_encoder::decoder::VaapiDecoder; +use gpu_video_encoder::encoder::ZeroCopyEncoder; + +#[test] +fn vaapi_decode_roundtrip() { + // 256-wide so the R8 Y readback row (256 B) is already 256-aligned. + let (w, h) = (256u32, 256u32); + let out = std::env::temp_dir().join("gpu_video_encoder_decode_rt.mp4"); + let _ = std::fs::remove_file(&out); + + // --- Encode 10 frames of solid mid-gray. Full range → Y == luma ≈ 128. --- + { + let mut enc = match ZeroCopyEncoder::new(w, h, 30, 4000, &out, true) { + Ok(e) => e, + Err(e) => { + eprintln!("[decode-rt] encode unavailable, skipping: {e}"); + return; + } + }; + let device = enc.device(); + let src = device.create_texture(&wgpu::TextureDescriptor { + label: Some("gray"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let gray = vec![128u8; (w * h * 4) as usize]; + enc.queue().write_texture( + wgpu::TexelCopyTextureInfo { texture: &src, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + &gray, + wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w * 4), rows_per_image: Some(h) }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + for _ in 0..10 { + enc.encode_rgba(&src).expect("encode_rgba"); + } + enc.finish().expect("finish"); + } + + // --- Decode it back on the GPU. --- + let mut dec = match VaapiDecoder::new(&out) { + Ok(d) => d, + Err(e) => { + eprintln!("[decode-rt] decode unavailable, skipping: {e}"); + return; + } + }; + let frame = dec.next_frame().expect("next_frame").expect("expected at least one frame"); + assert_eq!(frame.y().width(), w, "decoded Y width"); + assert_eq!(frame.y().height(), h, "decoded Y height"); + + // Read back the Y plane (R8) and check it's ≈ the gray we encoded. + let device = dec.device(); + let buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("y_readback"), + size: (w * h) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mut cmd = device.create_command_encoder(&Default::default()); + cmd.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { texture: frame.y(), mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyBufferInfo { + buffer: &buf, + layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w), rows_per_image: Some(h) }, + }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + dec.queue().submit(Some(cmd.finish())); + buf.slice(..).map_async(wgpu::MapMode::Read, |_| {}); + let _ = device.poll(wgpu::PollType::wait_indefinitely()); + + let data = buf.slice(..).get_mapped_range(); + let mean = data.iter().map(|&b| b as f64).sum::() / data.len() as f64; + eprintln!("[decode-rt] decoded {w}x{h}, mean Y = {mean:.1}"); + assert!( + (mean - 128.0).abs() < 12.0, + "mean Y {mean} not ≈ 128 — decode produced wrong pixels" + ); + eprintln!("[decode-rt] ✅ VAAPI decode → wgpu texture verified"); +} From ca612d7807ea695d2a57a6d416c86ac396e98178 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 01:11:36 -0400 Subject: [PATCH 06/31] editor: run on a shared VAAPI-capable wgpu device (Stage 3a) Foundation for hardware video decode in BOTH preview and export: wgpu textures can't cross devices, and a hardware-decoded frame is a DMA-BUF-imported texture that needs the import extensions (only addable via wgpu-hal device_from_raw). So eframe + the compositor + decode + encode must share ONE custom device. - vk_device::create_windowed(): the existing import-capable DrmDevice plus VK_KHR_swapchain (the WSI surface instance extensions are already enabled by Instance::init), for use as the editor's main device. - main.rs: on Linux, build the shared device and inject it into eframe via WgpuSetup::Existing (the egui fork supports it); fall back to wgpu's normal device + software decode on any failure, on other platforms, or via LB_NO_SHARED_DEVICE. The DrmDevice's wgpu handles are cloned into eframe (Arc-backed), so the VkDevice persists with them. Runtime-verified: the editor launches and renders identically (canvas/vello, video, panels) on the shared device, and the env-var fallback works. --- .../gpu-video-encoder/src/vk_device.rs | 23 ++++- .../lightningbeam-editor/src/main.rs | 92 +++++++++++++------ 2 files changed, 81 insertions(+), 34 deletions(-) diff --git a/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs b/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs index 889b57d..93a67aa 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs @@ -22,12 +22,20 @@ pub struct DrmDevice { pub raw_instance: ash::Instance, } -/// Create the device, or `Err` if Vulkan/the extension isn't available (caller falls back). +/// Create a headless DMA-BUF-import device (encoder/decoder), or `Err` if Vulkan/the extension +/// isn't available (caller falls back). pub fn create() -> Result { - unsafe { create_inner() } + unsafe { create_inner(false) } } -unsafe fn create_inner() -> Result { +/// Like [`create`] but also enables `VK_KHR_swapchain` so the device can present to a window — +/// for use as the editor's **shared** wgpu device (eframe + compositor + decode + encode all on +/// one device, so hardware-decoded DMA-BUF textures are usable by the preview compositor). +pub fn create_windowed() -> Result { + unsafe { create_inner(true) } +} + +unsafe fn create_inner(windowed: bool) -> Result { use wgpu_hal::vulkan::Api as Vk; // Bring the HAL Instance trait into scope for `init` / `enumerate_adapters`. use wgpu_hal::Instance as _; @@ -72,14 +80,19 @@ unsafe fn create_inner() -> Result { exposed.adapter.required_device_extensions(exposed.features); // Only the genuine extensions; external_memory / bind_memory2 / ycbcr / format_list // are core in Vulkan 1.1+ (this device is 1.3) so they need no enabling. - let extra: &[&'static CStr] = &[ + let mut extra: Vec<&'static CStr> = vec![ ash::ext::image_drm_format_modifier::NAME, ash::khr::external_memory_fd::NAME, ash::ext::external_memory_dma_buf::NAME, ash::ext::queue_family_foreign::NAME, ]; + // Presentation (windowed shared device only): the WSI surface instance extensions are already + // enabled by `Instance::init`; the device needs the swapchain extension to present. + if windowed { + extra.push(ash::khr::swapchain::NAME); + } for e in extra { - if !ext_names.contains(e) { + if !ext_names.contains(&e) { ext_names.push(e); } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 70e7789..08a522a 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -79,6 +79,40 @@ struct Args { cpu_renderer: bool, } +/// The default eframe wgpu device setup (wgpu picks the adapter/device). Used on non-Linux, when +/// the shared VAAPI device is unavailable, or when disabled via `LB_NO_SHARED_DEVICE`. +fn lb_default_wgpu_setup() -> egui_wgpu::WgpuSetup { + egui_wgpu::WgpuSetup::CreateNew(egui_wgpu::WgpuSetupCreateNew { + device_descriptor: std::sync::Arc::new(|adapter| { + let features = adapter.features(); + // Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's + // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability). + // TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer + // (gpu_timer.rs); both are optional and no-op when unsupported. + let optional_features = wgpu::Features::SHADER_F16 + | wgpu::Features::TIMESTAMP_QUERY + | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS; + + let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { + wgpu::Limits::downlevel_webgl2_defaults() + } else { + wgpu::Limits::default() + }; + + wgpu::DeviceDescriptor { + label: Some("lightningbeam wgpu device"), + required_features: features & optional_features, + required_limits: wgpu::Limits { + max_texture_dimension_2d: 8192, + ..base_limits + }, + ..Default::default() + } + }), + ..Default::default() + }) +} + fn main() -> eframe::Result { println!("🚀 Starting Lightningbeam Editor..."); @@ -168,38 +202,38 @@ fn main() -> eframe::Result { viewport_builder = viewport_builder.with_icon(icon); } + // Prefer the shared VAAPI-capable wgpu device on Linux: eframe + the compositor + hardware + // video decode all run on one device, so decoded DMA-BUF frames are usable by the preview + // compositor (decode/encode can't share textures across devices). Falls back to wgpu's normal + // device (software video decode) when unavailable, on other platforms, or via LB_NO_SHARED_DEVICE. + // The DrmDevice's wgpu handles are cloned into eframe (Arc-backed, keep the device alive); the + // raw VkDevice persists with them. + #[allow(unused_mut)] + let mut wgpu_setup = lb_default_wgpu_setup(); + #[cfg(target_os = "linux")] + if std::env::var("LB_NO_SHARED_DEVICE").is_ok() { + println!("🖥 Shared device disabled via LB_NO_SHARED_DEVICE; default wgpu device (software video decode)"); + } else { + match gpu_video_encoder::vk_device::create_windowed() { + Ok(drm) => { + println!("🖥 Using shared VAAPI-capable wgpu device (hardware video decode enabled)"); + wgpu_setup = egui_wgpu::WgpuSetup::Existing(egui_wgpu::WgpuSetupExisting { + instance: drm.instance.clone(), + adapter: drm.adapter.clone(), + device: drm.device.clone(), + queue: drm.queue.clone(), + }); + } + Err(e) => { + println!("🖥 Shared device unavailable ({e}); default wgpu device (software video decode)"); + } + } + } + let options = eframe::NativeOptions { viewport: viewport_builder, wgpu_options: egui_wgpu::WgpuConfiguration { - wgpu_setup: egui_wgpu::WgpuSetup::CreateNew(egui_wgpu::WgpuSetupCreateNew { - device_descriptor: std::sync::Arc::new(|adapter| { - let features = adapter.features(); - // Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's - // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability). - // TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer - // (gpu_timer.rs); both are optional and no-op when unsupported. - let optional_features = wgpu::Features::SHADER_F16 - | wgpu::Features::TIMESTAMP_QUERY - | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS; - - let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { - wgpu::Limits::downlevel_webgl2_defaults() - } else { - wgpu::Limits::default() - }; - - wgpu::DeviceDescriptor { - label: Some("lightningbeam wgpu device"), - required_features: features & optional_features, - required_limits: wgpu::Limits { - max_texture_dimension_2d: 8192, - ..base_limits - }, - ..Default::default() - } - }), - ..Default::default() - }), + wgpu_setup, ..Default::default() }, ..Default::default() From 7909b51df1fa9e67d7a6f76d1d9e741c88fd4fe4 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 01:21:00 -0400 Subject: [PATCH 07/31] gpu-video-encoder: decouple dmabuf import from DrmDevice (Stage 3b foundation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit import_raw now takes (&wgpu::Device, &wgpu::Adapter) and extracts the raw Vulkan handles via as_hal, instead of taking the crate's own DrmDevice. This lets a VAAPI surface be imported onto ANY DMA-BUF-import-capable wgpu device — the encoder/decoder's own device today, and the editor's shared device (Stage 3a) next, so hardware-decoded frames are usable by the preview compositor. Encoder/decoder pass their DrmDevice's device+adapter; round-trip decode test still passes. --- .../gpu-video-encoder/src/decoder.rs | 2 +- .../gpu-video-encoder/src/dmabuf.rs | 55 ++++++++++++------- .../gpu-video-encoder/src/encoder.rs | 2 +- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/lightningbeam-ui/gpu-video-encoder/src/decoder.rs b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs index f53c251..9f64856 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/decoder.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs @@ -195,7 +195,7 @@ impl VaapiDecoder { uv_offset: uv_pl.offset as u64, uv_pitch: uv_pl.pitch as u64, }; - let imported = dmabuf::import_raw(&self.drm, &buf); + let imported = dmabuf::import_raw(&self.drm.device, &self.drm.adapter, &buf); ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan imported } diff --git a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs index a6bfc33..a6e1644 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs @@ -49,10 +49,11 @@ impl ImportedNv12 { } } -/// Convenience: map a freshly-allocated `MappedSurface` and import it. +/// Convenience: map a freshly-allocated `MappedSurface` and import it onto `drm`. pub fn import(drm: &DrmDevice, surf: &MappedSurface) -> Result { import_raw( - drm, + &drm.device, + &drm.adapter, &Nv12DmaBuf { fd: surf.fd, size: surf.size, @@ -67,12 +68,28 @@ pub fn import(drm: &DrmDevice, surf: &MappedSurface) -> Result Result { +/// Import an NV12 DMA-BUF (described by `buf`) as two wgpu plane textures **on `device`**. The raw +/// Vulkan handles are extracted from `device`/`adapter` via `as_hal`, so this works with any +/// DMA-BUF-import-capable wgpu device — the encoder/decoder's own `DrmDevice` *or* the editor's +/// shared device. The fd is duplicated, so the caller keeps ownership of theirs. +pub fn import_raw( + device: &wgpu::Device, + adapter: &wgpu::Adapter, + buf: &Nv12DmaBuf, +) -> Result { + use wgpu_hal::vulkan::Api as Vk; unsafe { - let device = drm.raw_device.clone(); - let instance = &drm.raw_instance; + let hal_device = device + .as_hal::() + .ok_or("device is not Vulkan")?; + let raw_device = hal_device.raw_device().clone(); + let raw_instance = adapter + .as_hal::() + .ok_or("adapter is not Vulkan")? + .shared_instance() + .raw_instance() + .clone(); + let instance = &raw_instance; let dup_fd = libc::dup(buf.fd); if dup_fd < 0 { @@ -103,7 +120,7 @@ pub fn import_raw(drm: &DrmDevice, buf: &Nv12DmaBuf) -> Result Result Result() - .ok_or("device is not Vulkan")?; let wrap = |img: vk::Image, format: wgpu::TextureFormat, w: u32, h: u32| -> wgpu::Texture { // wgpu destroys the image (after wait-idle) when the texture drops; the // captured Arc frees the shared memory once both have run. - let dev = device.clone(); + let dev = raw_device.clone(); let guard = mem_guard.clone(); let cb: wgpu_hal::DropCallback = Box::new(move || { dev.destroy_image(img, None); @@ -170,7 +183,7 @@ pub fn import_raw(drm: &DrmDevice, buf: &Nv12DmaBuf) -> Result( + device.create_texture_from_hal::( hal_tex, &wgpu::TextureDescriptor { label: Some("vaapi-plane"), diff --git a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs index e334bac..c18d52a 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs @@ -215,7 +215,7 @@ impl ZeroCopyEncoder { uv_offset: uv.offset as u64, uv_pitch: uv.pitch as u64, }; - let imported = match dmabuf::import_raw(&self.drm, &buf) { + let imported = match dmabuf::import_raw(&self.drm.device, &self.drm.adapter, &buf) { Ok(i) => i, Err(e) => { ff::av_frame_free(&mut (drm_f as *mut _)); From 863edc80fc9c9b89808908ea0d01a1492e0f501b Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:06:28 -0400 Subject: [PATCH 08/31] core: hardware video decode engine in VideoDecoder (Stage 3b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the existing VideoDecoder with an optional hardware path, reusing its demux/seek/keyframe/blob engine (no duplication): - GpuVideoFrame (NV12 plane wgpu textures) + HwVideoImporter trait (editor implements the DMA-BUF import; the AVFrame crosses as an opaque pointer so core needn't reference the GPU crate) + HwDeviceHandle (opaque AVBufferRef). - VideoManager::set_hardware_decode injects the VAAPI device + importer; each decoder attaches hw_device_ctx + a get_format(VAAPI) callback before opening, decodes into VAAPI surfaces, and imports them (no CPU copy). - get_frame returns DecodedFrame::{Cpu,Gpu}; VideoFrame/VideoRenderInstance gain an optional `gpu`. The frame cache budgets GPU frames as ~w*h*3/2 and keys on whether the consumer wanted GPU. - A hardware decoder serving a CPU consumer (export, render_hardware_ok=false) downloads the surface via av_hwframe_transfer_data then swscales — so export stays software/correct and only the preview goes GPU-resident. HW init or import failure falls back to software per clip. Dormant until the editor injects an importer (next): no importer => software, unchanged. --- .../lightningbeam-core/src/renderer.rs | 9 +- .../lightningbeam-core/src/video.rs | 418 ++++++++++++++---- 2 files changed, 334 insertions(+), 93 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index c367a15..f838b23 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -221,8 +221,11 @@ fn decode_image_brush(data: &[u8]) -> Option { /// A single decoded video frame ready for GPU upload, with its document-space transform. pub struct VideoRenderInstance { - /// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). + /// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). Empty when `gpu` is `Some`. pub rgba_data: Arc>, + /// Hardware-decoded frame as GPU NV12 textures (preview path). When `Some`, the compositor + /// samples it directly (NV12→RGB) instead of uploading `rgba_data`. + pub gpu: Option, pub width: u32, pub height: u32, /// Affine transform that maps from video-pixel space to document space. @@ -609,6 +612,7 @@ pub fn render_layer_isolated( }; instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: frame.gpu.clone(), width: frame.width, height: frame.height, transform: base_transform * clip_transform * frame_to_clip, @@ -629,6 +633,7 @@ pub fn render_layer_isolated( * Affine::scale(scale); instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: None, // webcam frames are always CPU width: frame.width, height: frame.height, transform: cam_transform, @@ -1260,6 +1265,7 @@ fn render_video_layer( if let Some(ex) = extract.as_deref_mut() { ex.instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: frame.gpu.clone(), width: frame.width, height: frame.height, transform: instance_transform * brush_transform, @@ -1314,6 +1320,7 @@ fn render_video_layer( if let Some(ex) = extract.as_deref_mut() { ex.instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: None, // webcam frames are always CPU width: frame.width, height: frame.height, transform: preview_transform, diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index e2beb5e..0ddd083 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -101,6 +101,34 @@ pub struct VideoDecoder { /// Building an swscale context isn't free; a stream's frames share one input format/size and a /// consumer keeps one output size, so it's built once and rebuilt only when either changes. scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>, + /// When set (and `hw_failed` is false), decode on the GPU: attach `hw_device` as the decoder's + /// `hw_device_ctx`, decode into VAAPI surfaces, and hand each surface to `importer` to import as + /// wgpu NV12 textures (no CPU copy). `None`/failure → the software swscale path. + hw_device: Option, + importer: Option>, + /// Set if hardware decode init failed for this clip — fall back to software permanently. + hw_failed: bool, +} + +/// A decoded frame: CPU RGBA (software) or GPU NV12 textures (hardware). +enum DecodedFrame { + Cpu { rgba: Vec, width: u32, height: u32 }, + Gpu(GpuVideoFrame), +} + +/// `get_format` callback for hardware decode: select VAAPI surfaces. With `hw_device_ctx` set, +/// FFmpeg auto-allocates the frames context. +unsafe extern "C" fn get_vaapi_format( + _ctx: *mut ffmpeg::ffi::AVCodecContext, + mut fmts: *const ffmpeg::ffi::AVPixelFormat, +) -> ffmpeg::ffi::AVPixelFormat { + while *fmts != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE { + if *fmts == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI { + return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI; + } + fmts = fmts.add(1); + } + ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE } /// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only @@ -197,9 +225,28 @@ impl VideoDecoder { last_decoded_ts: -1, keyframe_positions, scaler: None, + hw_device: None, + importer: None, + hw_failed: false, }) } + /// Configure hardware (VAAPI) decode for this clip. The next decoder open attaches `hw_device` + /// and decodes into VAAPI surfaces imported by `importer`. Resets any prior decoder so the new + /// mode takes effect on the next `get_frame`. + fn set_hardware(&mut self, hw_device: HwDeviceHandle, importer: Arc) { + self.hw_device = Some(hw_device); + self.importer = Some(importer); + self.hw_failed = false; + self.decoder = None; // force a rebuild with hw_device_ctx + self.input = None; + } + + /// Whether this decoder will hardware-decode (configured + not failed). + fn hw_active(&self) -> bool { + self.hw_device.is_some() && self.importer.is_some() && !self.hw_failed + } + /// The source this decoder reads from (file path or packed container blob). pub fn source(&self) -> VideoSource { self.source.clone() @@ -226,7 +273,11 @@ impl VideoDecoder { /// Decode a frame at the specified timestamp, at native resolution (public wrapper). pub fn decode_frame(&mut self, timestamp: f64) -> Result, String> { - self.get_frame(timestamp, self.native_width, self.native_height).map(|(d, _, _)| d) + // Software-only helper; request CPU output. + match self.get_frame(timestamp, self.native_width, self.native_height, false)? { + DecodedFrame::Cpu { rgba, .. } => Ok(rgba), + DecodedFrame::Gpu(_) => Err("decode_frame: unexpected GPU frame".into()), + } } /// Build an index of all keyframe positions in the video by scanning packets @@ -268,13 +319,19 @@ impl VideoDecoder { } } - /// Get a decoded frame at the specified timestamp - /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns the - /// RGBA bytes and the actual output dimensions. - fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32) -> Result<(Vec, u32, u32), String> { + /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns GPU + /// NV12 textures when hardware-decoding and `want_gpu` (the consumer is on the shared device, + /// i.e. the preview); otherwise CPU RGBA. A hardware decoder serving a CPU consumer (export) + /// downloads the surface via `av_hwframe_transfer_data` then swscales. The `VideoManager` caches + /// the result, so the inner RGBA cache here is for CPU output only. + fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Result { use std::time::Instant; let t_start = Instant::now(); + // `hw` = decoder is opened in hardware mode (produces VAAPI surfaces). + // `gpu_out` = return GPU textures (hw + the consumer can use them). + let hw = self.hw_active(); + let gpu_out = hw && want_gpu; let (out_w, out_h) = self.capped_output(target_w, target_h); // Round timestamp to nearest frame boundary to improve cache hits @@ -286,12 +343,14 @@ impl VideoDecoder { let frame_ts = (rounded_timestamp / self.time_base) as i64; let cache_key = (frame_ts, out_w, out_h); - // Check cache - if let Some(cached_frame) = self.frame_cache.get(&cache_key) { - if video_debug() { - eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + // Check the inner RGBA cache (CPU output only; GPU frames are cached by VideoManager). + if !gpu_out { + if let Some(cached_frame) = self.frame_cache.get(&cache_key) { + if video_debug() { + eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + } + return Ok(DecodedFrame::Cpu { rgba: cached_frame.clone(), width: out_w, height: out_h }); } - return Ok((cached_frame.clone(), out_w, out_h)); } // Determine if we need to seek @@ -336,9 +395,29 @@ impl VideoDecoder { input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() ).map_err(|e| e.to_string())?; - let decoder = context_decoder.decoder().video() - .map_err(|e| e.to_string())?; - self.decoder = Some(decoder); + let mut dec_ctx = context_decoder.decoder(); + if hw { + // Attach the VAAPI device + format selector before opening so the decoder + // produces hardware surfaces. + unsafe { + let ctx = dec_ctx.as_mut_ptr(); + let hwdev = self.hw_device.unwrap().0 as *mut ffmpeg::ffi::AVBufferRef; + (*ctx).hw_device_ctx = ffmpeg::ffi::av_buffer_ref(hwdev); + (*ctx).get_format = Some(get_vaapi_format); + } + } + match dec_ctx.video() { + Ok(decoder) => self.decoder = Some(decoder), + Err(e) if hw => { + // Hardware decode unavailable for this clip — fall back to software. This + // frame fails; the next call rebuilds a software decoder. + eprintln!("[Video] hardware decode unavailable ({e}); falling back to software"); + self.hw_failed = true; + self.decoder = None; + return Err(format!("hw decode init failed: {e}")); + } + Err(e) => return Err(e.to_string()), + } } self.input = Some(owned); // Set last_decoded_ts to just before the seek target so forward playback works @@ -351,12 +430,14 @@ impl VideoDecoder { // Decode frames until we find the one closest to our target timestamp let mut best_frame_data: Option> = None; + let mut best_gpu: Option = None; let mut best_frame_ts: Option = None; let t_decode_start = Instant::now(); let mut decode_count = 0; let mut scale_time_ms = 0u128; + let mut hw_import_failed = false; - for (stream, packet) in input.packets() { + 'decode: for (stream, packet) in input.packets() { if stream.index() == self.stream_index { decoder.send_packet(&packet) .map_err(|e| e.to_string())?; @@ -376,73 +457,127 @@ impl VideoDecoder { }; if is_better { - let t_scale_start = Instant::now(); - - // Reuse the RGBA scaler across frames; rebuild only if the input - // format/size or the requested output size changes. - let need_new = match &self.scaler { - Some((fmt, w, h, ow, oh, _)) => { - *fmt != frame.format() || *w != frame.width() || *h != frame.height() - || *ow != out_w || *oh != out_h + if gpu_out { + // Hardware + GPU consumer: import the VAAPI surface as wgpu NV12 textures + // (no CPU copy). + let importer = self.importer.as_ref().unwrap(); + match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } { + Some(gpu) => { + best_gpu = Some(gpu); + best_frame_ts = Some(current_frame_ts); + } + None => { + // Import failed → fall back to software for this clip. + self.hw_failed = true; + hw_import_failed = true; + break 'decode; + } } - None => true, - }; - if need_new { - let ctx = ffmpeg::software::scaling::context::Context::get( - frame.format(), - frame.width(), - frame.height(), - ffmpeg::format::Pixel::RGBA, - out_w, - out_h, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ).map_err(|e| e.to_string())?; - self.scaler = Some((frame.format(), frame.width(), frame.height(), out_w, out_h, SendScaler(ctx))); + } else { + let t_scale_start = Instant::now(); + + // A hardware decoder produces VAAPI surfaces; a CPU consumer (export) + // downloads to system memory first, then swscales like the software path. + let downloaded; + let src: &ffmpeg::util::frame::Video = if hw { + let mut dl = ffmpeg::util::frame::Video::empty(); + let r = unsafe { + ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0) + }; + if r < 0 { + return Err(format!("av_hwframe_transfer_data failed: {r}")); + } + downloaded = dl; + &downloaded + } else { + &frame + }; + + // Reuse the RGBA scaler across frames; rebuild only if the input + // format/size or the requested output size changes. + let need_new = match &self.scaler { + Some((fmt, w, h, ow, oh, _)) => { + *fmt != src.format() || *w != src.width() || *h != src.height() + || *ow != out_w || *oh != out_h + } + None => true, + }; + if need_new { + let ctx = ffmpeg::software::scaling::context::Context::get( + src.format(), + src.width(), + src.height(), + ffmpeg::format::Pixel::RGBA, + out_w, + out_h, + ffmpeg::software::scaling::flag::Flags::BILINEAR, + ).map_err(|e| e.to_string())?; + self.scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx))); + } + let scaler = &mut self.scaler.as_mut().unwrap().5.0; + + let mut rgb_frame = ffmpeg::util::frame::Video::empty(); + scaler.run(src, &mut rgb_frame) + .map_err(|e| e.to_string())?; + + // Remove stride padding to create tightly packed RGBA data + let width = out_w as usize; + let height = out_h as usize; + let stride = rgb_frame.stride(0); + let row_size = width * 4; // RGBA = 4 bytes per pixel + let source_data = rgb_frame.data(0); + + let mut packed_data = Vec::with_capacity(row_size * height); + for y in 0..height { + let row_start = y * stride; + let row_end = row_start + row_size; + packed_data.extend_from_slice(&source_data[row_start..row_end]); + } + + scale_time_ms += t_scale_start.elapsed().as_millis(); + best_frame_data = Some(packed_data); + best_frame_ts = Some(current_frame_ts); } - let scaler = &mut self.scaler.as_mut().unwrap().5.0; - - let mut rgb_frame = ffmpeg::util::frame::Video::empty(); - scaler.run(&frame, &mut rgb_frame) - .map_err(|e| e.to_string())?; - - // Remove stride padding to create tightly packed RGBA data - let width = out_w as usize; - let height = out_h as usize; - let stride = rgb_frame.stride(0); - let row_size = width * 4; // RGBA = 4 bytes per pixel - let source_data = rgb_frame.data(0); - - let mut packed_data = Vec::with_capacity(row_size * height); - for y in 0..height { - let row_start = y * stride; - let row_end = row_start + row_size; - packed_data.extend_from_slice(&source_data[row_start..row_end]); - } - - scale_time_ms += t_scale_start.elapsed().as_millis(); - best_frame_data = Some(packed_data); - best_frame_ts = Some(current_frame_ts); } // If we've reached or passed the target timestamp, we can stop if current_frame_ts >= frame_ts { - // Found our frame, cache and return it - if let Some(data) = best_frame_data { - if video_debug() { - let total_time = t_start.elapsed().as_millis(); - let decode_time = t_decode_start.elapsed().as_millis(); - eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", - timestamp, decode_count, decode_time, scale_time_ms, total_time); - } - self.frame_cache.put(cache_key, data.clone()); - return Ok((data, out_w, out_h)); + if video_debug() { + let total_time = t_start.elapsed().as_millis(); + let decode_time = t_decode_start.elapsed().as_millis(); + eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}", + timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" }); } - break; + if gpu_out { + if let Some(gpu) = best_gpu.take() { + return Ok(DecodedFrame::Gpu(gpu)); + } + } else if let Some(data) = best_frame_data { + self.frame_cache.put(cache_key, data.clone()); + return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h }); + } + break 'decode; } } } } + // Reached EOF without hitting the target, or HW import failed mid-stream. + if hw_import_failed { + self.decoder = None; // force a software rebuild next call (decoder borrow ended here) + self.input = None; + return Err("hardware frame import failed; retrying software".to_string()); + } + // EOF: return the closest frame we found, if any. + if gpu_out { + if let Some(gpu) = best_gpu.take() { + return Ok(DecodedFrame::Gpu(gpu)); + } + } else if let Some(data) = best_frame_data { + self.frame_cache.put(cache_key, data.clone()); + return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h }); + } + eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp); Err("Failed to decode frame".to_string()) } @@ -494,7 +629,8 @@ pub fn generate_keyframe_thumbnails( continue; } // Decode at the thumbnail width (large height so width is the constraint), capped to native. - if let Ok((rgba, _, _)) = decoder.get_frame(ks, thumb_width, 100_000) { + // Thumbnail decoders are always software (no hardware importer). + if let Ok(DecodedFrame::Cpu { rgba, .. }) = decoder.get_frame(ks, thumb_width, 100_000, false) { on_thumb(ks, Arc::new(rgba)); } } @@ -559,10 +695,57 @@ pub fn probe_video(source: &VideoSource) -> Result { pub struct VideoFrame { pub width: u32, pub height: u32, + /// CPU-decoded sRGB RGBA8 (software path). Empty when `gpu` is `Some`. pub rgba_data: Arc>, + /// Hardware-decoded frame living on the GPU (NV12 plane textures). When `Some`, the compositor + /// samples it directly and `rgba_data` is empty. + pub gpu: Option, pub timestamp: f64, } +/// A hardware-decoded video frame on the GPU: two NV12 plane textures (Y = R8, UV = RG8) imported +/// from a VAAPI DMA-BUF on the editor's shared wgpu device. The compositor samples these directly +/// (NV12→RGB), no CPU copy. +#[derive(Clone, Debug)] +pub struct GpuVideoFrame { + pub y: Arc, + pub uv: Arc, + pub width: u32, + pub height: u32, + /// Source YUV range: true = full/PC (0–255), false = limited/TV (16–235). Drives the NV12→RGB + /// offset/scale in the compositor. + pub full_range: bool, +} + +/// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't +/// reference the GPU crate's ffmpeg-sys types) into [`GpuVideoFrame`] textures on the shared device. +/// Implemented by the editor; `gpu-video-encoder` does the actual DMA-BUF import. +pub trait HwVideoImporter: Send + Sync { + /// # Safety + /// `av_frame` must be a valid `*mut ffmpeg_sys_next::AVFrame` holding a VAAPI surface. + unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option; +} + +/// Opaque handle to the FFmpeg VAAPI hardware device (`*mut AVBufferRef`), created by the editor and +/// handed to core so decoders can attach it as `hw_device_ctx`. Core never frees it (the editor owns +/// it for the app's lifetime). +#[derive(Clone, Copy)] +pub struct HwDeviceHandle(pub *mut std::ffi::c_void); +// SAFETY: the pointer is an AVBufferRef whose refcount is managed by FFmpeg; we only `av_buffer_ref` +// it (atomic) and never free it, so sharing the handle across threads is sound. +unsafe impl Send for HwDeviceHandle {} +unsafe impl Sync for HwDeviceHandle {} + +/// Approximate resident bytes of a cached frame for the byte budget: the CPU RGBA buffer, or for a +/// GPU (NV12) frame ~`w*h*3/2` of VRAM, so GPU frames stay bounded too. +fn frame_cache_bytes(frame: &VideoFrame) -> usize { + if frame.gpu.is_some() { + (frame.width as usize * frame.height as usize * 3) / 2 + } else { + frame.rgba_data.len() + } +} + /// Manages video decoders and frame caching for multiple video clips pub struct VideoManager { /// Pool of video decoders, one per clip @@ -572,7 +755,7 @@ pub struct VideoManager { /// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which /// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600) /// so playback of arbitrarily long video never grows unbounded. - frame_cache: LruCache<(Uuid, i64, u32, u32), Arc>, + frame_cache: LruCache<(Uuid, i64, u32, u32, bool), Arc>, /// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len), /// kept in sync on insert/evict/remove so eviction is O(1) per frame. frame_cache_bytes: usize, @@ -588,6 +771,15 @@ pub struct VideoManager { /// Maximum number of frames to cache per decoder cache_size: usize, + + /// Hardware (VAAPI) decode, injected by the editor once the shared device is up. When set, each + /// decoder attaches the VAAPI device and imports frames as GPU textures via `hw_importer`. + hw_device: Option, + hw_importer: Option>, + /// Whether the current render pass can consume GPU textures (preview = true; export = false, + /// since it composites on a different device → a hardware decoder downloads to CPU instead). + /// Set by the render caller before each pass. + render_hardware_ok: bool, } /// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames). @@ -610,9 +802,34 @@ impl VideoManager { thumbnail_cache: HashMap::new(), thumbnails_complete: std::collections::HashSet::new(), cache_size, + hw_device: None, + hw_importer: None, + render_hardware_ok: true, } } + /// Set whether the upcoming render pass can consume GPU video textures (preview = true; export = + /// false). Call before `render_document_for_compositing`. + pub fn set_render_hardware_ok(&mut self, ok: bool) { + self.render_hardware_ok = ok; + } + + /// Enable hardware (VAAPI) decode for all clips. Injected by the editor once the shared wgpu + /// device is active; `hw_device` is the FFmpeg VAAPI device and `importer` imports decoded + /// surfaces as GPU textures on that device. Applies to existing and future decoders. Clears the + /// frame cache (cached CPU frames would otherwise hide the new GPU frames). + pub fn set_hardware_decode(&mut self, hw_device: HwDeviceHandle, importer: Arc) { + self.hw_device = Some(hw_device); + self.hw_importer = Some(Arc::clone(&importer)); + for dec in self.decoders.values() { + if let Ok(mut d) = dec.lock() { + d.set_hardware(hw_device, Arc::clone(&importer)); + } + } + self.frame_cache.clear(); + self.frame_cache_bytes = 0; + } + /// Load a video file and create a decoder for it /// /// `target_width` and `target_height` specify the maximum dimensions @@ -632,7 +849,7 @@ impl VideoManager { let metadata = probe_video(&source)?; // Create decoder with target dimensions, without building keyframe index - let decoder = VideoDecoder::new( + let mut decoder = VideoDecoder::new( source, self.cache_size, Some(target_width), @@ -640,6 +857,11 @@ impl VideoManager { false, // Don't build keyframe index synchronously )?; + // Inherit hardware decode if the manager has it configured. + if let (Some(hw), Some(imp)) = (self.hw_device, &self.hw_importer) { + decoder.set_hardware(hw, Arc::clone(imp)); + } + // Store decoder in pool self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder))); @@ -648,14 +870,16 @@ impl VideoManager { /// Get a decoded frame for a specific clip at a specific timestamp /// - /// Returns None if the clip is not loaded or decoding fails. - /// Frames are cached for performance. + /// Returns None if the clip is not loaded or decoding fails. Frames are cached. + /// Whether a hardware decoder returns a GPU texture or downloads to CPU RGBA depends on + /// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the + /// preview, false for export, which composites on a different device). pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { - // Convert timestamp to milliseconds for cache key. The target size is part of the key: the - // canvas (preview res) and an in-progress export (export res) request the same clip/time at - // different sizes, and must not collide. + let hardware_ok = self.render_hardware_ok; + // The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export + // (CPU, export res) request the same clip/time and must not collide or cross representation. let timestamp_ms = (timestamp * 1000.0) as i64; - let cache_key = (*clip_id, timestamp_ms, target_w, target_h); + let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok); // Check frame cache first if let Some(cached_frame) = self.frame_cache.get(&cache_key) { @@ -668,15 +892,25 @@ impl VideoManager { let mut decoder = decoder_arc.lock().ok()?; // Decode the frame at the requested target (capped to native by the decoder). - let (rgba_data, width, height) = decoder.get_frame(timestamp, target_w, target_h).ok()?; + let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?; drop(decoder); // release the lock before touching `self` - // Create VideoFrame and cache it - let frame = Arc::new(VideoFrame { - width, - height, - rgba_data: Arc::new(rgba_data), - timestamp, + // Create VideoFrame and cache it. + let frame = Arc::new(match decoded { + DecodedFrame::Cpu { rgba, width, height } => VideoFrame { + width, + height, + rgba_data: Arc::new(rgba), + gpu: None, + timestamp, + }, + DecodedFrame::Gpu(gpu) => VideoFrame { + width: gpu.width, + height: gpu.height, + rgba_data: Arc::new(Vec::new()), + gpu: Some(gpu), + timestamp, + }, }); self.cache_frame(cache_key, Arc::clone(&frame)); @@ -686,16 +920,16 @@ impl VideoManager { /// Insert a frame into the byte-budgeted cache, evicting least-recently-used /// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`]. - fn cache_frame(&mut self, key: (Uuid, i64, u32, u32), frame: Arc) { - let bytes = frame.rgba_data.len(); + fn cache_frame(&mut self, key: (Uuid, i64, u32, u32, bool), frame: Arc) { + let bytes = frame_cache_bytes(&frame); if let Some(old) = self.frame_cache.put(key, frame) { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&old)); } self.frame_cache_bytes += bytes; // Keep at least one frame resident even if it alone exceeds the budget. while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 { if let Some((_, evicted)) = self.frame_cache.pop_lru() { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(evicted.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&evicted)); } else { break; } @@ -802,15 +1036,15 @@ impl VideoManager { // Remove all cached frames for this clip (LruCache has no retain; collect // matching keys, then pop each, keeping the byte total in sync). - let keys: Vec<(Uuid, i64, u32, u32)> = self + let keys: Vec<(Uuid, i64, u32, u32, bool)> = self .frame_cache .iter() - .filter(|((id, _, _, _), _)| id == clip_id) + .filter(|((id, _, _, _, _), _)| id == clip_id) .map(|(k, _)| *k) .collect(); for key in keys { if let Some(frame) = self.frame_cache.pop(&key) { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&frame)); } } From 1848c920d9d03afa137adc54b32c92633d9c5f88 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:21:14 -0400 Subject: [PATCH 09/31] editor: wire hardware video decode + NV12 preview compositing (Stage 3c-preview) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the dormant core HW-decode engine live for the preview path: - hw_video.rs: editor's HwVideoImporter — maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane textures on the *shared* device (the only one with the import extensions). install() creates the VAAPI device and injects it + the importer into the VideoManager. - main.rs: track whether the shared device is actually in use; only then (Linux, not LB_NO_SHARED_DEVICE) install hardware decode, using the CreationContext's shared device + adapter. - nv12_blit.rs + nv12_blit.wgsl: NV12 plane textures → BT.709 → sRGB-encoded → linear, written straight into the Rgba16Float HDR layer (no CPU upload). Colour math mirrors the software path so HW/SW video match; honours full_range. - stage.rs: the preview Video arm branches on inst.gpu (NV12 blit) vs rgba_data (existing upload+blit_straight); sets render_hardware_ok = !cpu_renderer so the CPU fallback still gets software frames. - video_exporter.rs: sets render_hardware_ok(false) before both compositing passes — export composites on the encoder's separate device, so a hardware decoder downloads to CPU instead (export stays software, correct). - dmabuf.rs: imported plane textures now also carry SAMPLED/TEXTURE_BINDING so they can be sampled by the NV12 blit (they were render-target-only); into_planes hands the textures to the longer-lived GpuVideoFrame. - video.rs: cache-key the GPU/CPU representation on want_gpu (HW-configured AND render_hardware_ok) so software-only decode keeps a single cache entry. Preview only this pass; export GPU-residency is the 3c-export follow-up. Untested at runtime here (no GPU/display in container) — both crates compile. --- .../gpu-video-encoder/src/dmabuf.rs | 13 +- .../lightningbeam-core/src/video.rs | 10 +- .../src/export/video_exporter.rs | 12 ++ .../lightningbeam-editor/src/hw_video.rs | 89 +++++++++ .../lightningbeam-editor/src/main.rs | 15 ++ .../lightningbeam-editor/src/nv12_blit.rs | 180 ++++++++++++++++++ .../src/panes/shaders/nv12_blit.wgsl | 84 ++++++++ .../lightningbeam-editor/src/panes/stage.rs | 38 +++- 8 files changed, 425 insertions(+), 16 deletions(-) create mode 100644 lightningbeam-ui/lightningbeam-editor/src/hw_video.rs create mode 100644 lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs create mode 100644 lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl diff --git a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs index a6e1644..492e0b4 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs @@ -47,6 +47,10 @@ impl ImportedNv12 { pub fn uv(&self) -> &wgpu::Texture { &self.uv } + /// Consume into the `(Y, UV)` plane textures (for handing to a longer-lived owner). + pub fn into_planes(self) -> (wgpu::Texture, wgpu::Texture) { + (self.y, self.uv) + } } /// Convenience: map a freshly-allocated `MappedSurface` and import it onto `drm`. @@ -113,6 +117,7 @@ pub fn import_raw( .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) .usage( vk::ImageUsageFlags::COLOR_ATTACHMENT + | vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_SRC | vk::ImageUsageFlags::TRANSFER_DST, ) @@ -178,7 +183,9 @@ pub fn import_raw( sample_count: 1, dimension: wgpu::TextureDimension::D2, format, - usage: wgpu_types::TextureUses::COLOR_TARGET | wgpu_types::TextureUses::COPY_SRC, + usage: wgpu_types::TextureUses::COLOR_TARGET + | wgpu_types::TextureUses::RESOURCE + | wgpu_types::TextureUses::COPY_SRC, memory_flags: wgpu_hal::MemoryFlags::empty(), view_formats: vec![], }; @@ -192,7 +199,9 @@ pub fn import_raw( sample_count: 1, dimension: wgpu::TextureDimension::D2, format, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, view_formats: &[], }, ) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 0ddd083..958f197 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -875,11 +875,13 @@ impl VideoManager { /// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the /// preview, false for export, which composites on a different device). pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { - let hardware_ok = self.render_hardware_ok; - // The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export + // Whether this pass wants (and can produce) a GPU frame. Gated on HW being configured at all + // so that with software-only decode preview and export share one cache entry (no double-cache). + let want_gpu = self.render_hardware_ok && self.hw_device.is_some(); + // The cache key includes (target size, want_gpu): preview (GPU, preview res) and export // (CPU, export res) request the same clip/time and must not collide or cross representation. let timestamp_ms = (timestamp * 1000.0) as i64; - let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok); + let cache_key = (*clip_id, timestamp_ms, target_w, target_h, want_gpu); // Check frame cache first if let Some(cached_frame) = self.frame_cache.get(&cache_key) { @@ -892,7 +894,7 @@ impl VideoManager { let mut decoder = decoder_arc.lock().ok()?; // Decode the frame at the requested target (capped to native by the decoder). - let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?; + let decoded = decoder.get_frame(timestamp, target_w, target_h, want_gpu).ok()?; drop(decoder); // release the lock before touching `self` // Create VideoFrame and cache it. diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index efa7990..a19a227 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -1066,6 +1066,13 @@ pub fn render_frame_to_rgba_hdr( Affine::IDENTITY }; + // Export composites on the encoder's own device, not the shared one, so it cannot use + // hardware-decoded GPU frames (textures can't cross devices). Force software frames; a + // hardware decoder downloads its surface to CPU instead. + if let Ok(mut vm) = video_manager.lock() { + vm.set_render_hardware_ok(false); + } + // Render document for compositing (returns per-layer scenes) let composite_result = render_document_for_compositing( document, @@ -1329,6 +1336,11 @@ pub fn render_frame_to_gpu_rgba( Affine::IDENTITY }; + // Export composites on a separate device — force software frames (see above). + if let Ok(mut vm) = video_manager.lock() { + vm.set_render_hardware_ok(false); + } + // Render document for compositing (returns per-layer scenes) let composite_result = render_document_for_compositing( document, diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs new file mode 100644 index 0000000..7032ae8 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -0,0 +1,89 @@ +//! Hardware video decode glue (Linux/VAAPI). The editor implements core's [`HwVideoImporter`]: +//! it maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane +//! textures on the **shared** device (the one eframe + the compositor run on, which has the +//! DMA-BUF-import extensions). [`install`] creates the VAAPI device and wires it into the +//! `VideoManager`. + +use ffmpeg_next::ffi as ff; +use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf}; +use lightningbeam_core::video::{GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager}; +use std::sync::{Arc, Mutex}; + +/// Imports decoded VAAPI surfaces onto the shared wgpu device. Holds clones of the shared +/// device + adapter (Arc-backed, cheap). +struct SharedHwImporter { + device: wgpu::Device, + adapter: wgpu::Adapter, +} + +impl HwVideoImporter for SharedHwImporter { + unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option { + let frame = av_frame as *mut ff::AVFrame; + + // Map the VAAPI surface to a DRM-PRIME DMA-BUF (read-only). + let drm_f = ff::av_frame_alloc(); + (*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + let flags = ff::AV_HWFRAME_MAP_DIRECT as i32 | ff::AV_HWFRAME_MAP_READ as i32; + if ff::av_hwframe_map(drm_f, frame, flags) < 0 { + ff::av_frame_free(&mut (drm_f as *mut _)); + return None; + } + + let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor; + let obj = &(*desc).objects[0]; + let width = (*frame).width as u32; + let height = (*frame).height as u32; + // NV12: Y then UV — two layers (one plane each) or one layer with two planes. + let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 { + (&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0]) + } else { + (&(*desc).layers[0].planes[0], &(*desc).layers[0].planes[1]) + }; + let buf = Nv12DmaBuf { + fd: obj.fd, + size: obj.size as u64, + modifier: obj.format_modifier, + width, + height, + y_offset: y_pl.offset as u64, + y_pitch: y_pl.pitch as u64, + uv_offset: uv_pl.offset as u64, + uv_pitch: uv_pl.pitch as u64, + }; + let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG; + + let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); + ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan + let (y, uv) = imported.ok()?.into_planes(); + Some(GpuVideoFrame { + y: Arc::new(y), + uv: Arc::new(uv), + width, + height, + full_range, + }) + } +} + +/// Create the VAAPI hardware device and install hardware decode into `vm`, importing onto the +/// shared `device`/`adapter`. Logs and no-ops if VAAPI is unavailable (→ software decode). +pub fn install(vm: &Arc>, device: &wgpu::Device, adapter: &wgpu::Adapter) { + match gpu_video_encoder::vaapi::create_device() { + Ok(hw_device) => { + let importer = Arc::new(SharedHwImporter { + device: device.clone(), + adapter: adapter.clone(), + }); + if let Ok(mut vm) = vm.lock() { + vm.set_hardware_decode( + HwDeviceHandle(hw_device as *mut std::ffi::c_void), + importer, + ); + } + println!("🎞 Hardware video decode enabled (VAAPI → shared device)"); + } + Err(e) => { + println!("🎞 Hardware video decode unavailable ({e}); using software decode"); + } + } +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 08a522a..d9152ad 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -51,6 +51,9 @@ mod custom_cursor; mod tablet; mod debug_overlay; mod gpu_timer; +#[cfg(target_os = "linux")] +mod hw_video; +mod nv12_blit; #[cfg(debug_assertions)] mod test_mode; @@ -210,6 +213,10 @@ fn main() -> eframe::Result { // raw VkDevice persists with them. #[allow(unused_mut)] let mut wgpu_setup = lb_default_wgpu_setup(); + // Whether the shared VAAPI-capable device is in use — only then can decoded DMA-BUF frames be + // imported + composited, so hardware decode is injected into the VideoManager only if true. + #[allow(unused_assignments, unused_mut)] + let mut shared_device_active = false; #[cfg(target_os = "linux")] if std::env::var("LB_NO_SHARED_DEVICE").is_ok() { println!("🖥 Shared device disabled via LB_NO_SHARED_DEVICE; default wgpu device (software video decode)"); @@ -223,6 +230,7 @@ fn main() -> eframe::Result { device: drm.device.clone(), queue: drm.queue.clone(), }); + shared_device_active = true; } Err(e) => { println!("🖥 Shared device unavailable ({e}); default wgpu device (software video decode)"); @@ -294,6 +302,13 @@ fn main() -> eframe::Result { let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); #[cfg(not(debug_assertions))] let app = EditorApp::new(cc, layouts, theme); + // Wire hardware video decode into the VideoManager now that the shared device exists. + #[cfg(target_os = "linux")] + if shared_device_active { + if let Some(rs) = cc.wgpu_render_state.as_ref() { + hw_video::install(&app.video_manager, &rs.device, &rs.adapter); + } + } Ok(Box::new(app)) }), ) diff --git a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs new file mode 100644 index 0000000..3f26640 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs @@ -0,0 +1,180 @@ +//! NV12 → linear-RGB blit: composites a hardware-decoded video frame (two wgpu plane textures, +//! Y = R8Unorm + CbCr = Rg8Unorm) directly into the Rgba16Float HDR layer, with no CPU upload. +//! The colour math mirrors the software path (BT.709 → sRGB-encoded → linear) so hardware- and +//! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`. + +use crate::gpu_brush::BlitTransform; + +/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]) plus the +/// full-range flag. 64 bytes (48 for the matrix + a u32 + 12 padding). +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Nv12Params { + transform: BlitTransform, + full_range: u32, + _pad: [u32; 3], +} + +pub struct Nv12BlitPipeline { + pipeline: wgpu::RenderPipeline, + bg_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, +} + +impl Nv12BlitPipeline { + pub fn new(device: &wgpu::Device) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("nv12_blit_shader"), + source: wgpu::ShaderSource::Wgsl( + include_str!("panes/shaders/nv12_blit.wgsl").into(), + ), + }); + + let tex_entry = |binding: u32| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + + let bg_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("nv12_blit_bgl"), + entries: &[ + tex_entry(0), // Y plane (R8Unorm) + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + tex_entry(3), // CbCr plane (Rg8Unorm) + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("nv12_blit_pl"), + bind_group_layouts: &[&bg_layout], + push_constant_ranges: &[], + }); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("nv12_blit_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba16Float, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + // Bilinear: the frame is scaled to the output size; nearest would look blocky. + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("nv12_blit_sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + Self { pipeline, bg_layout, sampler } + } + + /// Convert + blit the NV12 frame into `target_view` (Rgba16Float, cleared to transparent), + /// positioned by `transform` (built like the RGBA video path's `BlitTransform`). + #[allow(clippy::too_many_arguments)] + pub fn blit( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + y_view: &wgpu::TextureView, + uv_view: &wgpu::TextureView, + target_view: &wgpu::TextureView, + transform: &BlitTransform, + full_range: bool, + ) { + let params = Nv12Params { + transform: *transform, + full_range: full_range as u32, + _pad: [0; 3], + }; + let param_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("nv12_blit_params"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + queue.write_buffer(¶m_buf, 0, bytemuck::bytes_of(¶ms)); + + let bg = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("nv12_blit_bg"), + layout: &self.bg_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(y_view) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&self.sampler) }, + wgpu::BindGroupEntry { binding: 2, resource: param_buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(uv_view) }, + ], + }); + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("nv12_blit_encoder"), + }); + { + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("nv12_blit_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: target_view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + }); + rp.set_pipeline(&self.pipeline); + rp.set_bind_group(0, &bg, &[]); + rp.draw(0..4, 0..1); + } + queue.submit(Some(encoder.finish())); + } +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl new file mode 100644 index 0000000..46d5d79 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -0,0 +1,84 @@ +// NV12 → linear-RGB blit shader. +// +// Samples a hardware-decoded video frame stored as two planes — Y (R8Unorm) and +// interleaved CbCr (Rg8Unorm, half-res) — converts BT.709 Y'CbCr → gamma-encoded +// R'G'B', then sRGB→linear, and writes straight-alpha linear into the Rgba16Float +// HDR layer. This mirrors the software path (swscale → sRGB RGBA8 → sampled as +// Rgba8UnormSrgb → linear) so hardware- and software-decoded video match. +// +// The affine transform (viewport UV → frame UV) is the same packing as +// canvas_blit.wgsl's BlitTransform; `full_range` selects full vs. studio-swing +// de-quantization. + +struct Nv12Params { + col0: vec4, + col1: vec4, + col2: vec4, + full_range: u32, + _pad: vec3, +} + +@group(0) @binding(0) var y_tex: texture_2d; +@group(0) @binding(1) var samp: sampler; +@group(0) @binding(2) var params: Nv12Params; +@group(0) @binding(3) var uv_tex: texture_2d; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var out: VertexOutput; + let x = f32((vertex_index & 1u) << 1u); + let y = f32(vertex_index & 2u); + out.position = vec4(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0); + out.uv = vec2(x, y); + return out; +} + +fn srgb_to_linear(c: vec3) -> vec3 { + let lo = c / 12.92; + let hi = pow((c + vec3(0.055)) / 1.055, vec3(2.4)); + return select(lo, hi, c > vec3(0.04045)); +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let m = mat3x3(params.col0.xyz, params.col1.xyz, params.col2.xyz); + let frame_uv = (m * vec3(in.uv.x, in.uv.y, 1.0)).xy; + + if frame_uv.x < 0.0 || frame_uv.x > 1.0 + || frame_uv.y < 0.0 || frame_uv.y > 1.0 { + return vec4(0.0, 0.0, 0.0, 0.0); + } + + let yv = textureSample(y_tex, samp, frame_uv).r; + let cbcr = textureSample(uv_tex, samp, frame_uv).rg; + + var Y: f32; + var Cb: f32; + var Cr: f32; + if params.full_range != 0u { + // Full ("JPEG") range: [0,255] luma, chroma centered at 128. + Y = yv; + Cb = cbcr.r - 0.5; + Cr = cbcr.g - 0.5; + } else { + // Studio swing: Y'∈[16,235], Cb/Cr∈[16,240]. + Y = (yv * 255.0 - 16.0) / 219.0; + Cb = (cbcr.r * 255.0 - 128.0) / 224.0; + Cr = (cbcr.g * 255.0 - 128.0) / 224.0; + } + + // BT.709 Y'CbCr → gamma-encoded R'G'B'. + let r = Y + 1.5748 * Cr; + let g = Y - 0.1873 * Cb - 0.4681 * Cr; + let b = Y + 1.8556 * Cb; + let rgb_gamma = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); + + // R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer. + let rgb_lin = srgb_to_linear(rgb_gamma); + return vec4(rgb_lin, 1.0); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 5bd3181..de42f0c 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -152,6 +152,8 @@ struct SharedVelloResources { gpu_brush: Mutex, /// Canvas blit pipeline (renders GPU canvas to layer sRGB buffer) canvas_blit: crate::gpu_brush::CanvasBlitPipeline, + /// NV12→linear blit for hardware-decoded video frames (GPU plane textures → HDR layer). + nv12_blit: crate::nv12_blit::Nv12BlitPipeline, /// True when Vello is running its CPU software renderer (either forced or GPU fallback). /// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area. is_cpu_renderer: bool, @@ -372,6 +374,7 @@ impl SharedVelloResources { // Initialize GPU raster brush engine let gpu_brush = crate::gpu_brush::GpuBrushEngine::new(device); let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device); + let nv12_blit = crate::nv12_blit::Nv12BlitPipeline::new(device); println!("✅ Vello shared resources initialized (renderer, shaders, HDR compositor, effect processor, color converter, and GPU brush engine)"); @@ -389,6 +392,7 @@ impl SharedVelloResources { srgb_to_linear, gpu_brush: Mutex::new(gpu_brush), canvas_blit, + nv12_blit, is_cpu_renderer: use_cpu || is_cpu_renderer, gpu_timer: Mutex::new(None), video_frame_cache: Mutex::new(VideoFrameTexCache::new()), @@ -1062,6 +1066,12 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // Let the cache page image bytes from the project container on a decode miss. image_cache.set_container_path(self.ctx.container_path.clone()); + // Preview composites on the shared device, so it can consume hardware-decoded GPU + // frames — but only the GPU renderer; the CPU fallback needs software frames. + if let Ok(mut vm) = shared.video_manager.lock() { + vm.set_render_hardware_ok(!shared.is_cpu_renderer); + } + let composite_result = if shared.is_cpu_renderer { lightningbeam_core::renderer::render_document_for_compositing_cpu( &self.ctx.document, @@ -1678,25 +1688,33 @@ impl egui_wgpu::CallbackTrait for VelloCallback { RenderedLayerType::Video { instances } => { // Video layer — per-instance: (cached) frame texture → blit → composite. for inst in instances { - if inst.rgba_data.is_empty() { continue; } + if inst.gpu.is_none() && inst.rgba_data.is_empty() { continue; } let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); if let (Some(hdr_layer_view), Some(hdr_view)) = ( buffer_pool.get_view(hdr_layer_handle), &instance_resources.hdr_texture_view, ) { - // Reuse the GPU texture for this frame if it's unchanged (a - // static/paused video → no CPU conversion, alloc, or upload). - // Timed into `blit_ms` (incl the cache lookup + per-frame view). let _t = std::time::Instant::now(); - let tex_view = shared - .video_frame_cache - .lock() - .unwrap() - .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height); let bt = crate::gpu_brush::BlitTransform::new( inst.transform, inst.width, inst.height, width, height, ); - shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + if let Some(gpu) = &inst.gpu { + // Hardware-decoded NV12 plane textures → linear RGB, no CPU upload. + let y_view = gpu.y.create_view(&Default::default()); + let uv_view = gpu.uv.create_view(&Default::default()); + shared.nv12_blit.blit( + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, + ); + } else { + // Reuse the GPU texture for this frame if it's unchanged (a + // static/paused video → no CPU conversion, alloc, or upload). + let tex_view = shared + .video_frame_cache + .lock() + .unwrap() + .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height); + shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + } cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0; let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new( From 1c537d99da664d823167d58104ebebb075e2dcae Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:26:39 -0400 Subject: [PATCH 10/31] fix: nv12_blit uniform size mismatch (64 vs 80 bytes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WGSL struct trailed `full_range: u32` with `_pad: vec3`, but vec3 has 16-byte alignment so the struct rounded up to 80 bytes while the Rust-side Nv12Params (BlitTransform + u32 + [u32;3]) is 64 — wgpu rejected the draw ("Buffer is bound with size 64 where the shader expects 80"). Pack the flag as a single `flags: vec4` (.x = full_range) so both sides are 64 bytes. --- .../lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl index 46d5d79..5a7e3f7 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -14,8 +14,9 @@ struct Nv12Params { col0: vec4, col1: vec4, col2: vec4, - full_range: u32, - _pad: vec3, + // .x = full_range flag; .yzw padding. A vec4 keeps the struct 64 bytes (matching the Rust + // `u32 + [u32; 3]`); a bare u32 + vec3 would round up to 80 and mismatch the bound buffer. + flags: vec4, } @group(0) @binding(0) var y_tex: texture_2d; @@ -60,7 +61,7 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { var Y: f32; var Cb: f32; var Cr: f32; - if params.full_range != 0u { + if params.flags.x != 0u { // Full ("JPEG") range: [0,255] luma, chroma centered at 128. Y = yv; Cb = cbcr.r - 0.5; From 6348e57de0af71089ba8604998fd24392f8cc691 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:32:05 -0400 Subject: [PATCH 11/31] nv12: convert with the source colorspace matrix (not hardcoded BT.709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NV12→RGB pass hardcoded BT.709, so SD (BT.601) clips had slightly wrong hues. Read each frame's AVColorSpace in the importer and derive the Y'CbCr→R'G'B' matrix (BT.709/601/240M/2020; Unspecified guessed by height like swscale/players), carry the four coefficients on GpuVideoFrame, and apply them in the shader. - core: GpuVideoFrame.coeffs + ycbcr_coeffs(kr, kb) helper. - hw_video.rs: map AVColorSpace → (kr, kb) → coeffs. - nv12_blit{.rs,.wgsl}: uniform grows to 80 bytes (adds a coeffs vec4); the matrix multiply uses params.coeffs instead of literals. BT.2020's transfer is still approximated as sRGB. The DRM-modifier-without-SAMPLED case stays a graceful software fallback. Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-core/src/video.rs | 16 +++++++++++ .../lightningbeam-editor/src/hw_video.rs | 28 ++++++++++++++++++- .../lightningbeam-editor/src/nv12_blit.rs | 7 +++-- .../src/panes/shaders/nv12_blit.wgsl | 14 ++++++---- .../lightningbeam-editor/src/panes/stage.rs | 2 +- 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 958f197..83cc257 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -715,6 +715,22 @@ pub struct GpuVideoFrame { /// Source YUV range: true = full/PC (0–255), false = limited/TV (16–235). Drives the NV12→RGB /// offset/scale in the compositor. pub full_range: bool, + /// Y'CbCr→R'G'B' matrix coefficients derived from the frame's colorspace (BT.709/601/2020), + /// so SD (BT.601) and HD/UHD clips each convert correctly: `[Cr→R, Cb→G, Cr→G, Cb→B]`. + /// R = Y + c[0]·Cr, G = Y + c[1]·Cb + c[2]·Cr, B = Y + c[3]·Cb + pub coeffs: [f32; 4], +} + +/// Y'CbCr→R'G'B' matrix coefficients (`[Cr→R, Cb→G, Cr→G, Cb→B]`) from the luma weights `kr`/`kb` +/// (`kg = 1−kr−kb`). BT.709 → `[1.5748, −0.1873, −0.4681, 1.8556]`. +pub fn ycbcr_coeffs(kr: f32, kb: f32) -> [f32; 4] { + let kg = 1.0 - kr - kb; + [ + 2.0 * (1.0 - kr), + -2.0 * kb * (1.0 - kb) / kg, + -2.0 * kr * (1.0 - kr) / kg, + 2.0 * (1.0 - kb), + ] } /// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs index 7032ae8..b32d963 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -6,7 +6,9 @@ use ffmpeg_next::ffi as ff; use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf}; -use lightningbeam_core::video::{GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager}; +use lightningbeam_core::video::{ + ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, +}; use std::sync::{Arc, Mutex}; /// Imports decoded VAAPI surfaces onto the shared wgpu device. Holds clones of the shared @@ -52,6 +54,29 @@ impl HwVideoImporter for SharedHwImporter { }; let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG; + // Luma weights (kr, kb) from the frame's matrix coefficients, so SD (BT.601) and HD/UHD + // (BT.709) clips each convert with the right matrix. Unspecified → guess by height, as + // players/swscale do. SMPTE240M and BT.2020 are handled too (the latter's transfer is still + // approximated as sRGB — fine for SDR; true HDR is out of scope). + let (kr, kb) = match (*frame).colorspace { + ff::AVColorSpace::AVCOL_SPC_BT709 => (0.2126, 0.0722), + ff::AVColorSpace::AVCOL_SPC_BT470BG | ff::AVColorSpace::AVCOL_SPC_SMPTE170M => { + (0.299, 0.114) + } + ff::AVColorSpace::AVCOL_SPC_SMPTE240M => (0.212, 0.087), + ff::AVColorSpace::AVCOL_SPC_BT2020_NCL | ff::AVColorSpace::AVCOL_SPC_BT2020_CL => { + (0.2627, 0.0593) + } + _ => { + if height <= 576 { + (0.299, 0.114) // SD → BT.601 + } else { + (0.2126, 0.0722) // HD/UHD → BT.709 + } + } + }; + let coeffs = ycbcr_coeffs(kr, kb); + let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan let (y, uv) = imported.ok()?.into_planes(); @@ -61,6 +86,7 @@ impl HwVideoImporter for SharedHwImporter { width, height, full_range, + coeffs, }) } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs index 3f26640..17d8900 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs @@ -5,12 +5,13 @@ use crate::gpu_brush::BlitTransform; -/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]) plus the -/// full-range flag. 64 bytes (48 for the matrix + a u32 + 12 padding). +/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]), the Y'CbCr→RGB +/// matrix coefficients, and the full-range flag. 80 bytes (48 matrix + 16 coeffs + u32 + 12 pad). #[repr(C)] #[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] struct Nv12Params { transform: BlitTransform, + coeffs: [f32; 4], full_range: u32, _pad: [u32; 3], } @@ -127,9 +128,11 @@ impl Nv12BlitPipeline { target_view: &wgpu::TextureView, transform: &BlitTransform, full_range: bool, + coeffs: [f32; 4], ) { let params = Nv12Params { transform: *transform, + coeffs, full_range: full_range as u32, _pad: [0; 3], }; diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl index 5a7e3f7..33dc8d4 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -14,8 +14,10 @@ struct Nv12Params { col0: vec4, col1: vec4, col2: vec4, - // .x = full_range flag; .yzw padding. A vec4 keeps the struct 64 bytes (matching the Rust - // `u32 + [u32; 3]`); a bare u32 + vec3 would round up to 80 and mismatch the bound buffer. + // Y'CbCr→R'G'B' matrix from the source colorspace: [Cr→R, Cb→G, Cr→G, Cb→B]. + coeffs: vec4, + // .x = full_range flag; .yzw padding. A vec4 keeps each block 16-aligned and the struct size + // matching the Rust `[f32;4] + u32 + [u32;3]` (80 bytes). flags: vec4, } @@ -73,10 +75,10 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { Cr = (cbcr.g * 255.0 - 128.0) / 224.0; } - // BT.709 Y'CbCr → gamma-encoded R'G'B'. - let r = Y + 1.5748 * Cr; - let g = Y - 0.1873 * Cb - 0.4681 * Cr; - let b = Y + 1.8556 * Cb; + // Y'CbCr → gamma-encoded R'G'B' using the source colorspace's matrix. + let r = Y + params.coeffs.x * Cr; + let g = Y + params.coeffs.y * Cb + params.coeffs.z * Cr; + let b = Y + params.coeffs.w * Cb; let rgb_gamma = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); // R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer. diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index de42f0c..4a52a03 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -1703,7 +1703,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback { let y_view = gpu.y.create_view(&Default::default()); let uv_view = gpu.uv.create_view(&Default::default()); shared.nv12_blit.blit( - device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, gpu.coeffs, ); } else { // Reuse the GPU texture for this frame if it's unchanged (a From ff490ab9ae1c2c3f47f1f6e4d3e80193978d5ec6 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:47:05 -0400 Subject: [PATCH 12/31] =?UTF-8?q?nv12:=20HDR-correct=20input=20=E2=80=94?= =?UTF-8?q?=20PQ/HLG=20EOTF=20+=20BT.2020=E2=86=92709=20gamut=20(Stage=20A?= =?UTF-8?q?=20pt=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode HDR video into the linear compositor correctly instead of approximating everything as sRGB/BT.709: - Read the frame's color_trc and color_primaries in the importer → VideoTransfer {Gamma,Pq,Hlg} + VideoPrimaries {Bt709,Bt2020} on GpuVideoFrame. - nv12_blit.wgsl: branch the EOTF — sRGB gamma (SDR), SMPTE2084 PQ (normalized so 203-nit graphics white = 1.0; highlights exceed 1.0), or HLG inverse-OETF (reference white ≈ 1.0). Then BT.2020→BT.709 primaries in linear light when wide-gamut, clamping out-of-709 colours. Establishes the white=1.0 scene-linear convention: SDR content is unchanged (stays in [0,1]); HDR video carries super-white highlights through compositing. SDR-output mapping (clip default vs highlight rolloff) is Part 2. HLG's display OOTF is omitted (scene-referred) — approximate but reasonable for SDR-out. --- .../lightningbeam-core/src/video.rs | 25 ++++++++ .../lightningbeam-editor/src/hw_video.rs | 25 +++++++- .../lightningbeam-editor/src/nv12_blit.rs | 21 +++++-- .../src/panes/shaders/nv12_blit.wgsl | 59 +++++++++++++++++-- .../lightningbeam-editor/src/panes/stage.rs | 3 +- 5 files changed, 121 insertions(+), 12 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 83cc257..31c31ce 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -719,6 +719,31 @@ pub struct GpuVideoFrame { /// so SD (BT.601) and HD/UHD clips each convert correctly: `[Cr→R, Cb→G, Cr→G, Cb→B]`. /// R = Y + c[0]·Cr, G = Y + c[1]·Cb + c[2]·Cr, B = Y + c[3]·Cb pub coeffs: [f32; 4], + /// Opto-electronic transfer of the encoded R'G'B' — the compositor applies the matching EOTF to + /// reach scene-linear (graphics white = 1.0). HDR (PQ/HLG) values exceed 1.0. + pub transfer: VideoTransfer, + /// Colour primaries; BT.2020 is gamut-mapped to the compositor's BT.709 space in linear light. + pub primaries: VideoPrimaries, +} + +/// Transfer characteristic of a decoded video frame (selects the EOTF in the NV12→linear pass). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VideoTransfer { + /// SDR gamma (BT.709/sRGB/601/gamma22) — approximated by the sRGB EOTF. + Gamma, + /// SMPTE ST 2084 (PQ) — absolute, normalized so 203 nits (graphics white) = 1.0. + Pq, + /// ARIB STD-B67 (HLG) — scene-referred, normalized so reference white ≈ 1.0. + Hlg, +} + +/// Colour primaries of a decoded video frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VideoPrimaries { + /// BT.709 / sRGB (also used for BT.601, whose primaries differ only slightly). + Bt709, + /// BT.2020 (wide gamut) — converted to BT.709 in linear light by the compositor. + Bt2020, } /// Y'CbCr→R'G'B' matrix coefficients (`[Cr→R, Cb→G, Cr→G, Cb→B]`) from the luma weights `kr`/`kb` diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs index b32d963..d3f915d 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -7,7 +7,8 @@ use ffmpeg_next::ffi as ff; use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf}; use lightningbeam_core::video::{ - ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, + ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, VideoPrimaries, + VideoTransfer, }; use std::sync::{Arc, Mutex}; @@ -77,6 +78,26 @@ impl HwVideoImporter for SharedHwImporter { }; let coeffs = ycbcr_coeffs(kr, kb); + // Transfer characteristic → which EOTF the compositor applies to reach scene-linear. + let transfer = match (*frame).color_trc { + ff::AVColorTransferCharacteristic::AVCOL_TRC_SMPTE2084 => VideoTransfer::Pq, + ff::AVColorTransferCharacteristic::AVCOL_TRC_ARIB_STD_B67 => VideoTransfer::Hlg, + _ => VideoTransfer::Gamma, + }; + // Primaries → BT.2020 is gamut-mapped to BT.709; unspecified follows the matrix guess above. + let primaries = match (*frame).color_primaries { + ff::AVColorPrimaries::AVCOL_PRI_BT2020 => VideoPrimaries::Bt2020, + ff::AVColorPrimaries::AVCOL_PRI_UNSPECIFIED + if matches!( + (*frame).colorspace, + ff::AVColorSpace::AVCOL_SPC_BT2020_NCL | ff::AVColorSpace::AVCOL_SPC_BT2020_CL + ) => + { + VideoPrimaries::Bt2020 + } + _ => VideoPrimaries::Bt709, + }; + let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan let (y, uv) = imported.ok()?.into_planes(); @@ -87,6 +108,8 @@ impl HwVideoImporter for SharedHwImporter { height, full_range, coeffs, + transfer, + primaries, }) } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs index 17d8900..c7949f7 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs @@ -4,16 +4,17 @@ //! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`. use crate::gpu_brush::BlitTransform; +use lightningbeam_core::video::{VideoPrimaries, VideoTransfer}; /// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]), the Y'CbCr→RGB -/// matrix coefficients, and the full-range flag. 80 bytes (48 matrix + 16 coeffs + u32 + 12 pad). +/// matrix coefficients, and a flags vec4. 80 bytes (48 matrix + 16 coeffs + 16 flags). +/// `flags`: `[full_range, transfer (0 gamma / 1 PQ / 2 HLG), primaries (0 BT.709 / 1 BT.2020), pad]`. #[repr(C)] #[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] struct Nv12Params { transform: BlitTransform, coeffs: [f32; 4], - full_range: u32, - _pad: [u32; 3], + flags: [u32; 4], } pub struct Nv12BlitPipeline { @@ -129,12 +130,22 @@ impl Nv12BlitPipeline { transform: &BlitTransform, full_range: bool, coeffs: [f32; 4], + transfer: VideoTransfer, + primaries: VideoPrimaries, ) { + let transfer_code = match transfer { + VideoTransfer::Gamma => 0, + VideoTransfer::Pq => 1, + VideoTransfer::Hlg => 2, + }; + let primaries_code = match primaries { + VideoPrimaries::Bt709 => 0, + VideoPrimaries::Bt2020 => 1, + }; let params = Nv12Params { transform: *transform, coeffs, - full_range: full_range as u32, - _pad: [0; 3], + flags: [full_range as u32, transfer_code, primaries_code, 0], }; let param_buf = device.create_buffer(&wgpu::BufferDescriptor { label: Some("nv12_blit_params"), diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl index 33dc8d4..cf906ea 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -16,8 +16,8 @@ struct Nv12Params { col2: vec4, // Y'CbCr→R'G'B' matrix from the source colorspace: [Cr→R, Cb→G, Cr→G, Cb→B]. coeffs: vec4, - // .x = full_range flag; .yzw padding. A vec4 keeps each block 16-aligned and the struct size - // matching the Rust `[f32;4] + u32 + [u32;3]` (80 bytes). + // .x = full_range; .y = transfer (0 gamma, 1 PQ, 2 HLG); .z = primaries (0 BT.709, 1 BT.2020). + // A vec4 keeps each block 16-aligned and the struct 80 bytes (Rust `[f32;4] + u32 + [u32;3]`). flags: vec4, } @@ -47,6 +47,41 @@ fn srgb_to_linear(c: vec3) -> vec3 { return select(lo, hi, c > vec3(0.04045)); } +// SMPTE ST 2084 (PQ) EOTF: encoded [0,1] → absolute luminance, then normalize so the 203-nit +// graphics white = 1.0 (HDR highlights exceed 1.0). Per-channel. +fn pq_to_linear(c: vec3) -> vec3 { + let m1 = 0.1593017578125; + let m2 = 78.84375; + let c1 = 0.8359375; + let c2 = 18.8515625; + let c3 = 18.6875; + let e = pow(max(c, vec3(0.0)), vec3(1.0 / m2)); + let num = max(e - vec3(c1), vec3(0.0)); + let den = vec3(c2) - c3 * e; + let nits = pow(num / den, vec3(1.0 / m1)) * 10000.0; // 0..10000 cd/m² + return nits / 203.0; +} + +// ARIB STD-B67 (HLG) inverse-OETF → scene light, normalized so reference white (signal 0.75) = 1.0. +// The display OOTF is omitted (scene-referred compositing); approximate but reasonable for SDR-out. +fn hlg_to_linear(c: vec3) -> vec3 { + let a = 0.17883277; + let b = 0.28466892; + let cc = 0.55991073; + let lo = (c * c) / 3.0; + let hi = (exp((c - vec3(cc)) / a) + vec3(b)) / 12.0; + let scene = select(lo, hi, c > vec3(0.5)); + return scene / 0.26496256; // hlg_inv_oetf(0.75): put reference white at 1.0 +} + +// BT.2020 → BT.709 primaries, linear light (ITU-R BT.2087). Out-of-709 colours go negative → clamp. +fn bt2020_to_bt709(c: vec3) -> vec3 { + let r = 1.660491 * c.r - 0.587641 * c.g - 0.072850 * c.b; + let g = -0.124550 * c.r + 1.132900 * c.g - 0.008349 * c.b; + let b = -0.018151 * c.r - 0.100579 * c.g + 1.118730 * c.b; + return max(vec3(r, g, b), vec3(0.0)); +} + @fragment fn fs_main(in: VertexOutput) -> @location(0) vec4 { let m = mat3x3(params.col0.xyz, params.col1.xyz, params.col2.xyz); @@ -79,9 +114,23 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { let r = Y + params.coeffs.x * Cr; let g = Y + params.coeffs.y * Cb + params.coeffs.z * Cr; let b = Y + params.coeffs.w * Cb; - let rgb_gamma = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); + // Valid encoded signal is [0,1]; clamp before the EOTF (HDR comes from the EOTF, not overshoot). + let rgb_enc = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); + + // Encoded R'G'B' → scene-linear (graphics white = 1.0; HDR may exceed 1.0). + var rgb_lin: vec3; + if params.flags.y == 1u { + rgb_lin = pq_to_linear(rgb_enc); + } else if params.flags.y == 2u { + rgb_lin = hlg_to_linear(rgb_enc); + } else { + rgb_lin = srgb_to_linear(rgb_enc); + } + + // Wide-gamut → BT.709 in linear light to match the compositor's primaries. + if params.flags.z == 1u { + rgb_lin = bt2020_to_bt709(rgb_lin); + } - // R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer. - let rgb_lin = srgb_to_linear(rgb_gamma); return vec4(rgb_lin, 1.0); } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 4a52a03..7582049 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -1703,7 +1703,8 @@ impl egui_wgpu::CallbackTrait for VelloCallback { let y_view = gpu.y.create_view(&Default::default()); let uv_view = gpu.uv.create_view(&Default::default()); shared.nv12_blit.blit( - device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, gpu.coeffs, + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, + gpu.full_range, gpu.coeffs, gpu.transfer, gpu.primaries, ); } else { // Reuse the GPU texture for this frame if it's unchanged (a From b76eefb40437505c75337311b40d21969dfe88f0 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 03:01:11 -0400 Subject: [PATCH 13/31] hdr: per-document Clip vs Highlight-rolloff output mode (Stage A pt 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a per-document HDR→SDR mapping applied at the final linear→sRGB encode, so super-white (HDR) video highlights can be recovered instead of hard-clipped: - core: HdrOutputMode {Clip (default), HighlightRolloff} on Document (serde default), with a SetDocumentPropertiesAction variant for undoable edits. - shaders: a fs_main_rolloff entry point (preview linear_to_srgb.wgsl + the export inline shader) applying a C1 highlight knee — identity below 0.8, smooth rolloff [0.8,∞)→[0.8,1). SDR below the knee is untouched; Clip stays the historical path. - preview (stage.rs) and both export encodes (video_exporter.rs) pick the pipeline variant from document.hdr_output_mode — one value per frame, so no per-pixel uniform; mirrors the existing fs_main_straight pattern. - UI: an "HDR output" dropdown in the Document section of the info panel. Default (Clip) is bit-identical to previous behaviour. Completes Stage A: HDR-correct input (pt 1) + SDR-safe output mapping. HDR export (10-bit P010/PQ) and HDR display remain Stages B/C. --- .../src/actions/set_document_properties.rs | 19 ++++- .../lightningbeam-core/src/document.rs | 25 ++++++ .../src/export/video_exporter.rs | 76 ++++++++++++++++++- .../src/panes/infopanel.rs | 21 ++++- .../src/panes/shaders/linear_to_srgb.wgsl | 31 ++++++++ .../lightningbeam-editor/src/panes/stage.rs | 44 ++++++++++- 6 files changed, 210 insertions(+), 6 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs b/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs index 48c5e38..fc038b3 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs @@ -4,7 +4,7 @@ //! with undo/redo support. use crate::action::Action; -use crate::document::Document; +use crate::document::{Document, HdrOutputMode}; use crate::shape::ShapeColor; /// Individual property change for a document @@ -15,6 +15,7 @@ pub enum DocumentPropertyChange { Duration(f64), Framerate(f64), BackgroundColor(ShapeColor), + HdrOutputMode(HdrOutputMode), } /// Stored old value for undo (either f64 or color) @@ -22,6 +23,7 @@ pub enum DocumentPropertyChange { enum OldValue { F64(f64), Color(ShapeColor), + Hdr(HdrOutputMode), } /// Action that sets a property on the document @@ -72,6 +74,14 @@ impl SetDocumentPropertiesAction { old_value: None, } } + + /// Create a new action to set the HDR→SDR output mode + pub fn set_hdr_output_mode(mode: HdrOutputMode) -> Self { + Self { + property: DocumentPropertyChange::HdrOutputMode(mode), + old_value: None, + } + } } impl Action for SetDocumentPropertiesAction { @@ -83,6 +93,7 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(_) => OldValue::F64(document.duration), DocumentPropertyChange::Framerate(_) => OldValue::F64(document.framerate), DocumentPropertyChange::BackgroundColor(_) => OldValue::Color(document.background_color), + DocumentPropertyChange::HdrOutputMode(_) => OldValue::Hdr(document.hdr_output_mode), }); } @@ -92,6 +103,7 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(v) => document.duration = *v, DocumentPropertyChange::Framerate(v) => document.framerate = *v, DocumentPropertyChange::BackgroundColor(c) => document.background_color = *c, + DocumentPropertyChange::HdrOutputMode(m) => document.hdr_output_mode = *m, } Ok(()) } @@ -106,11 +118,15 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(_) => document.duration = v, DocumentPropertyChange::Framerate(_) => document.framerate = v, DocumentPropertyChange::BackgroundColor(_) => {} + DocumentPropertyChange::HdrOutputMode(_) => {} } } Some(OldValue::Color(c)) => { document.background_color = *c; } + Some(OldValue::Hdr(m)) => { + document.hdr_output_mode = *m; + } None => {} } Ok(()) @@ -123,6 +139,7 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(_) => "duration", DocumentPropertyChange::Framerate(_) => "framerate", DocumentPropertyChange::BackgroundColor(_) => "background color", + DocumentPropertyChange::HdrOutputMode(_) => "HDR output mode", }; format!("Set {}", property_name) } diff --git a/lightningbeam-ui/lightningbeam-core/src/document.rs b/lightningbeam-ui/lightningbeam-core/src/document.rs index 9fd4168..62eba1c 100644 --- a/lightningbeam-ui/lightningbeam-core/src/document.rs +++ b/lightningbeam-ui/lightningbeam-core/src/document.rs @@ -145,6 +145,26 @@ pub enum TimelineMode { Frames, } +/// How super-white (HDR) values are mapped to the SDR display/export output at the final +/// linear→sRGB encode. SDR content sits in [0,1] and is unaffected by either mode. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum HdrOutputMode { + /// Hard-clip values above 1.0 (the historical behaviour). SDR-exact; HDR highlights blow out. + #[default] + Clip, + /// Roll highlights above a knee smoothly toward 1.0 to recover detail. Slightly dims near-white. + HighlightRolloff, +} + +impl HdrOutputMode { + pub fn name(&self) -> &'static str { + match self { + HdrOutputMode::Clip => "Clip", + HdrOutputMode::HighlightRolloff => "Highlight rolloff", + } + } +} + /// Asset category for folder tree access #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AssetCategory { @@ -244,6 +264,10 @@ pub struct Document { #[serde(default)] pub timeline_mode: TimelineMode, + /// How HDR (super-white) values are mapped to the SDR output at the final encode. + #[serde(default)] + pub hdr_output_mode: HdrOutputMode, + /// Current UI layout state (serialized for save/load) #[serde(default, skip_serializing_if = "Option::is_none")] pub ui_layout: Option, @@ -278,6 +302,7 @@ impl Default for Document { ml }, duration: 10.0, + hdr_output_mode: HdrOutputMode::default(), root: GraphicsObject::default(), vector_clips: HashMap::new(), video_clips: HashMap::new(), diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index a19a227..bb10917 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -75,6 +75,8 @@ pub struct ExportGpuResources { pub staging_buffer: wgpu::Buffer, /// Linear to sRGB blit pipeline for final output pub linear_to_srgb_pipeline: wgpu::RenderPipeline, + /// Variant with highlight rolloff (document HDR output mode = Highlight rolloff). + pub linear_to_srgb_pipeline_rolloff: wgpu::RenderPipeline, /// Bind group layout for linear to sRGB blit pub linear_to_srgb_bind_group_layout: wgpu::BindGroupLayout, /// Sampler for linear to sRGB conversion @@ -236,6 +238,41 @@ impl ExportGpuResources { cache: None, }); + // Highlight-rolloff variant: identical but the `fs_main_rolloff` entry point. + let linear_to_srgb_pipeline_rolloff = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("linear_to_srgb_pipeline_rolloff"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main_rolloff"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + polygon_mode: wgpu::PolygonMode::Fill, + unclipped_depth: false, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + let linear_to_srgb_sampler = device.create_sampler(&wgpu::SamplerDescriptor { label: Some("linear_to_srgb_sampler"), address_mode_u: wgpu::AddressMode::ClampToEdge, @@ -263,6 +300,7 @@ impl ExportGpuResources { yuv_texture_view, staging_buffer, linear_to_srgb_pipeline, + linear_to_srgb_pipeline_rolloff, linear_to_srgb_bind_group_layout, linear_to_srgb_sampler, canvas_blit, @@ -340,6 +378,32 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { return vec4(srgb, a); } + +// Highlight rolloff: identity below the knee, smooth C1 rolloff [knee,∞)→[knee,1) above (recovers +// super-white HDR detail). SDR below the knee is untouched. Mirrors panes/shaders/linear_to_srgb.wgsl. +fn highlight_rolloff_ch(x: f32) -> f32 { + let knee = 0.8; + if x <= knee { + return x; + } + let headroom = 1.0 - knee; + return knee + headroom * (1.0 - exp(-(x - knee) / headroom)); +} + +// Variant of fs_main with highlight rolloff (document HDR output mode = Highlight rolloff). +@fragment +fn fs_main_rolloff(in: VertexOutput) -> @location(0) vec4 { + let src = textureSample(source_tex, source_sampler, in.uv); + let a = src.a; + let straight = select(src.rgb / a, vec3(0.0), a <= 0.0); + let rolled = vec3( + highlight_rolloff_ch(straight.r), + highlight_rolloff_ch(straight.g), + highlight_rolloff_ch(straight.b), + ); + let srgb = linear_to_srgb(rolled); + return vec4(srgb, a); +} "#; /// Convert RGBA8 pixels to YUV420p format using BT.709 color space @@ -1127,7 +1191,11 @@ pub fn render_frame_to_rgba_hdr( timestamp_writes: None, }); - render_pass.set_pipeline(&gpu_resources.linear_to_srgb_pipeline); + let final_pipeline = match document.hdr_output_mode { + lightningbeam_core::document::HdrOutputMode::HighlightRolloff => &gpu_resources.linear_to_srgb_pipeline_rolloff, + lightningbeam_core::document::HdrOutputMode::Clip => &gpu_resources.linear_to_srgb_pipeline, + }; + render_pass.set_pipeline(final_pipeline); render_pass.set_bind_group(0, &bind_group, &[]); render_pass.draw(0..4, 0..1); } @@ -1394,7 +1462,11 @@ pub fn render_frame_to_gpu_rgba( timestamp_writes: None, }); - render_pass.set_pipeline(&gpu_resources.linear_to_srgb_pipeline); + let final_pipeline = match document.hdr_output_mode { + lightningbeam_core::document::HdrOutputMode::HighlightRolloff => &gpu_resources.linear_to_srgb_pipeline_rolloff, + lightningbeam_core::document::HdrOutputMode::Clip => &gpu_resources.linear_to_srgb_pipeline, + }; + render_pass.set_pipeline(final_pipeline); render_pass.set_bind_group(0, &bind_group, &[]); render_pass.draw(0..4, 0..1); } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs index 5813e8e..f730d72 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs @@ -980,10 +980,10 @@ impl InfopanelPane { // Extract all needed values up front, then drop the borrow before closures // that need mutable access to shared or self. - let (mut width, mut height, mut duration, mut framerate, layer_count, background_color) = { + let (mut width, mut height, mut duration, mut framerate, layer_count, background_color, mut hdr_mode) = { let document = shared.action_executor.document(); (document.width, document.height, document.duration, document.framerate, - document.root.children.len(), document.background_color) + document.root.children.len(), document.background_color, document.hdr_output_mode) }; // Canvas width @@ -1087,6 +1087,23 @@ impl InfopanelPane { } }); + // HDR output mode (how super-white video highlights map to SDR output) + ui.horizontal(|ui| { + use lightningbeam_core::document::HdrOutputMode; + ui.label("HDR output:"); + egui::ComboBox::from_id_salt(("hdr_output_mode", path)) + .selected_text(hdr_mode.name()) + .show_ui(ui, |ui| { + let mut changed = false; + changed |= ui.selectable_value(&mut hdr_mode, HdrOutputMode::Clip, HdrOutputMode::Clip.name()).changed(); + changed |= ui.selectable_value(&mut hdr_mode, HdrOutputMode::HighlightRolloff, HdrOutputMode::HighlightRolloff.name()).changed(); + if changed { + let action = SetDocumentPropertiesAction::set_hdr_output_mode(hdr_mode); + shared.pending_actions.push(Box::new(action)); + } + }); + }); + // Layer count (read-only) ui.horizontal(|ui| { ui.label("Layers:"); diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl index 8537377..bc52aec 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl @@ -50,3 +50,34 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { a ); } + +// Highlight rolloff: identity below the knee, then a smooth C1 rolloff that maps [knee, ∞) → [knee, 1) +// so super-white (HDR) detail is recovered instead of hard-clipped. SDR below the knee is untouched. +fn highlight_rolloff(x: f32) -> f32 { + let knee = 0.8; + if x <= knee { + return x; + } + let headroom = 1.0 - knee; + return knee + headroom * (1.0 - exp(-(x - knee) / headroom)); +} + +// Variant of fs_main with highlight rolloff (document HDR output mode = Highlight rolloff). +@fragment +fn fs_main_rolloff(in: VertexOutput) -> @location(0) vec4 { + let linear = textureSample(input_tex, input_sampler, in.uv); + let a = linear.a; + let straight = select(linear.rgb / a, vec3(0.0), a <= 0.0); + let rolled = vec3( + highlight_rolloff(straight.r), + highlight_rolloff(straight.g), + highlight_rolloff(straight.b), + ); + + return vec4( + linear_to_srgb_channel(rolled.r), + linear_to_srgb_channel(rolled.g), + linear_to_srgb_channel(rolled.b), + a + ); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 7582049..fc8daa0 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -135,6 +135,8 @@ struct SharedVelloResources { blit_bind_group_layout: wgpu::BindGroupLayout, /// HDR to sRGB blit pipeline (linear→sRGB conversion for display) hdr_blit_pipeline: wgpu::RenderPipeline, + /// Variant of `hdr_blit_pipeline` with highlight rolloff (document HDR output mode). + hdr_blit_pipeline_rolloff: wgpu::RenderPipeline, sampler: wgpu::Sampler, /// Shared image cache for avoiding re-decoding images every frame image_cache: Mutex, @@ -346,6 +348,41 @@ impl SharedVelloResources { cache: None, }); + // Variant with highlight rolloff (document HDR output mode); identical but `fs_main_rolloff`. + let hdr_blit_pipeline_rolloff = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("hdr_blit_pipeline_rolloff"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &hdr_shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &hdr_shader, + entry_point: Some("fs_main_rolloff"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + unclipped_depth: false, + polygon_mode: wgpu::PolygonMode::Fill, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + // Create sampler let sampler = device.create_sampler(&wgpu::SamplerDescriptor { label: Some("vello_blit_sampler"), @@ -383,6 +420,7 @@ impl SharedVelloResources { blit_pipeline, blit_bind_group_layout, hdr_blit_pipeline, + hdr_blit_pipeline_rolloff, sampler, image_cache: Mutex::new(lightningbeam_core::renderer::ImageCache::new()), video_manager, @@ -2947,7 +2985,11 @@ impl egui_wgpu::CallbackTrait for VelloCallback { occlusion_query_set: None, }); - render_pass.set_pipeline(&shared.hdr_blit_pipeline); + let hdr_pipeline = match self.ctx.document.hdr_output_mode { + lightningbeam_core::document::HdrOutputMode::HighlightRolloff => &shared.hdr_blit_pipeline_rolloff, + lightningbeam_core::document::HdrOutputMode::Clip => &shared.hdr_blit_pipeline, + }; + render_pass.set_pipeline(hdr_pipeline); render_pass.set_bind_group(0, hdr_bind_group, &[]); render_pass.draw(0..3, 0..1); // Full-screen triangle (3 vertices) } From 6c8cb5b9e08ee29935cb8eede2c5080e1e81bb2a Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 03:16:28 -0400 Subject: [PATCH 14/31] hw: P010 (10-bit) DMA-BUF import for real HDR video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 10/12-bit HDR decodes to P010-style VAAPI surfaces (16-bit planes), which the 8-bit NV12 import couldn't handle — real HDR clips fell back to software (losing the HDR). Import them as R16/Rg16 plane textures: - dmabuf::Nv12DmaBuf gains `ten_bit`; import_raw picks R16_UNORM/R16G16_UNORM (vk) + R16Unorm/Rg16Unorm (wgpu) when set, else the existing R8/Rg8. - hw_video.rs detects 10-bit from the hw frames context sw_format (P010/P012/P016) and sets it. The NV12→RGB shader is unchanged: it samples normalized floats, and the limited/full de-quant lands at the same values regardless of bit depth. - vk_device requests TEXTURE_FORMAT_16BIT_NORM when the adapter supports it (R16/ Rg16 need it); absent → 10-bit falls back to software, 8-bit unaffected. Pairs with the PQ/HLG/BT.2020 colour math so HDR10/HLG clips now reach the GPU path end-to-end. NV12 callers (encoder, decode primitive) pass ten_bit=false. Co-Authored-By: Claude Opus 4.8 --- .../gpu-video-encoder/src/decoder.rs | 1 + .../gpu-video-encoder/src/dmabuf.rs | 27 +++++++++++++++---- .../gpu-video-encoder/src/encoder.rs | 1 + .../gpu-video-encoder/src/vk_device.rs | 5 +++- .../lightningbeam-editor/src/hw_video.rs | 21 +++++++++++++++ 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/lightningbeam-ui/gpu-video-encoder/src/decoder.rs b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs index 9f64856..884c66f 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/decoder.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs @@ -194,6 +194,7 @@ impl VaapiDecoder { y_pitch: y_pl.pitch as u64, uv_offset: uv_pl.offset as u64, uv_pitch: uv_pl.pitch as u64, + ten_bit: false, }; let imported = dmabuf::import_raw(&self.drm.device, &self.drm.adapter, &buf); ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan diff --git a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs index 492e0b4..7cc72b4 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs @@ -6,7 +6,7 @@ use crate::vaapi::MappedSurface; use crate::vk_device::DrmDevice; use ash::vk; -/// Plane layout for a single-object NV12 DMA-BUF (the common VAAPI case). +/// Plane layout for a single-object NV12/P010 DMA-BUF (the common VAAPI case). #[derive(Clone, Copy)] pub struct Nv12DmaBuf { pub fd: i32, @@ -18,6 +18,9 @@ pub struct Nv12DmaBuf { pub y_pitch: u64, pub uv_offset: u64, pub uv_pitch: u64, + /// True for 10/12/16-bit content (P010 etc.): planes are 16-bit (R16/Rg16) rather than 8-bit + /// (R8/Rg8). The sampled float is normalized either way, so the consumer needs no change. + pub ten_bit: bool, } /// Frees the shared imported `VkDeviceMemory` once both plane images are gone. Held by @@ -68,6 +71,7 @@ pub fn import(drm: &DrmDevice, surf: &MappedSurface) -> Result i, diff --git a/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs b/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs index 93a67aa..fcff428 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs @@ -143,7 +143,10 @@ unsafe fn create_inner(windowed: bool) -> Result { open_device, &wgpu::DeviceDescriptor { label: Some("drm-import-device"), - required_features: wgpu::Features::empty(), + // R16/Rg16 plane textures for P010 (10-bit HDR) import need this; request it only + // when the adapter supports it (else 10-bit falls back to software decode). + required_features: wgpu_adapter.features() + & wgpu::Features::TEXTURE_FORMAT_16BIT_NORM, // Vello's compute pipelines need more than downlevel limits (e.g. // max_storage_buffers_per_shader_stage >= 5). This device only ever runs on a // real VAAPI-capable GPU, so request the adapter's full limits. diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs index d3f915d..f3226d6 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -36,6 +36,26 @@ impl HwVideoImporter for SharedHwImporter { let obj = &(*desc).objects[0]; let width = (*frame).width as u32; let height = (*frame).height as u32; + + // 10/12/16-bit content decodes to P010-style surfaces (16-bit planes). Detect via the hw + // frames context's software format so the import builds R16/Rg16 textures. + let ten_bit = { + let hwfc = (*frame).hw_frames_ctx; + if hwfc.is_null() { + false + } else { + let ctx = (*hwfc).data as *const ff::AVHWFramesContext; + matches!( + (*ctx).sw_format, + ff::AVPixelFormat::AV_PIX_FMT_P010LE + | ff::AVPixelFormat::AV_PIX_FMT_P010BE + | ff::AVPixelFormat::AV_PIX_FMT_P012LE + | ff::AVPixelFormat::AV_PIX_FMT_P012BE + | ff::AVPixelFormat::AV_PIX_FMT_P016LE + | ff::AVPixelFormat::AV_PIX_FMT_P016BE + ) + } + }; // NV12: Y then UV — two layers (one plane each) or one layer with two planes. let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 { (&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0]) @@ -52,6 +72,7 @@ impl HwVideoImporter for SharedHwImporter { y_pitch: y_pl.pitch as u64, uv_offset: uv_pl.offset as u64, uv_pitch: uv_pl.pitch as u64, + ten_bit, }; let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG; From 83410bd4b4ed56e1385f2fe9e811ee6c122b18af Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 03:20:27 -0400 Subject: [PATCH 15/31] fix: P010 plane textures are sample-only (R16 isn't renderable) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R16Unorm/Rg16Unorm aren't renderable, but the import gave every plane texture RENDER_ATTACHMENT/COLOR_ATTACHMENT usage (needed only for the encoder's RGBA→NV12 write path) — so create_view panicked on 10-bit decode ("R16Unorm is not renderable"). Gate the render-target usage (vk COLOR_ATTACHMENT, hal COLOR_TARGET, wgpu RENDER_ATTACHMENT) on 8-bit; 16-bit P010 planes are sample-only (TEXTURE_BINDING/RESOURCE + COPY_SRC), which is all the decode path needs. Co-Authored-By: Claude Opus 4.8 --- .../gpu-video-encoder/src/dmabuf.rs | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs index 7cc72b4..4922f16 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs @@ -104,6 +104,18 @@ pub fn import_raw( return Err("dup(dma-buf fd) failed".into()); } + // 16-bit-norm plane formats (P010) are NOT renderable, so the import is sample-only for + // those (decode path). 8-bit planes keep COLOR_ATTACHMENT for the encoder's RGBA→NV12 write. + let vk_usage = if buf.ten_bit { + vk::ImageUsageFlags::SAMPLED + | vk::ImageUsageFlags::TRANSFER_SRC + | vk::ImageUsageFlags::TRANSFER_DST + } else { + vk::ImageUsageFlags::COLOR_ATTACHMENT + | vk::ImageUsageFlags::SAMPLED + | vk::ImageUsageFlags::TRANSFER_SRC + | vk::ImageUsageFlags::TRANSFER_DST + }; let make_image = |format: vk::Format, w: u32, h: u32, pitch: u64| -> Result { let mut ext = vk::ExternalMemoryImageCreateInfo::default() .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); @@ -119,12 +131,7 @@ pub fn import_raw( .array_layers(1) .samples(vk::SampleCountFlags::TYPE_1) .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) - .usage( - vk::ImageUsageFlags::COLOR_ATTACHMENT - | vk::ImageUsageFlags::SAMPLED - | vk::ImageUsageFlags::TRANSFER_SRC - | vk::ImageUsageFlags::TRANSFER_DST, - ) + .usage(vk_usage) .sharing_mode(vk::SharingMode::EXCLUSIVE) .initial_layout(vk::ImageLayout::UNDEFINED) .push_next(&mut ext) @@ -184,6 +191,22 @@ pub fn import_raw( // Shared guard: frees `memory` once both images' drop callbacks have run. let mem_guard = std::sync::Arc::new(MemoryGuard { device: raw_device.clone(), memory }); + // Match the Vulkan usage: 16-bit-norm planes (P010) are sample-only (not renderable). + let (hal_usage, wgpu_usage) = if buf.ten_bit { + ( + wgpu_types::TextureUses::RESOURCE | wgpu_types::TextureUses::COPY_SRC, + wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_SRC, + ) + } else { + ( + wgpu_types::TextureUses::COLOR_TARGET + | wgpu_types::TextureUses::RESOURCE + | wgpu_types::TextureUses::COPY_SRC, + wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + ) + }; let wrap = |img: vk::Image, format: wgpu::TextureFormat, w: u32, h: u32| -> wgpu::Texture { // wgpu destroys the image (after wait-idle) when the texture drops; the // captured Arc frees the shared memory once both have run. @@ -200,9 +223,7 @@ pub fn import_raw( sample_count: 1, dimension: wgpu::TextureDimension::D2, format, - usage: wgpu_types::TextureUses::COLOR_TARGET - | wgpu_types::TextureUses::RESOURCE - | wgpu_types::TextureUses::COPY_SRC, + usage: hal_usage, memory_flags: wgpu_hal::MemoryFlags::empty(), view_formats: vec![], }; @@ -216,9 +237,7 @@ pub fn import_raw( sample_count: 1, dimension: wgpu::TextureDimension::D2, format, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT - | wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_SRC, + usage: wgpu_usage, view_formats: &[], }, ) From 7c2ac0e4d8ae5b66e6d588b53ba90cc7359d9125 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 03:35:16 -0400 Subject: [PATCH 16/31] hw: propagate stream colour tags to VAAPI frames (fixes washed-out HDR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symptom: properly-converted BT.2020 PQ/HLG clips rendered progressively desaturated vs the SDR reference — the signature of the transfer/primaries tags not reaching the NV12→RGB shader, so HDR code values were decoded as plain sRGB/BT.709 (no gamut expansion, wrong EOTF). VAAPI hardware-decoded frames frequently leave color_trc/color_primaries/ colorspace/color_range unspecified — the authoritative values live on the codec context (parsed from the bitstream), which the importer never sees. Before importing, copy any unspecified colour field from the codec context onto the frame, so the importer detects PQ/HLG + BT.2020 correctly. Also add a one-time importer log (LB_VIDEO_DEBUG) of the detected ten_bit/full_range/colorspace/primaries/trc to confirm what's picked up. Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-core/src/video.rs | 21 +++++++++++++++++++ .../lightningbeam-editor/src/hw_video.rs | 13 ++++++++++++ 2 files changed, 34 insertions(+) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 31c31ce..8fb8552 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -460,6 +460,27 @@ impl VideoDecoder { if gpu_out { // Hardware + GPU consumer: import the VAAPI surface as wgpu NV12 textures // (no CPU copy). + // VAAPI hw frames often don't carry the stream's colour tags, so the + // importer (which only sees the frame) would mis-detect transfer/gamut. + // Copy the authoritative values from the codec context (parsed from the + // bitstream) onto the frame when it left them unspecified. + unsafe { + use ffmpeg::ffi::*; + let fp = frame.as_mut_ptr(); + let cp = decoder.as_ptr(); + if (*fp).color_trc == AVColorTransferCharacteristic::AVCOL_TRC_UNSPECIFIED { + (*fp).color_trc = (*cp).color_trc; + } + if (*fp).color_primaries == AVColorPrimaries::AVCOL_PRI_UNSPECIFIED { + (*fp).color_primaries = (*cp).color_primaries; + } + if (*fp).colorspace == AVColorSpace::AVCOL_SPC_UNSPECIFIED { + (*fp).colorspace = (*cp).colorspace; + } + if (*fp).color_range == AVColorRange::AVCOL_RANGE_UNSPECIFIED { + (*fp).color_range = (*cp).color_range; + } + } let importer = self.importer.as_ref().unwrap(); match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } { Some(gpu) => { diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs index f3226d6..2f349f9 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -17,6 +17,8 @@ use std::sync::{Arc, Mutex}; struct SharedHwImporter { device: wgpu::Device, adapter: wgpu::Adapter, + /// Log the detected colour info once (under LB_VIDEO_DEBUG) rather than per frame. + logged: std::sync::atomic::AtomicBool, } impl HwVideoImporter for SharedHwImporter { @@ -99,6 +101,16 @@ impl HwVideoImporter for SharedHwImporter { }; let coeffs = ycbcr_coeffs(kr, kb); + if std::env::var("LB_VIDEO_DEBUG").is_ok() + && !self.logged.swap(true, std::sync::atomic::Ordering::Relaxed) + { + eprintln!( + "[hw_video] {}x{} ten_bit={} full_range={} colorspace={:?} primaries={:?} trc={:?}", + width, height, ten_bit, full_range, + (*frame).colorspace, (*frame).color_primaries, (*frame).color_trc, + ); + } + // Transfer characteristic → which EOTF the compositor applies to reach scene-linear. let transfer = match (*frame).color_trc { ff::AVColorTransferCharacteristic::AVCOL_TRC_SMPTE2084 => VideoTransfer::Pq, @@ -143,6 +155,7 @@ pub fn install(vm: &Arc>, device: &wgpu::Device, adapter: &w let importer = Arc::new(SharedHwImporter { device: device.clone(), adapter: adapter.clone(), + logged: std::sync::atomic::AtomicBool::new(false), }); if let Ok(mut vm) = vm.lock() { vm.set_hardware_decode( From 28b14b2ad02e012bd68b3f0b0b4c7f8091fb2da1 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 03:59:39 -0400 Subject: [PATCH 17/31] hw: surface import_raw failures (diagnose washed-out 10-bit HDR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed DMA-BUF import returned None silently → core falls back to software, whose RGBA path does no BT.2020→709 gamut conversion, so HDR clips render washed out. The detection log fires before import_raw, so it didn't reveal this. Log the actual import error (with ten_bit) instead of swallowing it, to confirm whether 10-bit P010 import is failing. Co-Authored-By: Claude Opus 4.8 --- lightningbeam-ui/lightningbeam-editor/src/hw_video.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs index 2f349f9..906599e 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -133,7 +133,15 @@ impl HwVideoImporter for SharedHwImporter { let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan - let (y, uv) = imported.ok()?.into_planes(); + let (y, uv) = match imported { + Ok(t) => t.into_planes(), + Err(e) => { + // Surface the failure: a silent None here makes core fall back to software (no gamut + // conversion → BT.2020 looks washed out). 10-bit P010 import is the likely culprit. + eprintln!("[hw_video] import_raw failed (ten_bit={ten_bit}): {e}"); + return None; + } + }; Some(GpuVideoFrame { y: Arc::new(y), uv: Arc::new(uv), From 41e4f3b12b425357307380e9ce637b6c26836bf8 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 04:32:32 -0400 Subject: [PATCH 18/31] =?UTF-8?q?export:=20HDR=20encoder=20scaffolding=20?= =?UTF-8?q?=E2=80=94=2010-bit=20HEVC=20+=20BT.2020/PQ/HLG=20tags=20(Stage?= =?UTF-8?q?=20B=20pt=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings + encoder side of HDR video export (frame data still SDR until pt 2): - core: HdrExportMode {Sdr (default), Pq, Hlg} on VideoExportSettings (serde default), with transfer-name helpers. - setup_video_encoder takes the mode: HDR → YUV420P10LE, BT.2020 NCL matrix, limited range, color_primaries=bt2020, color_trc=smpte2084/arib-std-b67, profile=main10. SDR path unchanged (8-bit full-range BT.709). - run_video_encoder forces HEVC when HDR is selected (the only 10-bit codec wired up). encode_frame is parameterized by pixel format and now copies planes row-by-row honoring stride (10-bit / non-aligned widths can have row padding). Dormant: no UI exposes the mode yet and the frame data is still 8-bit SDR, so selecting HDR is not yet wired. The render→PQ/HLG 10-bit frame path is pt 2. Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-core/src/export.rs | 37 ++++++++++ .../lightningbeam-editor/src/export/mod.rs | 73 ++++++++++++------- .../src/export/video_exporter.rs | 49 ++++++++----- 3 files changed, 113 insertions(+), 46 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/export.rs b/lightningbeam-ui/lightningbeam-core/src/export.rs index 3d90c88..33bfdc9 100644 --- a/lightningbeam-ui/lightningbeam-core/src/export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/export.rs @@ -288,6 +288,38 @@ impl ColorRange { } } +/// HDR output mode for video export. SDR encodes BT.709 8-bit as before; the HDR modes encode +/// 10-bit BT.2020 with the PQ (HDR10) or HLG transfer, preserving super-white highlights from the +/// linear compositor. HDR requires a 10-bit codec (HEVC Main10) — the exporter forces H.265. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum HdrExportMode { + #[default] + Sdr, + /// PQ (SMPTE ST 2084) — HDR10. Graphics white at 203 nits (matches the compositor convention). + Pq, + /// HLG (ARIB STD-B67) — broadcast HDR, also displayable as SDR. + Hlg, +} + +impl HdrExportMode { + pub fn is_hdr(&self) -> bool { !matches!(self, HdrExportMode::Sdr) } + pub fn name(&self) -> &'static str { + match self { + HdrExportMode::Sdr => "SDR (BT.709, 8-bit)", + HdrExportMode::Pq => "HDR10 / PQ (BT.2020, 10-bit)", + HdrExportMode::Hlg => "HLG (BT.2020, 10-bit)", + } + } + /// FFmpeg transfer-characteristic name for the color tags. + pub fn transfer_name(&self) -> &'static str { + match self { + HdrExportMode::Sdr => "bt709", + HdrExportMode::Pq => "smpte2084", + HdrExportMode::Hlg => "arib-std-b67", + } + } +} + /// Video quality presets #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum VideoQuality { @@ -349,6 +381,10 @@ pub struct VideoExportSettings { #[serde(default)] pub color_range: ColorRange, + /// HDR output mode. HDR forces 10-bit HEVC (BT.2020 + PQ/HLG); SDR is the default. + #[serde(default)] + pub hdr: HdrExportMode, + /// Audio settings (None = no audio) pub audio: Option, @@ -368,6 +404,7 @@ impl Default for VideoExportSettings { framerate: 60.0, quality: VideoQuality::High, color_range: ColorRange::Limited, + hdr: HdrExportMode::Sdr, audio: Some(AudioExportSettings::high_quality_aac()), start_time: 0.0, end_time: 60.0, diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index b89ca54..0777a11 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -1556,13 +1556,18 @@ impl ExportOrchestrator { // Initialize FFmpeg ffmpeg_next::init().map_err(|e| format!("Failed to initialize FFmpeg: {}", e))?; - // Convert codec enum to FFmpeg codec ID - let codec_id = match settings.codec { - VideoCodec::H264 => ffmpeg_next::codec::Id::H264, - VideoCodec::H265 => ffmpeg_next::codec::Id::HEVC, - VideoCodec::VP8 => ffmpeg_next::codec::Id::VP8, - VideoCodec::VP9 => ffmpeg_next::codec::Id::VP9, - VideoCodec::ProRes422 => ffmpeg_next::codec::Id::PRORES, + // Convert codec enum to FFmpeg codec ID. HDR requires 10-bit HEVC (Main10), so force HEVC + // regardless of the chosen codec when an HDR mode is selected. + let codec_id = if settings.hdr.is_hdr() { + ffmpeg_next::codec::Id::HEVC + } else { + match settings.codec { + VideoCodec::H264 => ffmpeg_next::codec::Id::H264, + VideoCodec::H265 => ffmpeg_next::codec::Id::HEVC, + VideoCodec::VP8 => ffmpeg_next::codec::Id::VP8, + VideoCodec::VP9 => ffmpeg_next::codec::Id::VP9, + VideoCodec::ProRes422 => ffmpeg_next::codec::Id::PRORES, + } }; // Get bitrate from quality settings @@ -1605,8 +1610,16 @@ impl ExportOrchestrator { height, framerate, bitrate_kbps, + settings.hdr, )?; + // Pixel format the encoder frames are built in (matches setup_video_encoder). + let pixel_format = if settings.hdr.is_hdr() { + ffmpeg_next::format::Pixel::YUV420P10LE + } else { + ffmpeg_next::format::Pixel::YUV420P + }; + // Create output file let mut output = ffmpeg_next::format::output(&output_path) .map_err(|e| format!("Failed to create output file: {}", e))?; @@ -1635,6 +1648,7 @@ impl ExportOrchestrator { width, height, timestamp, + pixel_format, )?; // Send progress update for first frame @@ -1662,6 +1676,7 @@ impl ExportOrchestrator { width, height, timestamp, + pixel_format, )?; frames_encoded += 1; @@ -1706,29 +1721,33 @@ impl ExportOrchestrator { width: u32, height: u32, timestamp: f64, + pixel_format: ffmpeg_next::format::Pixel, ) -> Result<(), String> { - // YUV planes already converted by GPU (no CPU conversion needed) + // YUV planes already converted (8-bit YUV420P, or 10-bit YUV420P10LE for HDR). - // Create FFmpeg video frame - let mut video_frame = ffmpeg_next::frame::Video::new( - ffmpeg_next::format::Pixel::YUV420P, - width, - height, - ); + // Create FFmpeg video frame in the encoder's pixel format. + let mut video_frame = ffmpeg_next::frame::Video::new(pixel_format, width, height); - // Copy YUV planes to frame - // Use safe slice copy - LLVM optimizes this to memcpy, same performance as copy_nonoverlapping - let y_dest = video_frame.data_mut(0); - let y_len = y_plane.len().min(y_dest.len()); - y_dest[..y_len].copy_from_slice(&y_plane[..y_len]); - - let u_dest = video_frame.data_mut(1); - let u_len = u_plane.len().min(u_dest.len()); - u_dest[..u_len].copy_from_slice(&u_plane[..u_len]); - - let v_dest = video_frame.data_mut(2); - let v_len = v_plane.len().min(v_dest.len()); - v_dest[..v_len].copy_from_slice(&v_plane[..v_len]); + // Copy each plane row-by-row honoring the frame's stride (10-bit / arbitrary widths can have + // row padding that a flat copy would misalign). `bytes_per_row` = samples × sample size. + let ten_bit = matches!(pixel_format, ffmpeg_next::format::Pixel::YUV420P10LE); + let sample_bytes = if ten_bit { 2usize } else { 1usize }; + let copy_plane = |frame: &mut ffmpeg_next::frame::Video, idx: usize, src: &[u8], w: usize, h: usize| { + let bytes_per_row = w * sample_bytes; + let stride = frame.stride(idx); + let dst = frame.data_mut(idx); + for row in 0..h { + let s = row * bytes_per_row; + let d = row * stride; + let n = bytes_per_row.min(src.len().saturating_sub(s)).min(dst.len().saturating_sub(d)); + if n == 0 { break; } + dst[d..d + n].copy_from_slice(&src[s..s + n]); + } + }; + let (w, h) = (width as usize, height as usize); + copy_plane(&mut video_frame, 0, y_plane, w, h); + copy_plane(&mut video_frame, 1, u_plane, w / 2, h / 2); + copy_plane(&mut video_frame, 2, v_plane, w / 2, h / 2); // Set PTS (presentation timestamp) in encoder's time base // Encoder time base is 1/(framerate * 1000), so PTS = timestamp * (framerate * 1000) diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index bb10917..85587b1 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -528,6 +528,7 @@ pub fn setup_video_encoder( height: u32, framerate: f64, bitrate_kbps: u32, + hdr: lightningbeam_core::export::HdrExportMode, ) -> Result<(ffmpeg::encoder::Video, ffmpeg::Codec), String> { // Try to find codec by ID first println!("🔍 Looking for codec: {:?}", codec_id); @@ -583,31 +584,41 @@ pub fn setup_video_encoder( // Configure encoder parameters BEFORE opening (critical!) encoder.set_width(aligned_width); encoder.set_height(aligned_height); - encoder.set_format(ffmpeg::format::Pixel::YUV420P); + // HDR encodes 10-bit BT.2020 (limited range); SDR keeps 8-bit full-range BT.709. + if hdr.is_hdr() { + encoder.set_format(ffmpeg::format::Pixel::YUV420P10LE); + } else { + encoder.set_format(ffmpeg::format::Pixel::YUV420P); + } encoder.set_time_base(ffmpeg::Rational(1, (framerate * 1000.0) as i32)); encoder.set_frame_rate(Some(ffmpeg::Rational(framerate as i32, 1))); encoder.set_bit_rate((bitrate_kbps * 1000) as usize); encoder.set_gop(framerate as u32); // 1 second GOP - // Tag the color metadata so players interpret the YUV correctly. Our - // RGB→YUV conversion uses the BT.709 matrix with FULL-range (0–255) luma - // and no transfer applied to the already-sRGB-encoded RGB. Tagging this - // as full-range BT.709 (matrix/primaries/transfer) prevents the level/ - // hue shift that occurs when a player assumes limited-range or BT.601. - // colorspace (matrix) and range have safe setters; primaries and trc are - // generic AVCodecContext options set via the open dictionary below. - encoder.set_colorspace(ffmpeg::color::Space::BT709); - encoder.set_color_range(ffmpeg::color::Range::JPEG); // full range - - println!("📐 Video dimensions: {}×{} (aligned to {}×{} for H.264)", - width, height, aligned_width, aligned_height); - - // Open encoder with codec (like working MP3 export). color_primaries and - // color_trc have no typed setter on the encoder, so pass them as generic - // AVCodecContext options (BT.709) through the open dictionary. + // Tag the color metadata so players interpret the YUV correctly. + // SDR: our RGB→YUV uses the BT.709 matrix with FULL-range (0–255) luma and no transfer applied + // to the already-sRGB-encoded RGB, so tag full-range BT.709 to avoid level/hue shifts. + // HDR: BT.2020 non-constant-luminance matrix, LIMITED range (standard for HDR10/HLG), with the + // PQ or HLG transfer; the 10-bit YUV is produced from PQ/HLG-encoded BT.2020 RGB. let mut color_opts = ffmpeg::Dictionary::new(); - color_opts.set("color_primaries", "bt709"); - color_opts.set("color_trc", "bt709"); + if hdr.is_hdr() { + encoder.set_colorspace(ffmpeg::color::Space::BT2020NCL); + encoder.set_color_range(ffmpeg::color::Range::MPEG); // limited + color_opts.set("color_primaries", "bt2020"); + color_opts.set("color_trc", hdr.transfer_name()); + // HEVC 10-bit profile (the only HDR-capable codec we wire up). + color_opts.set("profile", "main10"); + } else { + encoder.set_colorspace(ffmpeg::color::Space::BT709); + encoder.set_color_range(ffmpeg::color::Range::JPEG); // full range + color_opts.set("color_primaries", "bt709"); + color_opts.set("color_trc", "bt709"); + } + + println!("📐 Video dimensions: {}×{} (aligned to {}×{}){}", + width, height, aligned_width, aligned_height, + if hdr.is_hdr() { " [HDR 10-bit BT.2020]" } else { "" }); + let encoder = encoder .open_as_with(codec, color_opts) .map_err(|e| format!("Failed to open video encoder: {}", e))?; From 33dec6f327f760d345421f4be52917f317e01c2a Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 04:47:35 -0400 Subject: [PATCH 19/31] export: 10-bit HDR video frame path + UI (Stage B pt 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire HDR frame production end-to-end (encoder side was pt 1): - linear_to_pq.wgsl: linear scene HDR (BT.709, white=1.0) → BT.2020 primaries → PQ (203-nit) or HLG OETF → gamma-encoded R'G'B'. Inverse of the nv12 decode, so a decode→encode round-trip is the identity. - hdr_frame.rs (isolated module): runs that pass into an Rgba16Float target, does a synchronous readback (f16-decoded on CPU), then BT.2020 R'G'B'→Y'CbCr limited 4:2:0 10-bit pack → YUV420P10LE planes. Rgba16Float avoids the 16BIT_NORM device feature dependency. - video_exporter::render_frame_to_yuv10_hdr: composite + HDR encode, returning 10-bit planes; lazily builds the pipeline. - orchestrator: HDR uses a synchronous 1-frame-per-call path (the async RGBA pipeline is 8-bit only); zero-copy is skipped for HDR. - dialog: "Dynamic range" dropdown (SDR / HDR10 PQ / HLG). Software HEVC Main10 path; favors correctness over throughput. Compiles; runtime verification needs a GPU + HDR-capable player. Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-editor/src/export/dialog.rs | 16 + .../src/export/hdr_frame.rs | 308 ++++++++++++++++++ .../lightningbeam-editor/src/export/mod.rs | 47 ++- .../src/export/shaders/linear_to_pq.wgsl | 78 +++++ .../src/export/video_exporter.rs | 53 +++ 5 files changed, 501 insertions(+), 1 deletion(-) create mode 100644 lightningbeam-ui/lightningbeam-editor/src/export/hdr_frame.rs create mode 100644 lightningbeam-ui/lightningbeam-editor/src/export/shaders/linear_to_pq.wgsl diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs index 2016a59..6d06263 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs @@ -541,6 +541,22 @@ impl ExportDialog { }); } + // HDR output: 10-bit BT.2020 PQ/HLG (HEVC). Forces H.265; software path (no zero-copy). + ui.horizontal(|ui| { + use lightningbeam_core::export::HdrExportMode; + ui.label("Dynamic range:"); + egui::ComboBox::from_id_salt("video_hdr_mode") + .selected_text(self.video_settings.hdr.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.video_settings.hdr, HdrExportMode::Sdr, HdrExportMode::Sdr.name()); + ui.selectable_value(&mut self.video_settings.hdr, HdrExportMode::Pq, HdrExportMode::Pq.name()); + ui.selectable_value(&mut self.video_settings.hdr, HdrExportMode::Hlg, HdrExportMode::Hlg.name()); + }); + }); + if self.video_settings.hdr.is_hdr() { + ui.label(egui::RichText::new("HDR exports as 10-bit HEVC (H.265), BT.2020.").weak().small()); + } + ui.checkbox(&mut self.include_audio, "Include Audio"); ui.add_space(8.0); diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/hdr_frame.rs b/lightningbeam-ui/lightningbeam-editor/src/export/hdr_frame.rs new file mode 100644 index 0000000..4d78c50 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/export/hdr_frame.rs @@ -0,0 +1,308 @@ +//! 10-bit HDR frame production for video export (isolated from the SDR readback pipeline). +//! +//! Takes the compositor's Rgba16Float HDR accumulator and produces YUV420P10LE planes: +//! 1. GPU pass `linear_to_pq.wgsl` → PQ/HLG-encoded BT.2020 R'G'B' into an Rgba16Unorm texture +//! (the expensive per-pixel transfer + gamut work). +//! 2. Synchronous GPU→CPU readback of that texture. +//! 3. CPU BT.2020 R'G'B'→Y'CbCr (limited range), 4:2:0 average, 10-bit little-endian pack. +//! +//! Synchronous (no triple-buffering); HDR export favors correctness/simplicity over throughput. + +use lightningbeam_core::export::HdrExportMode; + +/// Round up to the wgpu copy row alignment (256 bytes). +fn align_256(n: u32) -> u32 { + (n + 255) & !255 +} + +pub struct HdrFramePipeline { + width: u32, + height: u32, + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + mode_buf: wgpu::Buffer, + /// PQ/HLG-encoded BT.2020 R'G'B' (Rgba16Unorm) render target. + enc_texture_view: wgpu::TextureView, + enc_texture: wgpu::Texture, + /// Staging buffer for readback; rows padded to 256-byte alignment. + staging: wgpu::Buffer, + padded_bytes_per_row: u32, +} + +impl HdrFramePipeline { + pub fn new(device: &wgpu::Device, width: u32, height: u32) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("linear_to_pq_shader"), + source: wgpu::ShaderSource::Wgsl(include_str!("shaders/linear_to_pq.wgsl").into()), + }); + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("linear_to_pq_bgl"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("linear_to_pq_pl"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("linear_to_pq_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + // Rgba16Float (not Unorm) so no TEXTURE_FORMAT_16BIT_NORM feature is needed; PQ/HLG + // values are in [0,1] where f16 has ~11 effective bits — ample for 10-bit output. + format: wgpu::TextureFormat::Rgba16Float, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("linear_to_pq_sampler"), + ..Default::default() + }); + + let mode_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("linear_to_pq_mode"), + size: 16, // vec4 + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + let enc_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("hdr_enc_texture"), + size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let enc_texture_view = enc_texture.create_view(&Default::default()); + + let padded_bytes_per_row = align_256(width * 8); // Rgba16Unorm = 8 bytes/texel + let staging = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("hdr_enc_staging"), + size: (padded_bytes_per_row * height) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + + Self { + width, + height, + pipeline, + bind_group_layout, + sampler, + mode_buf, + enc_texture_view, + enc_texture, + staging, + padded_bytes_per_row, + } + } + + /// Encode the composited HDR texture (`hdr_view`, Rgba16Float linear) to YUV420P10LE planes. + pub fn render_to_yuv10( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + hdr_view: &wgpu::TextureView, + mode: HdrExportMode, + ) -> (Vec, Vec, Vec) { + let mode_code: u32 = if matches!(mode, HdrExportMode::Hlg) { 1 } else { 0 }; + queue.write_buffer(&self.mode_buf, 0, bytemuck::cast_slice(&[mode_code, 0u32, 0, 0])); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("linear_to_pq_bg"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(hdr_view) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&self.sampler) }, + wgpu::BindGroupEntry { binding: 2, resource: self.mode_buf.as_entire_binding() }, + ], + }); + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("hdr_frame_encoder"), + }); + { + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("linear_to_pq_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.enc_texture_view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + }); + rp.set_pipeline(&self.pipeline); + rp.set_bind_group(0, &bind_group, &[]); + rp.draw(0..3, 0..1); + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.enc_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &self.staging, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.padded_bytes_per_row), + rows_per_image: Some(self.height), + }, + }, + wgpu::Extent3d { width: self.width, height: self.height, depth_or_array_layers: 1 }, + ); + queue.submit(Some(encoder.finish())); + + // Synchronous map + wait. + let slice = self.staging.slice(..); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |r| { let _ = tx.send(r); }); + let _ = device.poll(wgpu::PollType::wait_indefinitely()); + let _ = rx.recv(); + + let w = self.width as usize; + let h = self.height as usize; + let mapped = slice.get_mapped_range(); + // Un-pad rows; decode f16 → f32 into a tight RGBA buffer. + let mut rgba = vec![0f32; w * h * 4]; + let row_bytes = w * 8; + for row in 0..h { + let src = row * self.padded_bytes_per_row as usize; + let dst = row * w * 4; + let bytes = &mapped[src..src + row_bytes]; + for px in 0..w * 4 { + let half = u16::from_le_bytes([bytes[px * 2], bytes[px * 2 + 1]]); + rgba[dst + px] = f16_to_f32(half); + } + } + drop(mapped); + self.staging.unmap(); + + rgba_to_yuv420p10le(&rgba, w, h) + } +} + +/// Decode an IEEE 754 half-float. Inputs are in [0,1] so the inf/NaN paths don't occur in practice. +fn f16_to_f32(h: u16) -> f32 { + let sign = (h >> 15) & 1; + let exp = (h >> 10) & 0x1f; + let mant = h & 0x3ff; + let v = if exp == 0 { + (mant as f32) * 2f32.powi(-24) // subnormal + } else if exp == 31 { + if mant == 0 { f32::INFINITY } else { f32::NAN } + } else { + (1.0 + mant as f32 / 1024.0) * 2f32.powi(exp as i32 - 15) + }; + if sign == 1 { -v } else { v } +} + +/// BT.2020 non-constant-luminance R'G'B'→Y'CbCr, limited range, 4:2:0, 10-bit little-endian. +/// Input R'G'B' is already gamma-encoded (PQ/HLG) in [0,1]. +fn rgba_to_yuv420p10le(rgba: &[f32], w: usize, h: usize) -> (Vec, Vec, Vec) { + const KR: f32 = 0.2627; + const KB: f32 = 0.0593; + let kg = 1.0 - KR - KB; + + let luma = |r: f32, g: f32, b: f32| KR * r + kg * g + KB * b; + // 10-bit limited: Y' [64,940] (scale 876), Cb/Cr center 512, excursion ±0.5 → scale 896. + let pack_y = |y: f32| ((y * 876.0 + 64.0).round().clamp(0.0, 1023.0)) as u16; + let pack_c = |c: f32| ((c * 896.0 + 512.0).round().clamp(0.0, 1023.0)) as u16; + + let mut y_plane = vec![0u8; w * h * 2]; + for j in 0..h { + for i in 0..w { + let p = (j * w + i) * 4; + let y10 = pack_y(luma(rgba[p], rgba[p + 1], rgba[p + 2])); + let o = (j * w + i) * 2; + y_plane[o] = (y10 & 0xff) as u8; + y_plane[o + 1] = (y10 >> 8) as u8; + } + } + + let (cw, ch) = (w / 2, h / 2); + let mut u_plane = vec![0u8; cw * ch * 2]; + let mut v_plane = vec![0u8; cw * ch * 2]; + for j in 0..ch { + for i in 0..cw { + let (mut cb, mut cr) = (0.0f32, 0.0f32); + for dy in 0..2 { + for dx in 0..2 { + let p = ((j * 2 + dy) * w + (i * 2 + dx)) * 4; + let (r, g, b) = (rgba[p], rgba[p + 1], rgba[p + 2]); + let yy = luma(r, g, b); + cb += (b - yy) / (2.0 * (1.0 - KB)); + cr += (r - yy) / (2.0 * (1.0 - KR)); + } + } + let cb10 = pack_c(cb / 4.0); + let cr10 = pack_c(cr / 4.0); + let o = (j * cw + i) * 2; + u_plane[o] = (cb10 & 0xff) as u8; + u_plane[o + 1] = (cb10 >> 8) as u8; + v_plane[o] = (cr10 & 0xff) as u8; + v_plane[o + 1] = (cr10 >> 8) as u8; + } + } + + (y_plane, u_plane, v_plane) +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 0777a11..337843b 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -11,6 +11,7 @@ pub mod readback_pipeline; pub mod perf_metrics; pub mod cpu_yuv_converter; pub mod gpu_yuv; +pub mod hdr_frame; use lightningbeam_core::export::{AudioExportSettings, ImageExportSettings, VideoExportSettings, ExportProgress}; use lightningbeam_core::document::Document; @@ -52,6 +53,8 @@ pub struct VideoExportState { width: u32, /// Export height in pixels height: u32, + /// HDR output mode — HDR uses a synchronous 10-bit path instead of the async RGBA pipeline. + hdr: lightningbeam_core::export::HdrExportMode, /// Channel to send rendered frames to encoder thread frame_tx: Option>, /// HDR GPU resources for compositing pipeline (effects, color conversion) @@ -854,6 +857,7 @@ impl ExportOrchestrator { width: u32, height: u32, ) -> (std::thread::JoinHandle<()>, VideoExportState) { + let hdr = settings.hdr; let handle = std::thread::spawn(move || { Self::run_video_encoder(settings, output_path, frame_rx, progress_tx, cancel_flag, total_frames); }); @@ -866,6 +870,7 @@ impl ExportOrchestrator { framerate, width, height, + hdr, frame_tx: Some(frame_tx), gpu_resources: None, readback_pipeline: None, @@ -890,7 +895,10 @@ impl ExportOrchestrator { framerate: f64, output_path: &std::path::Path, ) -> Option { - if !matches!(settings.codec, lightningbeam_core::export::VideoCodec::H264) { + // Zero-copy is 8-bit H.264 only; HDR needs the 10-bit HEVC software path. + if settings.hdr.is_hdr() + || !matches!(settings.codec, lightningbeam_core::export::VideoCodec::H264) + { return None; } let encoder = match gpu_video_encoder::encoder::ZeroCopyEncoder::new( @@ -1243,6 +1251,43 @@ impl ExportOrchestrator { let width = state.width; let height = state.height; + // HDR path: synchronous 10-bit render (composite → PQ/HLG → readback → 10-bit YUV), one + // frame per call. Bypasses the SDR async RGBA pipeline (which is 8-bit only). + if state.hdr.is_hdr() { + if state.gpu_resources.is_none() { + println!("🎬 [VIDEO EXPORT] Initializing HDR GPU resources {}x{} ({})", width, height, state.hdr.name()); + state.gpu_resources = Some(video_exporter::ExportGpuResources::new(device, width, height)); + } + if state.current_frame < state.total_frames { + let timestamp = state.start_time + (state.current_frame as f64 / state.framerate); + let gpu_resources = state.gpu_resources.as_mut().unwrap(); + let (y, u, v) = video_exporter::render_frame_to_yuv10_hdr( + document, timestamp, width, height, + device, queue, renderer, image_cache, video_manager, + gpu_resources, state.hdr, raster_store, + )?; + if let Some(tx) = &state.frame_tx { + tx.send(VideoFrameMessage::Frame { + frame_num: state.current_frame, + timestamp, + y_plane: y, + u_plane: u, + v_plane: v, + }).map_err(|_| "Failed to send HDR frame")?; + } + state.current_frame += 1; + } + if state.current_frame >= state.total_frames { + println!("🎬 [VIDEO EXPORT] HDR complete: {} frames", state.total_frames); + if let Some(tx) = state.frame_tx.take() { + tx.send(VideoFrameMessage::Done).ok(); + } + state.gpu_resources = None; + return Ok(false); + } + return Ok(true); + } + // Initialize GPU resources and readback pipeline on first frame if state.gpu_resources.is_none() { println!("🎬 [VIDEO EXPORT] Initializing HDR GPU + async pipeline {}x{}", width, height); diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/shaders/linear_to_pq.wgsl b/lightningbeam-ui/lightningbeam-editor/src/export/shaders/linear_to_pq.wgsl new file mode 100644 index 0000000..6b71562 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/export/shaders/linear_to_pq.wgsl @@ -0,0 +1,78 @@ +// Linear-HDR → PQ/HLG BT.2020 encode (for 10-bit HDR video export). +// +// Input: the compositor's Rgba16Float HDR accumulator — PREMULTIPLIED scene-linear, BT.709 +// primaries, graphics white = 1.0, HDR highlights > 1.0. +// Output: gamma-encoded R'G'B' in BT.2020 primaries, PQ (mode 0) or HLG (mode 1), to an +// Rgba16Unorm target. A later CPU pass does only BT.2020 R'G'B'→Y'CbCr (no transfer) + 4:2:0 + 10-bit. +// +// This is the encode inverse of panes/shaders/nv12_blit.wgsl's decode (203-nit PQ white; HLG +// reference white at signal 0.75), so a decode→encode round-trip is the identity. + +@group(0) @binding(0) var input_tex: texture_2d; +@group(0) @binding(1) var input_sampler: sampler; +@group(0) @binding(2) var params: vec4; // .x = mode (0 = PQ, 1 = HLG) + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var out: VertexOutput; + let x = f32((vertex_index & 1u) << 1u); + let y = f32(vertex_index & 2u); + out.position = vec4(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0); + out.uv = vec2(x, y); + return out; +} + +// BT.709 → BT.2020 primaries, linear light (ITU-R BT.2087). +fn bt709_to_bt2020(c: vec3) -> vec3 { + let r = 0.627404 * c.r + 0.329283 * c.g + 0.043313 * c.b; + let g = 0.069097 * c.r + 0.919540 * c.g + 0.011362 * c.b; + let b = 0.016391 * c.r + 0.088013 * c.g + 0.895595 * c.b; + return vec3(r, g, b); +} + +// SMPTE ST 2084 (PQ) OETF: scene-linear (white = 1.0 = 203 nits) → PQ code [0,1]. +fn pq_oetf(lin: vec3) -> vec3 { + let nits = max(lin, vec3(0.0)) * 203.0; + let ln = min(nits / 10000.0, vec3(1.0)); + let m1 = 0.1593017578125; + let m2 = 78.84375; + let c1 = 0.8359375; + let c2 = 18.8515625; + let c3 = 18.6875; + let lm = pow(ln, vec3(m1)); + return pow((vec3(c1) + c2 * lm) / (vec3(1.0) + c3 * lm), vec3(m2)); +} + +// ARIB STD-B67 (HLG) OETF: scene-linear (white = 1.0) → HLG signal [0,1]. Reference white maps to +// signal 0.75 (matching the decode's /0.26496256 normalization). Display OOTF omitted (scene-referred). +fn hlg_oetf(lin: vec3) -> vec3 { + let a = 0.17883277; + let b = 0.28466892; + let c = 0.55991073; + let e = clamp(lin * 0.26496256, vec3(0.0), vec3(1.0)); + let lo = sqrt(3.0 * e); + let hi = a * log(12.0 * e - vec3(b)) + vec3(c); + return select(lo, hi, e > vec3(1.0 / 12.0)); +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + // Compositor stores PREMULTIPLIED linear; unpremultiply to straight (video is opaque, a≈1). + let texel = textureSample(input_tex, input_sampler, in.uv); + let a = texel.a; + let straight = select(texel.rgb / a, vec3(0.0), a <= 0.0); + + let bt2020 = max(bt709_to_bt2020(straight), vec3(0.0)); + var enc: vec3; + if params.x == 1u { + enc = hlg_oetf(bt2020); + } else { + enc = pq_oetf(bt2020); + } + return vec4(enc, 1.0); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 85587b1..15f432e 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -90,6 +90,8 @@ pub struct ExportGpuResources { /// texture copy each frame instead of a full Vello render + 2 passes/submits. `None` until the /// first frame; invalidated on resize. cached_bg_hdr: Option, + /// HDR encode pipeline (linear→PQ/HLG BT.2020 → 10-bit YUV). Lazily built on the first HDR frame. + hdr_pipeline: Option, } impl ExportGpuResources { @@ -306,6 +308,7 @@ impl ExportGpuResources { canvas_blit, raster_cache: std::collections::HashMap::new(), cached_bg_hdr: None, + hdr_pipeline: None, } } @@ -1369,6 +1372,56 @@ fn fault_in_raster_for_frame( } } +/// Render one frame as 10-bit HDR YUV420P10LE planes (BT.2020 + PQ/HLG). Synchronous: composites, +/// runs the linear→PQ/HLG GPU pass, reads it back, and CPU-converts to 10-bit YUV. Used by the +/// HDR export path instead of the async readback pipeline. +#[allow(clippy::too_many_arguments)] +pub fn render_frame_to_yuv10_hdr( + document: &mut Document, + timestamp: f64, + width: u32, + height: u32, + device: &wgpu::Device, + queue: &wgpu::Queue, + renderer: &mut vello::Renderer, + image_cache: &mut ImageCache, + video_manager: &Arc>, + gpu_resources: &mut ExportGpuResources, + hdr_mode: lightningbeam_core::export::HdrExportMode, + raster_store: Option<&lightningbeam_core::raster_store::RasterStore>, +) -> Result<(Vec, Vec, Vec), String> { + use vello::kurbo::Affine; + + document.current_time = timestamp; + fault_in_raster_for_frame(document, raster_store); + + let base_transform = if document.width > 0.0 && document.height > 0.0 { + Affine::scale_non_uniform(width as f64 / document.width, height as f64 / document.height) + } else { + Affine::IDENTITY + }; + + // Export composites on a separate device → force software video frames. + if let Ok(mut vm) = video_manager.lock() { + vm.set_render_hardware_ok(false); + } + + let composite_result = render_document_for_compositing( + document, base_transform, image_cache, video_manager, None, None, false, + ); + composite_document_to_hdr(&composite_result, document, device, queue, renderer, gpu_resources, width, height, false)?; + + if gpu_resources.hdr_pipeline.is_none() { + gpu_resources.hdr_pipeline = Some(super::hdr_frame::HdrFramePipeline::new(device, width, height)); + } + let planes = gpu_resources + .hdr_pipeline + .as_ref() + .unwrap() + .render_to_yuv10(device, queue, &gpu_resources.hdr_texture_view, hdr_mode); + Ok(planes) +} + pub fn render_frame_to_gpu_rgba( document: &mut Document, timestamp: f64, From bb3369b7098e6349766fd64b7a5739671cba6ca1 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 05:07:02 -0400 Subject: [PATCH 20/31] export: GPU-resident decode for software + HDR export (Stage 3c-export pt 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The software and HDR export paths already composite on the shared device (the eframe device, via render_next_video_frame), so they can consume the same hardware-decoded NV12 GPU frames the preview uses — no CPU download/upload: - ExportGpuResources gains the Nv12BlitPipeline; the export compositor's Video arm branches on inst.gpu (NV12 blit, mirroring stage.rs) vs the RGBA upload path. - render_frame_to_gpu_rgba takes a hardware_ok flag: true for the software video + image export (shared device), false for the zero-copy path (own device, must download). render_frame_to_yuv10_hdr sets it true. Drops the 4K software/HDR-export decode wall (HW decode + GPU composite, no per-frame RGBA upload). The common Linux H.264 zero-copy path is unchanged (separate device) — that's pt 2. Falls back to software when no shared device / importer (flag is harmless then: get_frame returns CPU anyway). Compiles. Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-editor/src/export/mod.rs | 3 ++ .../src/export/video_exporter.rs | 40 ++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 337843b..d435749 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -625,6 +625,7 @@ impl ExportOrchestrator { floating_selection, state.settings.allow_transparency, raster_store, + true, // image export composites on the shared device )?; queue.submit(Some(encoder.finish())); @@ -1393,6 +1394,7 @@ impl ExportOrchestrator { None, // No floating selection during video export false, // Video export is never transparent raster_store, + true, // software export composites on the shared device → may use HW frames )?; let render_end = Instant::now(); @@ -1498,6 +1500,7 @@ impl ExportOrchestrator { None, false, Some(&raster_store), + false, // zero-copy runs on its own device → download HW frames to CPU ) { Ok(cmd) => cmd, Err(e) => { diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 15f432e..696a222 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -83,6 +83,8 @@ pub struct ExportGpuResources { pub linear_to_srgb_sampler: wgpu::Sampler, /// Canvas blit pipeline for raster/video/float layers (bypasses Vello). pub canvas_blit: crate::gpu_brush::CanvasBlitPipeline, + /// NV12→linear blit for hardware-decoded video frames (export on the shared device). + pub nv12_blit: crate::nv12_blit::Nv12BlitPipeline, /// Per-keyframe GPU texture cache for raster layers during export. pub raster_cache: std::collections::HashMap, /// Cached HDR accumulator state after the (static) background is composited in. The document @@ -287,6 +289,7 @@ impl ExportGpuResources { }); let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device); + let nv12_blit = crate::nv12_blit::Nv12BlitPipeline::new(device); Self { buffer_pool, @@ -306,6 +309,7 @@ impl ExportGpuResources { linear_to_srgb_bind_group_layout, linear_to_srgb_sampler, canvas_blit, + nv12_blit, raster_cache: std::collections::HashMap::new(), cached_bg_hdr: None, hdr_pipeline: None, @@ -959,16 +963,26 @@ fn composite_document_to_hdr( } RenderedLayerType::Video { instances } => { for inst in instances { - if inst.rgba_data.is_empty() { continue; } + if inst.gpu.is_none() && inst.rgba_data.is_empty() { continue; } let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { - // Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU - // decodes to linear on sample (no per-pixel CPU conversion). Blit with - // blit_straight so the shader doesn't unpremultiply. - let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex")); - let tex_view = tex.create_view(&Default::default()); let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height); - gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + if let Some(gpu) = &inst.gpu { + // Hardware-decoded NV12 plane textures → linear, no CPU upload. + let y_view = gpu.y.create_view(&Default::default()); + let uv_view = gpu.uv.create_view(&Default::default()); + gpu_resources.nv12_blit.blit( + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, + gpu.full_range, gpu.coeffs, gpu.transfer, gpu.primaries, + ); + } else { + // Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU + // decodes to linear on sample (no per-pixel CPU conversion). Blit with + // blit_straight so the shader doesn't unpremultiply. + let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex")); + let tex_view = tex.create_view(&Default::default()); + gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + } let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal); let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") }); gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None); @@ -1401,9 +1415,9 @@ pub fn render_frame_to_yuv10_hdr( Affine::IDENTITY }; - // Export composites on a separate device → force software video frames. + // HDR export composites on the shared device, so it can consume hardware-decoded GPU frames. if let Ok(mut vm) = video_manager.lock() { - vm.set_render_hardware_ok(false); + vm.set_render_hardware_ok(true); } let composite_result = render_document_for_compositing( @@ -1437,6 +1451,9 @@ pub fn render_frame_to_gpu_rgba( floating_selection: Option<&lightningbeam_core::selection::RasterFloatingSelection>, allow_transparency: bool, raster_store: Option<&lightningbeam_core::raster_store::RasterStore>, + // True when compositing on the shared device (software/image export) → may consume + // hardware-decoded GPU frames; false for the zero-copy path on its own device. + hardware_ok: bool, ) -> Result { use vello::kurbo::Affine; @@ -1468,9 +1485,10 @@ pub fn render_frame_to_gpu_rgba( Affine::IDENTITY }; - // Export composites on a separate device — force software frames (see above). + // GPU frames are usable only on the shared device (software/image export); the zero-copy path + // runs on its own device and must download to CPU. if let Ok(mut vm) = video_manager.lock() { - vm.set_render_hardware_ok(false); + vm.set_render_hardware_ok(hardware_ok); } // Render document for compositing (returns per-layer scenes) From ca9a70e10a21775c889a5772c08721f8ffcfe082 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 05:18:23 -0400 Subject: [PATCH 21/31] export: run the zero-copy H.264 encoder on the shared device (Stage 3c-export pt 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the common Linux H.264 export fully GPU-resident: decode (HW NV12) → composite → VAAPI encode all on one device, no CPU round-trips. - gpu-video-encoder: ZeroCopyEncoder now holds wgpu (device,queue,adapter) handles instead of owning a DrmDevice. New `new_on_device(device,queue,adapter,...)` runs the RGBA→NV12 render + DMA-BUF import on a passed device; `new` keeps building its own. The encoder only *imports* VAAPI surfaces (not export), so the shared import-capable device works directly. - editor: stash the shared device handles on EditorApp (set in the creation closure from the eframe render_state when the shared device is active) and thread them through start_video_export / start_video_with_audio_export → try_build_zero_copy, which uses new_on_device when available. ZeroCopyVideo carries on_shared_device → the export composite uses hardware_ok=true (consuming HW-decoded GPU frames from pt 1) only then. Falls back to the own-device encoder (decode downloads to CPU) when no shared device. Tradeoff: the export now shares the GPU with the UI thread (was a separate device); acceptable for the GPU-resident-decode win. Compiles; crate tests build. Co-Authored-By: Claude Opus 4.8 --- .../gpu-video-encoder/src/encoder.rs | 50 +++++++++++++++---- .../lightningbeam-editor/src/export/mod.rs | 43 +++++++++++----- .../lightningbeam-editor/src/main.rs | 19 +++++-- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs index 4347f0e..7ec9c36 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/encoder.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/encoder.rs @@ -6,7 +6,7 @@ use crate::dmabuf::{self, ImportedNv12, Nv12DmaBuf}; use crate::render_nv12::Rgba2Nv12; -use crate::vk_device::{self, DrmDevice}; +use crate::vk_device; use ffmpeg_sys_next as ff; use std::collections::HashMap; use std::ffi::CString; @@ -19,7 +19,11 @@ fn averror(e: i32) -> i32 { } pub struct ZeroCopyEncoder { - drm: DrmDevice, + /// wgpu handles the NV12 render runs on — either an own `DrmDevice`'s (via `new`) or the + /// editor's shared device (via `new_on_device`). Cloned (Arc-backed) so the source can drop. + device: wgpu::Device, + queue: wgpu::Queue, + adapter: wgpu::Adapter, renderer: Rgba2Nv12, hw_device: *mut ff::AVBufferRef, frames_ref: *mut ff::AVBufferRef, @@ -51,8 +55,30 @@ impl ZeroCopyEncoder { output_path: &Path, full_range: bool, ) -> Result { + // Build a dedicated DMA-BUF-import device and run the encoder on it. let drm = vk_device::create()?; - let renderer = Rgba2Nv12::new(&drm.device, full_range); + Self::new_on_device( + drm.device, drm.queue, drm.adapter, + width, height, framerate, bitrate_kbps, output_path, full_range, + ) + } + + /// Build the encoder running its NV12 render + DMA-BUF import on an existing wgpu device (the + /// editor's shared device), so decode→composite→encode stay GPU-resident on one device. The + /// device must have the DMA-BUF import extensions (a `DrmDevice` / `vk_device::create_windowed`). + #[allow(clippy::too_many_arguments)] + pub fn new_on_device( + device: wgpu::Device, + queue: wgpu::Queue, + adapter: wgpu::Adapter, + width: u32, + height: u32, + framerate: i32, + bitrate_kbps: u32, + output_path: &Path, + full_range: bool, + ) -> Result { + let renderer = Rgba2Nv12::new(&device, full_range); unsafe { let mut hw_device = crate::vaapi::create_device()?; let name = CString::new("h264_vaapi").unwrap(); @@ -153,7 +179,9 @@ impl ZeroCopyEncoder { let stream_tb = (*stream).time_base; Ok(Self { - drm, + device, + queue, + adapter, renderer, hw_device, frames_ref, @@ -172,10 +200,10 @@ impl ZeroCopyEncoder { /// The wgpu device frames must be rendered on (so the RGBA texture is importable). pub fn device(&self) -> &wgpu::Device { - &self.drm.device + &self.device } pub fn queue(&self) -> &wgpu::Queue { - &self.drm.queue + &self.queue } /// Render `rgba` (an `Rgba8Unorm` texture on [`Self::device`], `TEXTURE_BINDING`) @@ -216,7 +244,7 @@ impl ZeroCopyEncoder { uv_pitch: uv.pitch as u64, ten_bit: false, }; - let imported = match dmabuf::import_raw(&self.drm.device, &self.drm.adapter, &buf) { + let imported = match dmabuf::import_raw(&self.device, &self.adapter, &buf) { Ok(i) => i, Err(e) => { ff::av_frame_free(&mut (drm_f as *mut _)); @@ -233,10 +261,10 @@ impl ZeroCopyEncoder { let rgba_view = rgba.create_view(&Default::default()); let y_view = imp.y().create_view(&Default::default()); let uv_view = imp.uv().create_view(&Default::default()); - let mut cmd = self.drm.device.create_command_encoder(&Default::default()); - self.renderer.convert(&self.drm.device, &mut cmd, &rgba_view, &y_view, &uv_view); - self.drm.queue.submit(Some(cmd.finish())); - let _ = self.drm.device.poll(wgpu::PollType::wait_indefinitely()); + let mut cmd = self.device.create_command_encoder(&Default::default()); + self.renderer.convert(&self.device, &mut cmd, &rgba_view, &y_view, &uv_view); + self.queue.submit(Some(cmd.finish())); + let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); // Encode the surface. (*surf).pts = self.pts; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index d435749..5673da1 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -82,6 +82,8 @@ struct ZeroCopyVideo { gpu_resources: video_exporter::ExportGpuResources, /// Reused RGBA target (RENDER_ATTACHMENT | TEXTURE_BINDING) on the encoder's device. rgba: wgpu::Texture, + /// True when running on the shared device → compositing can consume hardware-decoded GPU frames. + on_shared_device: bool, } /// State for a single-frame image export (runs on the GPU render thread, one frame per update). @@ -895,6 +897,7 @@ impl ExportOrchestrator { height: u32, framerate: f64, output_path: &std::path::Path, + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Option { // Zero-copy is 8-bit H.264 only; HDR needs the 10-bit HEVC software path. if settings.hdr.is_hdr() @@ -902,14 +905,25 @@ impl ExportOrchestrator { { return None; } - let encoder = match gpu_video_encoder::encoder::ZeroCopyEncoder::new( - width, - height, - framerate.round() as i32, - settings.quality.bitrate_kbps(), - output_path, - settings.color_range.is_full(), - ) { + let bitrate = settings.quality.bitrate_kbps(); + let fr = framerate.round() as i32; + let full = settings.color_range.is_full(); + let on_shared_device = shared_device.is_some(); + // Prefer the shared device → decode→composite→encode stay GPU-resident on one device. + // Without it, the encoder builds its own device (decode still downloads to CPU per Step 1's + // hardware_ok=false on this path). + let encoder_result = match shared_device { + Some((device, queue, adapter)) => { + println!("🎬 [EXPORT] zero-copy on shared device (GPU-resident decode)"); + gpu_video_encoder::encoder::ZeroCopyEncoder::new_on_device( + device, queue, adapter, width, height, fr, bitrate, output_path, full, + ) + } + None => gpu_video_encoder::encoder::ZeroCopyEncoder::new( + width, height, fr, bitrate, output_path, full, + ), + }; + let encoder = match encoder_result { Ok(e) => e, Err(e) => { println!("🎬 [EXPORT] zero-copy unavailable ({e}); software path"); @@ -945,7 +959,7 @@ impl ExportOrchestrator { view_formats: &[], }); println!("🎬 [EXPORT] zero-copy VAAPI H.264 enabled"); - Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba }) + Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba, on_shared_device }) } /// Start a video export in the background. @@ -963,6 +977,7 @@ impl ExportOrchestrator { /// # Returns /// Ok(()) on success, Err on failure #[allow(clippy::too_many_arguments)] + #[allow(clippy::too_many_arguments)] pub fn start_video_export( &mut self, settings: VideoExportSettings, @@ -971,6 +986,8 @@ impl ExportOrchestrator { video_manager: Arc>, raster_store: lightningbeam_core::raster_store::RasterStore, container_path: Option, + // The shared VAAPI device, `Some` only when active → zero-copy encode runs on it. + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Result<(), String> { println!("🎬 [VIDEO EXPORT] Starting video export"); @@ -998,7 +1015,7 @@ impl ExportOrchestrator { // success it returns here; otherwise we fall through to the software encoder thread. #[cfg(target_os = "linux")] { - if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path) { + if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path, shared_device) { drop(frame_rx); let document_snapshot = document.clone(); let mut image_cache = ImageCache::new(); @@ -1062,6 +1079,8 @@ impl ExportOrchestrator { video_manager: Arc>, raster_store: lightningbeam_core::raster_store::RasterStore, container_path: Option, + // The shared VAAPI device, `Some` only when active → zero-copy encode runs on it. + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Result<(), String> { println!("🎬🎵 [PARALLEL EXPORT] Starting parallel video+audio export"); @@ -1128,7 +1147,7 @@ impl ExportOrchestrator { // by `render_next_video_frame` on the UI thread. #[cfg(target_os = "linux")] let (video_thread, video_state) = match Self::try_build_zero_copy( - &video_settings, video_width, video_height, video_framerate, &temp_video_path, + &video_settings, video_width, video_height, video_framerate, &temp_video_path, shared_device, ) { Some(zc) => { drop(frame_rx); // the zero-copy path renders internally, no frame channel @@ -1500,7 +1519,7 @@ impl ExportOrchestrator { None, false, Some(&raster_store), - false, // zero-copy runs on its own device → download HW frames to CPU + zc.on_shared_device, // GPU-resident decode only when on the shared device ) { Ok(cmd) => cmd, Err(e) => { diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index d9152ad..8c77c48 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -299,14 +299,18 @@ fn main() -> eframe::Result { options, Box::new(move |cc| { #[cfg(debug_assertions)] - let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); + #[allow(unused_mut)] + let mut app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); #[cfg(not(debug_assertions))] - let app = EditorApp::new(cc, layouts, theme); - // Wire hardware video decode into the VideoManager now that the shared device exists. + #[allow(unused_mut)] + let mut app = EditorApp::new(cc, layouts, theme); + // Wire hardware video decode into the VideoManager now that the shared device exists, and + // stash the shared device handles so the zero-copy export encoder can run on it too. #[cfg(target_os = "linux")] if shared_device_active { if let Some(rs) = cc.wgpu_render_state.as_ref() { hw_video::install(&app.video_manager, &rs.device, &rs.adapter); + app.shared_device = Some((rs.device.clone(), rs.queue.clone(), rs.adapter.clone())); } } Ok(Box::new(app)) @@ -990,6 +994,9 @@ struct EditorApp { audio_channels: u32, // Video decoding and management video_manager: std::sync::Arc>, // Shared video manager + /// The shared VAAPI-capable wgpu device (device, queue, adapter), `Some` only when active. Lets + /// the zero-copy export encoder run on it (GPU-resident decode→composite→encode). + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, // Webcam capture state webcam: Option, /// Latest polled webcam frame (updated each frame for preview) @@ -1327,6 +1334,7 @@ impl EditorApp { video_manager: std::sync::Arc::new(std::sync::Mutex::new( lightningbeam_core::video::VideoManager::new() )), + shared_device: None, webcam: None, webcam_frame: None, webcam_record_command: None, @@ -6125,6 +6133,9 @@ impl eframe::App for EditorApp { self.export_orchestrator = Some(export::ExportOrchestrator::new()); } + // Clone before the &mut self.export_orchestrator borrow below. + let shared_device = self.shared_device.clone(); + let export_started = if let Some(orchestrator) = &mut self.export_orchestrator { match export_result { ExportResult::Image(settings, output_path) => { @@ -6163,6 +6174,7 @@ impl eframe::App for EditorApp { Arc::clone(&self.video_manager), self.raster_store.clone(), self.current_file_path.clone(), + shared_device.clone(), ) { Ok(()) => true, Err(err) => { @@ -6184,6 +6196,7 @@ impl eframe::App for EditorApp { Arc::clone(&self.video_manager), self.raster_store.clone(), self.current_file_path.clone(), + shared_device.clone(), ) { Ok(()) => true, Err(err) => { From 701b57bfe8abe5ee2818b46bdff0d6fd1cb355c5 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 05:39:35 -0400 Subject: [PATCH 22/31] video: thumbnails force CPU frames (fix black thumbs + decoder thrash) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The asset-library thumbnail called VideoManager::get_frame, which honors the render-pass hardware flag — left ON by the preview, so the thumbnail got a GPU frame with empty rgba_data → an all-black thumbnail. The thumbnail is also the only consumer that requests a fixed low timestamp (1.0s) on the shared per-clip decoder, so when it re-decodes during playback it yanks the decoder back to ~1s; the next playback frame (6.x) is then ">2s forward" and re-seeks to the keyframe + re-decodes the whole GOP (the jerk: per-frame decode is ~5ms, but these seeks cost 40ms + N-frame catch-up). Add VideoManager::get_frame_cpu (forces want_gpu=false regardless of the render flag); the thumbnail uses it. Now it produces a real RGBA thumbnail that the editor texture-caches once, instead of re-decoding black frames. Co-Authored-By: Claude Opus 4.8 --- lightningbeam-ui/lightningbeam-core/src/video.rs | 11 +++++++++++ .../lightningbeam-editor/src/panes/asset_library.rs | 6 ++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 8fb8552..67770d4 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -940,6 +940,17 @@ impl VideoManager { // Whether this pass wants (and can produce) a GPU frame. Gated on HW being configured at all // so that with software-only decode preview and export share one cache entry (no double-cache). let want_gpu = self.render_hardware_ok && self.hw_device.is_some(); + self.get_frame_inner(clip_id, timestamp, target_w, target_h, want_gpu) + } + + /// Like [`get_frame`](Self::get_frame) but always returns a CPU (RGBA) frame, ignoring the + /// render-pass hardware flag. For consumers that need pixel bytes (thumbnails, image readback) + /// regardless of whether a render pass last enabled GPU frames. + pub fn get_frame_cpu(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { + self.get_frame_inner(clip_id, timestamp, target_w, target_h, false) + } + + fn get_frame_inner(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Option> { // The cache key includes (target size, want_gpu): preview (GPU, preview res) and export // (CPU, export res) request the same clip/time and must not collide or cross representation. let timestamp_ms = (timestamp * 1000.0) as i64; diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs index a76fc80..0132c8f 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs @@ -316,8 +316,10 @@ fn generate_video_thumbnail( let frame = { let mut video_mgr = video_manager.lock().ok()?; - // Small frame for the asset thumbnail (capped to native, aspect preserved). - video_mgr.get_frame(clip_id, timestamp, THUMBNAIL_SIZE, THUMBNAIL_SIZE)? + // Small CPU frame for the asset thumbnail (capped to native, aspect preserved). Force CPU: + // the render-pass hardware flag may be on, but the thumbnail needs RGBA bytes (a GPU frame + // has empty rgba_data → an all-black thumbnail). + video_mgr.get_frame_cpu(clip_id, timestamp, THUMBNAIL_SIZE, THUMBNAIL_SIZE)? }; let src_width = frame.width as usize; From 02566d571be387e43c3032c999d7afe497e848ca Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 06:27:01 -0400 Subject: [PATCH 23/31] video: fix sped-up + jerky 4K playback (frame-index cache + request-based seek) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs made 4K (and any) video playback wrong: 1. Sped-up playback. The VideoManager frame cache was keyed by milliseconds, but GPU (hardware-decoded) frames bypass the decoder's internal cache. A UI that refreshes finer than the video frame rate (a 60Hz canvas on a 30fps clip) therefore missed the cache on every sub-frame request and decoded the NEXT frame each time — advancing the video ~2x faster than the clock, and racing the decoder toward EOF. Key the cache on the video frame index (round(ts·fps)) instead, so all requests within one frame share an entry and each frame decodes exactly once → correct speed. 2. Periodic seek/jerk. The seek decision compared the rounded request frame_ts to the decoded frame's exact PTS, which sit on slightly different grids — so frame_ts landed ~1 frame "behind" the just-decoded PTS every ~10 frames and falsely read as a backward seek (40ms seek + GOP re-decode). Track the previous request (last_requested_ts), which is strictly monotonic during forward playback, and detect backward from that instead. Real scrubs still seek. Per-frame HW decode is ~3-5ms; both bugs were spurious decode/seek work on top. Keeps a gated LB_VIDEO_DEBUG [Video Seek?] diagnostic. Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-core/src/video.rs | 47 +++++++++++++++---- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 67770d4..9b57350 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -96,6 +96,10 @@ pub struct VideoDecoder { input: Option, decoder: Option, last_decoded_ts: i64, // Track the last decoded frame timestamp + /// The `frame_ts` requested by the previous `get_frame` call. Used to detect a genuine backward + /// jump (scrub/loop) from the request stream, which — unlike the decoded PTS — is strictly + /// monotonic during forward playback, so it never falses on the request-vs-PTS grid mismatch. + last_requested_ts: i64, keyframe_positions: Vec, // Index of keyframe timestamps for fast seeking /// Reused RGBA scaler, keyed by `(input format, input w, input h, output w, output h)`. /// Building an swscale context isn't free; a stream's frames share one input format/size and a @@ -223,6 +227,7 @@ impl VideoDecoder { input: None, decoder: None, last_decoded_ts: -1, + last_requested_ts: i64::MIN, keyframe_positions, scaler: None, hw_device: None, @@ -353,12 +358,29 @@ impl VideoDecoder { } } - // Determine if we need to seek - // Seek if: no decoder open, going backwards, or jumping forward more than 2 seconds + // Determine if we need to seek. Seek if: no decoder open, the *request* jumped backward + // (scrub/step-back/loop), or the request is more than 2s ahead of the decoder's position. + // + // Detect "backward" from the request stream, NOT the decoded frame PTS. The requested + // presentation time is rounded to a frame and converted through floats + // (`round(ts/fd)*fd / time_base`), which routinely lands a frame *behind* the exact PTS of + // the frame just produced — so `frame_ts < last_decoded_ts` falsely fires during smooth + // forward playback (every ~10 frames → a 40ms seek + whole-GOP re-decode = the 4K jerk). + // The request `frame_ts` is strictly monotonic while playing forward, so comparing against + // the previous request never falses; a real backward scrub still decreases it. let need_seek = self.decoder.is_none() - || frame_ts < self.last_decoded_ts + || frame_ts < self.last_requested_ts || frame_ts > self.last_decoded_ts + (2.0 / self.time_base) as i64; + if need_seek && video_debug() { + let reason = if self.decoder.is_none() { "no decoder" } + else if frame_ts < self.last_requested_ts { "backward" } + else { "forward>2s" }; + eprintln!("[Video Seek?] need_seek={reason} frame_ts={frame_ts} last_requested_ts={} last_decoded_ts={}", + self.last_requested_ts, self.last_decoded_ts); + } + self.last_requested_ts = frame_ts; + if need_seek { let t_seek_start = Instant::now(); @@ -951,19 +973,26 @@ impl VideoManager { } fn get_frame_inner(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Option> { - // The cache key includes (target size, want_gpu): preview (GPU, preview res) and export + // Get the decoder for this clip. Clone the Arc so we don't hold a borrow of + // `self.decoders` across the `&mut self` cache insert below. + let decoder_arc = Arc::clone(self.decoders.get(clip_id)?); + + // Key the cache on the video FRAME INDEX, not milliseconds. A UI that refreshes finer than + // the video frame rate (e.g. a 60 Hz canvas on a 30 fps clip) would otherwise miss the cache + // on every sub-frame request and decode the NEXT frame each time — advancing the video faster + // than the clock (it plays sped-up). Rounding to round(ts·fps) maps all requests within one + // frame to a single entry, so each frame is decoded exactly once. + let fps = decoder_arc.lock().ok()?.fps; + let frame_index = if fps > 0.0 { (timestamp * fps).round() as i64 } else { (timestamp * 1000.0) as i64 }; + // The key also includes (target size, want_gpu): preview (GPU, preview res) and export // (CPU, export res) request the same clip/time and must not collide or cross representation. - let timestamp_ms = (timestamp * 1000.0) as i64; - let cache_key = (*clip_id, timestamp_ms, target_w, target_h, want_gpu); + let cache_key = (*clip_id, frame_index, target_w, target_h, want_gpu); // Check frame cache first if let Some(cached_frame) = self.frame_cache.get(&cache_key) { return Some(Arc::clone(cached_frame)); } - // Get decoder for this clip. Clone the Arc so we don't hold a borrow of - // `self.decoders` across the `&mut self` cache insert below. - let decoder_arc = Arc::clone(self.decoders.get(clip_id)?); let mut decoder = decoder_arc.lock().ok()?; // Decode the frame at the requested target (capped to native by the decoder). From fffcf0679cbf2637f032d0081a68c21668b23340 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 06:54:49 -0400 Subject: [PATCH 24/31] aspect ratio: unify video placement + add export fit modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A) Imported video had a different aspect ratio depending on how it was placed: direct import used a uniform scale + center (preserve aspect), while the asset-library timeline drag used an independent scale_x/scale_y (stretch to fill). Add Transform::fit_centered (uniform scale, centered, aspect-preserving) and route both paths through it, so a clip looks identical however it's added. B) Exported video was stretched when the export resolution's aspect differed from the document's (base_transform was always scale_non_uniform). Add ExportFitMode {Stretch, Letterbox (default), Crop} on VideoExportSettings + a "Fit" dropdown in advanced export settings, and a shared export_base_transform() helper: Letterbox = uniform fit centered (black bars), Crop = uniform fill centered (trim), Stretch = the old distort-to-fill. Threaded through the software, HDR, and zero-copy export render paths (image export is doc-sized → identity). Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-core/src/export.rs | 28 ++++++++++++ .../lightningbeam-core/src/object.rs | 15 +++++++ .../lightningbeam-editor/src/export/dialog.rs | 13 ++++++ .../lightningbeam-editor/src/export/mod.rs | 16 ++++++- .../src/export/video_exporter.rs | 45 ++++++++++++------- .../lightningbeam-editor/src/main.rs | 21 +-------- .../src/panes/timeline.rs | 15 ++----- 7 files changed, 103 insertions(+), 50 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/export.rs b/lightningbeam-ui/lightningbeam-core/src/export.rs index 33bfdc9..21b9d46 100644 --- a/lightningbeam-ui/lightningbeam-core/src/export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/export.rs @@ -320,6 +320,29 @@ impl HdrExportMode { } } +/// How the document is fit into the export frame when the export resolution's aspect ratio differs +/// from the document's. Applied as the export `base_transform` (document space → export pixels). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum ExportFitMode { + /// Scale each axis independently to fill the frame — distorts when aspects differ. + Stretch, + /// Scale uniformly to fit, centered, with black bars (letterbox/pillarbox). Preserves aspect. + #[default] + Letterbox, + /// Scale uniformly to fill, centered, cropping the overflow. Preserves aspect, no bars. + Crop, +} + +impl ExportFitMode { + pub fn name(&self) -> &'static str { + match self { + ExportFitMode::Stretch => "Stretch (distort to fill)", + ExportFitMode::Letterbox => "Letterbox (fit, black bars)", + ExportFitMode::Crop => "Crop (fill, trim edges)", + } + } +} + /// Video quality presets #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum VideoQuality { @@ -385,6 +408,10 @@ pub struct VideoExportSettings { #[serde(default)] pub hdr: HdrExportMode, + /// How the document is fit into the export frame when aspect ratios differ (default Letterbox). + #[serde(default)] + pub fit: ExportFitMode, + /// Audio settings (None = no audio) pub audio: Option, @@ -405,6 +432,7 @@ impl Default for VideoExportSettings { quality: VideoQuality::High, color_range: ColorRange::Limited, hdr: HdrExportMode::Sdr, + fit: ExportFitMode::Letterbox, audio: Some(AudioExportSettings::high_quality_aac()), start_time: 0.0, end_time: 60.0, diff --git a/lightningbeam-ui/lightningbeam-core/src/object.rs b/lightningbeam-ui/lightningbeam-core/src/object.rs index 601bede..2e63008 100644 --- a/lightningbeam-ui/lightningbeam-core/src/object.rs +++ b/lightningbeam-ui/lightningbeam-core/src/object.rs @@ -47,6 +47,21 @@ impl Transform { Self::default() } + /// Set scale + position so `content_w × content_h` is fit **uniformly and centered** within + /// `doc_w × doc_h`, preserving aspect ratio (letterbox/pillarbox). The single shared way to + /// place a media clip (video/image) on the document — both import paths use this so a clip + /// looks identical however it was added. No-op for non-positive content dimensions. + pub fn fit_centered(&mut self, content_w: f64, content_h: f64, doc_w: f64, doc_h: f64) { + if content_w <= 0.0 || content_h <= 0.0 { + return; + } + let scale = (doc_w / content_w).min(doc_h / content_h); + self.scale_x = scale; + self.scale_y = scale; + self.x = (doc_w - content_w * scale) / 2.0; + self.y = (doc_h - content_h * scale) / 2.0; + } + /// Create a transform with position pub fn with_position(x: f64, y: f64) -> Self { Self { diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs index 6d06263..65e4f2b 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs @@ -504,6 +504,19 @@ impl ExportDialog { } }); + // Fit mode — how the document maps into the export frame when the aspect ratios differ. + ui.horizontal(|ui| { + use lightningbeam_core::export::ExportFitMode; + ui.label("Fit:"); + egui::ComboBox::from_id_salt("video_fit_mode") + .selected_text(self.video_settings.fit.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.video_settings.fit, ExportFitMode::Letterbox, ExportFitMode::Letterbox.name()); + ui.selectable_value(&mut self.video_settings.fit, ExportFitMode::Crop, ExportFitMode::Crop.name()); + ui.selectable_value(&mut self.video_settings.fit, ExportFitMode::Stretch, ExportFitMode::Stretch.name()); + }); + }); + ui.horizontal(|ui| { ui.label("FPS:"); egui::ComboBox::from_id_salt("framerate") diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 5673da1..9c62936 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -55,6 +55,8 @@ pub struct VideoExportState { height: u32, /// HDR output mode — HDR uses a synchronous 10-bit path instead of the async RGBA pipeline. hdr: lightningbeam_core::export::HdrExportMode, + /// How the document is fit into the export frame (stretch/letterbox/crop). + fit: lightningbeam_core::export::ExportFitMode, /// Channel to send rendered frames to encoder thread frame_tx: Option>, /// HDR GPU resources for compositing pipeline (effects, color conversion) @@ -84,6 +86,8 @@ struct ZeroCopyVideo { rgba: wgpu::Texture, /// True when running on the shared device → compositing can consume hardware-decoded GPU frames. on_shared_device: bool, + /// How the document is fit into the export frame (stretch/letterbox/crop). + fit: lightningbeam_core::export::ExportFitMode, } /// State for a single-frame image export (runs on the GPU render thread, one frame per update). @@ -628,6 +632,8 @@ impl ExportOrchestrator { state.settings.allow_transparency, raster_store, true, // image export composites on the shared device + // Image export renders at the document's own size, so the fit transform is identity. + lightningbeam_core::export::ExportFitMode::Letterbox, )?; queue.submit(Some(encoder.finish())); @@ -861,6 +867,7 @@ impl ExportOrchestrator { height: u32, ) -> (std::thread::JoinHandle<()>, VideoExportState) { let hdr = settings.hdr; + let fit = settings.fit; let handle = std::thread::spawn(move || { Self::run_video_encoder(settings, output_path, frame_rx, progress_tx, cancel_flag, total_frames); }); @@ -874,6 +881,7 @@ impl ExportOrchestrator { width, height, hdr, + fit, frame_tx: Some(frame_tx), gpu_resources: None, readback_pipeline: None, @@ -959,7 +967,7 @@ impl ExportOrchestrator { view_formats: &[], }); println!("🎬 [EXPORT] zero-copy VAAPI H.264 enabled"); - Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba, on_shared_device }) + Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba, on_shared_device, fit: settings.fit }) } /// Start a video export in the background. @@ -1270,6 +1278,7 @@ impl ExportOrchestrator { let width = state.width; let height = state.height; + let fit = state.fit; // HDR path: synchronous 10-bit render (composite → PQ/HLG → readback → 10-bit YUV), one // frame per call. Bypasses the SDR async RGBA pipeline (which is 8-bit only). @@ -1284,7 +1293,7 @@ impl ExportOrchestrator { let (y, u, v) = video_exporter::render_frame_to_yuv10_hdr( document, timestamp, width, height, device, queue, renderer, image_cache, video_manager, - gpu_resources, state.hdr, raster_store, + gpu_resources, state.hdr, fit, raster_store, )?; if let Some(tx) = &state.frame_tx { tx.send(VideoFrameMessage::Frame { @@ -1414,6 +1423,7 @@ impl ExportOrchestrator { false, // Video export is never transparent raster_store, true, // software export composites on the shared device → may use HW frames + fit, )?; let render_end = Instant::now(); @@ -1504,6 +1514,7 @@ impl ExportOrchestrator { let rgba_view = zc.rgba.create_view(&Default::default()); let t0 = std::time::Instant::now(); + let fit = zc.fit; let cmd = match video_exporter::render_frame_to_gpu_rgba( &mut document, timestamp, @@ -1520,6 +1531,7 @@ impl ExportOrchestrator { false, Some(&raster_store), zc.on_shared_device, // GPU-resident decode only when on the shared device + fit, ) { Ok(cmd) => cmd, Err(e) => { diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 696a222..1c65ace 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -16,6 +16,30 @@ use lightningbeam_core::gpu::{ SrgbToLinearConverter, EffectProcessor, YuvConverter, HDR_FORMAT, }; +/// The document→export-pixels transform for a given fit mode. Stretch distorts to fill; Letterbox +/// scales uniformly to fit (centered, black bars); Crop scales uniformly to fill (centered, trims). +pub fn export_base_transform( + doc_w: f64, + doc_h: f64, + out_w: f64, + out_h: f64, + fit: lightningbeam_core::export::ExportFitMode, +) -> vello::kurbo::Affine { + use lightningbeam_core::export::ExportFitMode; + use vello::kurbo::Affine; + if doc_w <= 0.0 || doc_h <= 0.0 { + return Affine::IDENTITY; + } + let (sx, sy) = (out_w / doc_w, out_h / doc_h); + match fit { + ExportFitMode::Stretch => Affine::scale_non_uniform(sx, sy), + ExportFitMode::Letterbox | ExportFitMode::Crop => { + let s = if matches!(fit, ExportFitMode::Letterbox) { sx.min(sy) } else { sx.max(sy) }; + Affine::translate(((out_w - doc_w * s) / 2.0, (out_h - doc_h * s) / 2.0)) * Affine::scale(s) + } + } +} + /// Reusable frame buffers to avoid allocations struct FrameBuffers { /// RGBA buffer from GPU readback (width * height * 4 bytes) @@ -1402,18 +1426,13 @@ pub fn render_frame_to_yuv10_hdr( video_manager: &Arc>, gpu_resources: &mut ExportGpuResources, hdr_mode: lightningbeam_core::export::HdrExportMode, + fit: lightningbeam_core::export::ExportFitMode, raster_store: Option<&lightningbeam_core::raster_store::RasterStore>, ) -> Result<(Vec, Vec, Vec), String> { - use vello::kurbo::Affine; - document.current_time = timestamp; fault_in_raster_for_frame(document, raster_store); - let base_transform = if document.width > 0.0 && document.height > 0.0 { - Affine::scale_non_uniform(width as f64 / document.width, height as f64 / document.height) - } else { - Affine::IDENTITY - }; + let base_transform = export_base_transform(document.width, document.height, width as f64, height as f64, fit); // HDR export composites on the shared device, so it can consume hardware-decoded GPU frames. if let Ok(mut vm) = video_manager.lock() { @@ -1454,9 +1473,8 @@ pub fn render_frame_to_gpu_rgba( // True when compositing on the shared device (software/image export) → may consume // hardware-decoded GPU frames; false for the zero-copy path on its own device. hardware_ok: bool, + fit: lightningbeam_core::export::ExportFitMode, ) -> Result { - use vello::kurbo::Affine; - // One-shot profiling of the render-bucket split (LB_RENDER_PROFILE=1): how much of the // per-frame CPU "render" is document build (incl. video decode) vs. composite-command // recording (incl. the frame texture upload) vs. the sRGB pass. Prints a running average. @@ -1476,14 +1494,7 @@ pub fn render_frame_to_gpu_rgba( // base transform into every layer (vector scenes, raster and video layer // transforms), so the whole stage scales up/down to fill the output. When the // export size matches the document this is the identity. - let base_transform = if document.width > 0.0 && document.height > 0.0 { - Affine::scale_non_uniform( - width as f64 / document.width, - height as f64 / document.height, - ) - } else { - Affine::IDENTITY - }; + let base_transform = export_base_transform(document.width, document.height, width as f64, height as f64, fit); // GPU frames are usable only on the shared device (software/image export); the zero-copy path // runs on its own device and must download to CPU. diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 8c77c48..07fe8a0 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -4974,25 +4974,8 @@ impl EditorApp { if asset_info.clip_type == panes::DragClipType::Video { if let Some((video_width, video_height)) = asset_info.dimensions { let doc = self.action_executor.document(); - let doc_width = doc.width; - let doc_height = doc.height; - - // Calculate scale to fit (use minimum to preserve aspect ratio) - let scale_x = doc_width / video_width; - let scale_y = doc_height / video_height; - let uniform_scale = scale_x.min(scale_y); - - clip_instance.transform.scale_x = uniform_scale; - clip_instance.transform.scale_y = uniform_scale; - - // Center the video in the document - let scaled_width = video_width * uniform_scale; - let scaled_height = video_height * uniform_scale; - let center_x = (doc_width - scaled_width) / 2.0; - let center_y = (doc_height - scaled_height) / 2.0; - - clip_instance.transform.x = center_x; - clip_instance.transform.y = center_y; + // Fit uniformly + centered (preserve aspect). Shared with the timeline drag path. + clip_instance.transform.fit_centered(video_width, video_height, doc.width, doc.height); } } else { // Audio clips are centered in document diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index 7762112..5cfdca3 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -6151,20 +6151,11 @@ impl PaneRenderer for TimelinePane { let mut clip_instance = ClipInstance::new(dragging.clip_id) .with_timeline_start(drop_time); - // For video clips, scale to fill document dimensions + // For video clips, fit uniformly + centered (preserve aspect). + // Shared with the direct-import path via Transform::fit_centered. if dragging.clip_type == DragClipType::Video { if let Some((video_width, video_height)) = dragging.dimensions { - // Calculate scale to fill document - let scale_x = doc.width / video_width; - let scale_y = doc.height / video_height; - - clip_instance.transform.scale_x = scale_x; - clip_instance.transform.scale_y = scale_y; - - // Position at (0, 0) to center the scaled video - // (scaled dimensions = document dimensions, so top-left at origin centers it) - clip_instance.transform.x = 0.0; - clip_instance.transform.y = 0.0; + clip_instance.transform.fit_centered(video_width, video_height, doc.width, doc.height); } else { // No dimensions available, use document center clip_instance.transform.x = center_x; From 5bc117c5181ba0d896950ffbc6cdb361804bbb6c Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 14:39:18 -0400 Subject: [PATCH 25/31] export: add Fit mode to image export too ImageExportSettings gains `fit: ExportFitMode` (default Letterbox) + a "Fit" dropdown in advanced image settings; the image render path uses it instead of the hardcoded Letterbox. Image export supports a resolution override, so the mode matters when the override aspect differs from the document. --- lightningbeam-ui/lightningbeam-core/src/export.rs | 5 ++++- .../lightningbeam-editor/src/export/dialog.rs | 13 +++++++++++++ .../lightningbeam-editor/src/export/mod.rs | 4 ++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/export.rs b/lightningbeam-ui/lightningbeam-core/src/export.rs index 21b9d46..6e62d50 100644 --- a/lightningbeam-ui/lightningbeam-core/src/export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/export.rs @@ -524,11 +524,14 @@ pub struct ImageExportSettings { /// When false, the image is composited onto an opaque background before encoding. /// Only meaningful for formats that support alpha (PNG, WebP). pub allow_transparency: bool, + /// How the document is fit into the output frame when aspect ratios differ (default Letterbox). + #[serde(default)] + pub fit: ExportFitMode, } impl Default for ImageExportSettings { fn default() -> Self { - Self { format: ImageFormat::Png, time: 0.0, width: None, height: None, quality: 90, allow_transparency: false } + Self { format: ImageFormat::Png, time: 0.0, width: None, height: None, quality: 90, allow_transparency: false, fit: ExportFitMode::Letterbox } } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs index 65e4f2b..ec716d7 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs @@ -378,6 +378,19 @@ impl ExportDialog { if changed_h { self.image_settings.height = if h == 0 { None } else { Some(h) }; } ui.weak("(0 = document size)"); }); + + // Fit mode — how the document maps into the output frame when aspect ratios differ. + ui.horizontal(|ui| { + use lightningbeam_core::export::ExportFitMode; + ui.label("Fit:"); + egui::ComboBox::from_id_salt("image_fit_mode") + .selected_text(self.image_settings.fit.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.image_settings.fit, ExportFitMode::Letterbox, ExportFitMode::Letterbox.name()); + ui.selectable_value(&mut self.image_settings.fit, ExportFitMode::Crop, ExportFitMode::Crop.name()); + ui.selectable_value(&mut self.image_settings.fit, ExportFitMode::Stretch, ExportFitMode::Stretch.name()); + }); + }); } /// Render advanced audio settings (sample rate, channels, bit depth, bitrate, time range) diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 9c62936..7b7a7fc 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -598,6 +598,7 @@ impl ExportOrchestrator { // ── First call: render the frame to the GPU output texture ──────── let w = state.width; let h = state.height; + let fit = state.settings.fit; if state.gpu_resources.is_none() { state.gpu_resources = Some(video_exporter::ExportGpuResources::new(device, w, h)); @@ -632,8 +633,7 @@ impl ExportOrchestrator { state.settings.allow_transparency, raster_store, true, // image export composites on the shared device - // Image export renders at the document's own size, so the fit transform is identity. - lightningbeam_core::export::ExportFitMode::Letterbox, + fit, )?; queue.submit(Some(encoder.finish())); From 69939c066db7e7c517b6cadf64135761bba657f7 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 15:00:28 -0400 Subject: [PATCH 26/31] stage: dashed black/yellow border around the active raster layer Outline the active raster layer's canvas bounds (document space) so its size is visible, especially when it differs from the document after a resize. Two strokes sharing one dash pattern, the yellow offset by a dash to fill the black's gaps - interlocking marching-ants. Sizes divided by zoom so the dashes stay ~constant on screen. --- .../lightningbeam-editor/src/panes/stage.rs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index fc8daa0..46beb28 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -2074,6 +2074,37 @@ impl egui_wgpu::CallbackTrait for VelloCallback { scene }; + // Active raster-layer border: a black+yellow dashed outline of the layer's canvas bounds (in + // document space) so its size is visible — especially when it differs from the document (e.g. + // after a doc resize). Two strokes sharing one dash pattern, offset by a dash so the yellow + // fills the black's gaps → interlocking black/yellow marching-ants. + if let Some(active_id) = self.ctx.active_layer_id { + if let Some(lightningbeam_core::layer::AnyLayer::Raster(rl)) = self.ctx.document.get_layer(&active_id) { + if let Some(kf) = rl.keyframe_at(self.ctx.document.current_time) { + let rect = vello::kurbo::Rect::new(0.0, 0.0, kf.width as f64, kf.height as f64); + // Sizes are in document space; divide by zoom so they're ~constant on screen. + let inv_zoom = 1.0 / (self.ctx.zoom as f64).max(1e-3); + let stroke_w = 1.5 * inv_zoom; + let dash = 6.0 * inv_zoom; + let pattern = [dash, dash]; + scene.stroke( + &vello::kurbo::Stroke::new(stroke_w).with_dashes(0.0, pattern), + camera_transform, + vello::peniko::Color::new([0.0, 0.0, 0.0, 1.0]), + None, + &rect, + ); + scene.stroke( + &vello::kurbo::Stroke::new(stroke_w).with_dashes(dash, pattern), + camera_transform, + vello::peniko::Color::new([1.0, 0.85, 0.0, 1.0]), + None, + &rect, + ); + } + } + } + // Render drag preview objects with transparency if let (Some(delta), Some(active_layer_id)) = (self.ctx.drag_delta, self.ctx.active_layer_id) { if let Some(layer) = self.ctx.document.get_layer(&active_layer_id) { From 5869e3ced17ca461c78f5198d15105c6d785ce36 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 15:40:16 -0400 Subject: [PATCH 27/31] raster: info-panel size + "Layer to document size" (scale/expand-crop) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resizing the document leaves raster layers as-is (canvas keeps its old pixel size, anchored top-left). To reconcile, the info panel now shows the active raster layer's canvas size — driven by the *active* layer, not selection focus, since painting doesn't focus the layer — and, when it differs from the document, a Scale / Expand-Crop toggle + a "Layer to document size" button. - RasterKeyframe::resize_to(w, h, mode): always applies the new declared size; resamples (Lanczos3) when Scale, top-left pad/trim when Canvas, and only touches the buffer when pixels are resident (a blank canvas just takes the new size). Sets texture_dirty so the stage's dirty-scan refreshes the GPU texture. - ResizeRasterLayerAction resizes every keyframe with undo, holding a (read-only) RasterStore so paged-out keyframes are loaded one at a time rather than bulk. Resized keyframes stay resident + dirty and persist on the next full save (no incremental store write to page them back out). Scale is lossy/compounds; Expand-Crop is lossless. --- .../lightningbeam-core/src/actions/mod.rs | 2 + .../src/actions/resize_raster_layer.rs | 90 +++++++++++++++++++ .../lightningbeam-core/src/raster_layer.rs | 51 +++++++++++ .../src/panes/infopanel.rs | 56 ++++++++++++ 4 files changed, 199 insertions(+) create mode 100644 lightningbeam-ui/lightningbeam-core/src/actions/resize_raster_layer.rs diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs b/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs index ed209cf..29db4f5 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs @@ -40,6 +40,7 @@ pub mod raster_diff; pub mod raster_stroke; pub mod raster_fill; pub mod add_raster_keyframe; +pub mod resize_raster_layer; pub mod move_layer; pub mod set_fill_paint; pub mod set_image_fill; @@ -77,6 +78,7 @@ pub use group_layers::GroupLayersAction; pub use raster_stroke::RasterStrokeAction; pub use raster_fill::RasterFillAction; pub use add_raster_keyframe::AddRasterKeyframeAction; +pub use resize_raster_layer::ResizeRasterLayerAction; pub use move_layer::MoveLayerAction; pub use set_fill_paint::SetFillPaintAction; pub use set_image_fill::SetImageFillAction; diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/resize_raster_layer.rs b/lightningbeam-ui/lightningbeam-core/src/actions/resize_raster_layer.rs new file mode 100644 index 0000000..7fa5420 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-core/src/actions/resize_raster_layer.rs @@ -0,0 +1,90 @@ +//! Resize every keyframe of a raster layer to a new canvas size (Scale or Canvas mode), with undo. +//! +//! Used by the info panel's "Layer to document size" button. Pixels must be resident (the editor +//! faults them in first) so the resample/copy is exact and a later page-in won't mismatch the size. + +use crate::action::Action; +use crate::document::Document; +use crate::layer::AnyLayer; +use crate::raster_layer::RasterResizeMode; +use crate::raster_store::RasterStore; +use uuid::Uuid; + +/// Per-keyframe state captured for undo. +struct OldKeyframe { + id: Uuid, + width: u32, + height: u32, + pixels: Vec, +} + +pub struct ResizeRasterLayerAction { + layer_id: Uuid, + new_w: u32, + new_h: u32, + mode: RasterResizeMode, + /// Read-only page-in for keyframes whose pixels aren't resident. (No incremental write exists, so + /// resized keyframes stay resident + dirty and persist on the next full save.) + store: RasterStore, + /// Captured on first execute for rollback. + old: Option>, +} + +impl ResizeRasterLayerAction { + pub fn new(layer_id: Uuid, new_w: u32, new_h: u32, mode: RasterResizeMode, store: RasterStore) -> Self { + Self { layer_id, new_w, new_h, mode, store, old: None } + } +} + +impl Action for ResizeRasterLayerAction { + fn execute(&mut self, document: &mut Document) -> Result<(), String> { + let Some(AnyLayer::Raster(rl)) = document.get_layer_mut(&self.layer_id) else { + return Err("ResizeRasterLayerAction: layer is not a raster layer".into()); + }; + let capture = self.old.is_none(); + let mut old = Vec::new(); + for kf in rl.keyframes.iter_mut() { + // Page the keyframe in one at a time so we never hold the whole layer in memory at once. + if kf.raw_pixels.is_empty() { + if let Some(px) = self.store.load_pixels(kf.id) { + kf.raw_pixels = px; + kf.needs_fault_in = false; + } + } + if capture { + old.push(OldKeyframe { id: kf.id, width: kf.width, height: kf.height, pixels: kf.raw_pixels.clone() }); + } + kf.resize_to(self.new_w, self.new_h, self.mode); + } + if capture { + self.old = Some(old); + } + Ok(()) + } + + fn rollback(&mut self, document: &mut Document) -> Result<(), String> { + let Some(AnyLayer::Raster(rl)) = document.get_layer_mut(&self.layer_id) else { + return Err("ResizeRasterLayerAction: layer is not a raster layer".into()); + }; + if let Some(old) = &self.old { + for o in old { + if let Some(kf) = rl.keyframes.iter_mut().find(|kf| kf.id == o.id) { + kf.width = o.width; + kf.height = o.height; + kf.raw_pixels = o.pixels.clone(); + kf.proxy = None; + kf.texture_dirty = true; + kf.dirty = true; + } + } + } + Ok(()) + } + + fn description(&self) -> String { + match self.mode { + RasterResizeMode::Scale => "Scale raster layer".into(), + RasterResizeMode::Canvas => "Resize raster canvas".into(), + } + } +} diff --git a/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs b/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs index 6a14d11..d158fd9 100644 --- a/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs @@ -169,6 +169,57 @@ impl RasterKeyframe { proxy: None, } } + + /// Change the canvas to `(new_w, new_h)`. `Scale` resamples the content (Lanczos3) to fill the + /// new size; `Canvas` keeps the content at native resolution anchored top-left, padding with + /// transparent (expand) or trimming (crop). No-op if the size is unchanged. If pixels aren't + /// resident the buffer is left empty and only the declared size changes — the caller must fault + /// pixels in first (a later load would otherwise mismatch the new size). + pub fn resize_to(&mut self, new_w: u32, new_h: u32, mode: RasterResizeMode) { + if new_w == 0 || new_h == 0 || (self.width == new_w && self.height == new_h) { + return; + } + // Resample/recanvas the buffer only when pixels are resident. A blank keyframe (no content + // and no store row) just takes the new declared size — there's nothing to corrupt. Paged-out + // keyframes are loaded by the caller (ResizeRasterLayerAction) before this runs. + if !self.raw_pixels.is_empty() { + let old = std::mem::take(&mut self.raw_pixels); + self.raw_pixels = match mode { + RasterResizeMode::Scale => match image::RgbaImage::from_raw(self.width, self.height, old) { + Some(img) => image::imageops::resize(&img, new_w, new_h, image::imageops::FilterType::Lanczos3).into_raw(), + None => vec![0u8; (new_w as usize) * (new_h as usize) * 4], + }, + RasterResizeMode::Canvas => { + // Copy the old pixels into a transparent new buffer, anchored top-left (matching + // the raster's (0,0) document anchor); right/bottom is padded or trimmed. + let mut buf = vec![0u8; (new_w as usize) * (new_h as usize) * 4]; + let copy_w = self.width.min(new_w) as usize; + let copy_h = self.height.min(new_h) as usize; + let (ow, nw) = (self.width as usize, new_w as usize); + for y in 0..copy_h { + let src = y * ow * 4; + let dst = y * nw * 4; + buf[dst..dst + copy_w * 4].copy_from_slice(&old[src..src + copy_w * 4]); + } + buf + } + }; + self.proxy = None; // invalidate any downsampled proxy + } + self.width = new_w; + self.height = new_h; + self.texture_dirty = true; + self.dirty = true; + } +} + +/// How a raster canvas resize treats existing pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum RasterResizeMode { + /// Resample the content to fill the new size (changes pixel resolution). + Scale, + /// Keep content at native resolution, anchored top-left; pad/trim the canvas. + Canvas, } /// A pixel-buffer painting layer diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs index f730d72..0cb524a 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs @@ -44,6 +44,8 @@ pub struct InfopanelPane { selected_tool_gradient_stop: Option, /// FPS value captured when a drag/focus-in starts (for single-undo-action on commit) fps_drag_start: Option, + /// Resize mode for the active raster layer's "to document size" action (scale vs canvas). + raster_resize_mode: lightningbeam_core::raster_layer::RasterResizeMode, } impl InfopanelPane { @@ -61,6 +63,7 @@ impl InfopanelPane { selected_shape_gradient_stop: None, selected_tool_gradient_stop: None, fps_drag_start: None, + raster_resize_mode: lightningbeam_core::raster_layer::RasterResizeMode::Scale, } } } @@ -1197,6 +1200,53 @@ impl InfopanelPane { }); } + /// Render a raster-layer section: shows the active keyframe's canvas dimensions and, when they + /// differ from the document, a Scale/Expand-Crop mode toggle + a "Layer to document size" button. + /// Driven by the *active* layer (not selection focus), since painting doesn't focus the layer. + fn render_raster_layer_section(&mut self, ui: &mut Ui, path: &NodePath, shared: &mut SharedPaneState, layer_id: Uuid) { + use lightningbeam_core::raster_layer::RasterResizeMode; + + // Pull the values, then drop the document borrow before mutating `shared`. + let time = *shared.playback_time; + let dims = { + let document = shared.action_executor.document(); + match document.get_layer(&layer_id) { + Some(AnyLayer::Raster(rl)) => rl + .keyframe_at(time) + .map(|kf| (kf.width, kf.height, document.width as u32, document.height as u32)), + _ => None, + } + }; + let Some((kf_w, kf_h, doc_w, doc_h)) = dims else { return }; + + egui::CollapsingHeader::new("Raster Layer") + .id_salt(("raster_layer", path)) + .default_open(true) + .show(ui, |ui| { + ui.add_space(4.0); + ui.horizontal(|ui| { + ui.label("Size:"); + ui.label(format!("{} × {}", kf_w, kf_h)); + }); + + if kf_w != doc_w || kf_h != doc_h { + ui.horizontal(|ui| { + ui.label("Mode:"); + ui.selectable_value(&mut self.raster_resize_mode, RasterResizeMode::Scale, "Scale"); + ui.selectable_value(&mut self.raster_resize_mode, RasterResizeMode::Canvas, "Expand/Crop"); + }); + if ui.button(format!("Layer to document size ({} × {})", doc_w, doc_h)).clicked() { + let store = lightningbeam_core::raster_store::RasterStore::new(shared.container_path.clone()); + let action = lightningbeam_core::actions::ResizeRasterLayerAction::new( + layer_id, doc_w, doc_h, self.raster_resize_mode, store, + ); + shared.pending_actions.push(Box::new(action)); + } + } + ui.add_space(4.0); + }); + } + /// Render clip instance info section fn render_clip_instance_section(&self, ui: &mut Ui, path: &NodePath, shared: &SharedPaneState, clip_ids: &[Uuid]) { let document = shared.action_executor.document(); @@ -1547,6 +1597,12 @@ impl PaneRenderer for InfopanelPane { } } + // Active raster layer's size + "to document size" — shown whenever a raster layer is + // active (independent of selection focus, since painting doesn't focus the layer). + if let Some(active_id) = *shared.active_layer_id { + self.render_raster_layer_section(ui, path, shared, active_id); + } + // Onion-skinning view settings — always available, regardless of selection. ui.add_space(8.0); ui.separator(); From 1fa4d744be604f9081e5d300a0f1e7dfb0ae0905 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 16:40:29 -0400 Subject: [PATCH 28/31] Add SVG vector import and export Export (lightningbeam-core/svg_export.rs): document_to_svg walks vector layers/groups at a given frame and emits per fill (solid or gradient via ) and per stroked edge. Wired into the export dialog as an "SVG" tab; written synchronously. Raster/video/effect layers are skipped (vector-only, lossless), structured for a later rasterize pass. Import (lightningbeam-editor/svg_import.rs): import_svg parses via usvg, bakes each path's absolute transform into geometry, converts segments to cubic edges, and maps solid/linear/radial paint to ShapeColor/ ShapeGradient. .svg is detected in the Ctrl+I Import handler and added as a new vector layer (keyframe at the playhead). file_types gains FileType::Vector + VECTOR_EXTENSIONS. Tests: svg_export::export_tests (core) and svg_import::tests (editor). --- .../lightningbeam-core/src/file_types.rs | 12 +- .../lightningbeam-core/src/renderer.rs | 2 +- .../lightningbeam-core/src/svg_export.rs | 259 ++++++++++++++ .../lightningbeam-editor/src/export/dialog.rs | 26 +- .../lightningbeam-editor/src/main.rs | 70 +++- .../lightningbeam-editor/src/svg_import.rs | 315 ++++++++++++++++++ 6 files changed, 678 insertions(+), 6 deletions(-) create mode 100644 lightningbeam-ui/lightningbeam-editor/src/svg_import.rs diff --git a/lightningbeam-ui/lightningbeam-core/src/file_types.rs b/lightningbeam-ui/lightningbeam-core/src/file_types.rs index 464ff36..01b837c 100644 --- a/lightningbeam-ui/lightningbeam-core/src/file_types.rs +++ b/lightningbeam-ui/lightningbeam-core/src/file_types.rs @@ -15,7 +15,9 @@ pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv", "webm", "m4v /// Supported MIDI file extensions pub const MIDI_EXTENSIONS: &[&str] = &["mid", "midi"]; -// Note: SVG import deferred to future task +/// Supported vector file extensions (imported as a new vector layer, not an asset) +pub const VECTOR_EXTENSIONS: &[&str] = &["svg"]; + // Note: .beam project files handled separately in file save/load feature /// File type categories for import routing @@ -25,6 +27,7 @@ pub enum FileType { Audio, Video, Midi, + Vector, } /// Detect file type from extension string @@ -53,6 +56,9 @@ pub fn get_file_type(extension: &str) -> Option { if MIDI_EXTENSIONS.contains(&ext.as_str()) { return Some(FileType::Midi); } + if VECTOR_EXTENSIONS.contains(&ext.as_str()) { + return Some(FileType::Vector); + } None } @@ -65,6 +71,7 @@ pub fn all_supported_extensions() -> Vec<&'static str> { all.extend_from_slice(AUDIO_EXTENSIONS); all.extend_from_slice(VIDEO_EXTENSIONS); all.extend_from_slice(MIDI_EXTENSIONS); + all.extend_from_slice(VECTOR_EXTENSIONS); all } @@ -90,7 +97,8 @@ mod tests { assert_eq!(get_file_type("midi"), Some(FileType::Midi)); assert_eq!(get_file_type("unknown"), None); - assert_eq!(get_file_type("svg"), None); // SVG deferred + assert_eq!(get_file_type("svg"), Some(FileType::Vector)); + assert_eq!(get_file_type("SVG"), Some(FileType::Vector)); } #[test] diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index f838b23..63ffcb3 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -1344,7 +1344,7 @@ fn render_video_layer( /// The axis is centred on the bbox midpoint and oriented at `angle_deg` degrees /// (0 = left→right, 90 = top→bottom). The axis extends ± half the bbox diagonal /// so the gradient covers the entire shape regardless of angle. -fn gradient_bbox_endpoints(angle_deg: f32, bbox: kurbo::Rect) -> (kurbo::Point, kurbo::Point) { +pub(crate) fn gradient_bbox_endpoints(angle_deg: f32, bbox: kurbo::Rect) -> (kurbo::Point, kurbo::Point) { let cx = bbox.center().x; let cy = bbox.center().y; let dx = bbox.width(); diff --git a/lightningbeam-ui/lightningbeam-core/src/svg_export.rs b/lightningbeam-ui/lightningbeam-core/src/svg_export.rs index 527aac3..ad36f80 100644 --- a/lightningbeam-ui/lightningbeam-core/src/svg_export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/svg_export.rs @@ -218,3 +218,262 @@ fn cubic_to_svg_path(curve: &CubicBez) -> String { curve.p3.x, curve.p3.y, ) } + +// =========================================================================== +// Document / VectorGraph → SVG (the current model). The functions above target +// the legacy DCEL and are kept only for the clipboard stub. +// =========================================================================== + +use crate::document::Document; +use crate::gradient::{GradientExtend, GradientType, ShapeGradient}; +use crate::layer::AnyLayer; +use crate::shape::{Cap, FillRule, Join, ShapeColor}; +use crate::vector_graph::{FillId, VectorGraph}; +use kurbo::{BezPath, PathEl, Rect, Shape}; + +/// Serialize the document's **vector** content to a standalone SVG string, at document time `time`. +/// Vector layers (and groups of them) only — raster/video/audio/effect layers are skipped (a later +/// pass can rasterize them to ``). Animation is a single static frame at `time`. +pub fn document_to_svg(document: &Document, time: f64) -> String { + let (w, h) = (document.width, document.height); + let mut defs = String::new(); + let mut body = String::new(); + let mut grad_n = 0usize; + + // Opaque background rect (skip if the document background is transparent). + let bg = document.background_color; + if bg.a > 0 { + body.push_str(&format!( + r#""#, + fill_attrs(&bg) + )); + } + + for layer in &document.root.children { + layer_to_svg(layer, time, 1.0, &mut body, &mut defs, &mut grad_n); + } + + format!( + concat!( + r#"{}{}"# + ), + w, h, w, h, defs, body + ) +} + +/// Append one layer's SVG. Recurses into groups (``); other non-vector layer types are skipped. +fn layer_to_svg(layer: &AnyLayer, time: f64, parent_opacity: f64, body: &mut String, defs: &mut String, grad_n: &mut usize) { + match layer { + AnyLayer::Vector(vl) => { + let opacity = parent_opacity * vl.layer.opacity; + if let Some(graph) = vl.tweened_graph_at(time) { + let wrap = opacity < 0.999; + if wrap { + body.push_str(&format!(r#""#)); + } + vector_graph_to_svg(&graph, body, defs, grad_n); + if wrap { + body.push_str(""); + } + } + // NOTE: placed clip instances (nested clips with their own transform) are not yet + // exported — a refinement once loose-geometry export is verified. + } + AnyLayer::Group(g) => { + let opacity = parent_opacity * g.layer.opacity; + body.push_str(&format!(r#""#)); + for child in &g.children { + layer_to_svg(child, time, 1.0, body, defs, grad_n); + } + body.push_str(""); + } + // Raster/Video/Audio/Effect have no lossless vector representation — skipped this pass. + _ => {} + } +} + +/// Emit a vector graph's fills (``) and stroked edges (``) into `body`, +/// accumulating any gradients into `defs`. Geometry is in document space (no per-layer transform). +fn vector_graph_to_svg(graph: &VectorGraph, body: &mut String, defs: &mut String, grad_n: &mut usize) { + // Fills first (drawn under strokes, matching the renderer). + for (i, fill) in graph.fills.iter().enumerate() { + if fill.deleted { + continue; + } + let path = graph.fill_to_bezpath(FillId(i as u32)); + let d = bezpath_to_d(&path); + if d.is_empty() { + continue; + } + let rule = match fill.fill_rule { + FillRule::NonZero => "nonzero", + FillRule::EvenOdd => "evenodd", + }; + + if let Some(grad) = &fill.gradient_fill { + let id = format!("grad{}", *grad_n); + *grad_n += 1; + defs.push_str(&gradient_to_svg(grad, &id, path.bounding_box())); + body.push_str(&format!(r#""#)); + } else if fill.image_fill.is_some() { + // Image fills need / + asset embedding — skipped this (vector-only) pass. + continue; + } else if let Some(c) = &fill.color { + body.push_str(&format!(r#""#, fill_attrs(c))); + } + } + + // Strokes: one per stroked edge (each edge may carry its own style). + for edge in &graph.edges { + if edge.deleted { + continue; + } + if let (Some(style), Some(color)) = (&edge.stroke_style, &edge.stroke_color) { + let d = cubic_to_svg_path(&edge.curve); + body.push_str(&format!( + r#""#, + stroke_attrs(color), style.width, cap_str(style.cap), join_str(style.join), style.miter_limit + )); + } + } +} + +/// `` / `` definition matching the renderer's start/end semantics. +fn gradient_to_svg(grad: &ShapeGradient, id: &str, bbox: Rect) -> String { + use kurbo::Point; + // Mirror renderer.rs: explicit world endpoints if present (radial reflects the edge through the + // center so midpoint(start,end) == center), else derive from angle + bbox. + let (start, end) = match (grad.start_world, grad.end_world) { + (Some((sx, sy)), Some((ex, ey))) => match grad.kind { + GradientType::Linear => (Point::new(sx, sy), Point::new(ex, ey)), + GradientType::Radial => (Point::new(2.0 * sx - ex, 2.0 * sy - ey), Point::new(ex, ey)), + }, + _ => crate::renderer::gradient_bbox_endpoints(grad.angle, bbox), + }; + + let stops: String = grad + .stops + .iter() + .map(|s| { + format!( + r##""##, + s.position, s.color.r, s.color.g, s.color.b, s.color.a as f32 / 255.0 + ) + }) + .collect(); + let spread = match grad.extend { + GradientExtend::Pad => "pad", + GradientExtend::Reflect => "reflect", + GradientExtend::Repeat => "repeat", + }; + + match grad.kind { + GradientType::Linear => format!( + r#"{stops}"#, + start.x, start.y, end.x, end.y + ), + GradientType::Radial => { + let (cx, cy) = ((start.x + end.x) * 0.5, (start.y + end.y) * 0.5); + let r = (((end.x - start.x).powi(2) + (end.y - start.y).powi(2)).sqrt()) * 0.5; + format!( + r#"{stops}"# + ) + } + } +} + +/// kurbo `BezPath` → SVG path-data string (`M/L/Q/C/Z`). +fn bezpath_to_d(path: &BezPath) -> String { + let mut d = String::new(); + for el in path.elements() { + match el { + PathEl::MoveTo(p) => d.push_str(&format!("M{:.3} {:.3} ", p.x, p.y)), + PathEl::LineTo(p) => d.push_str(&format!("L{:.3} {:.3} ", p.x, p.y)), + PathEl::QuadTo(p1, p) => d.push_str(&format!("Q{:.3} {:.3} {:.3} {:.3} ", p1.x, p1.y, p.x, p.y)), + PathEl::CurveTo(p1, p2, p) => d.push_str(&format!( + "C{:.3} {:.3} {:.3} {:.3} {:.3} {:.3} ", + p1.x, p1.y, p2.x, p2.y, p.x, p.y + )), + PathEl::ClosePath => d.push_str("Z "), + } + } + d.trim_end().to_string() +} + +// sRGB color → SVG attributes. Hex color + a separate `*-opacity` for max compatibility (Inkscape). +fn fill_attrs(c: &ShapeColor) -> String { + if c.a == 255 { + format!(r##"fill="#{:02x}{:02x}{:02x}""##, c.r, c.g, c.b) + } else { + format!(r##"fill="#{:02x}{:02x}{:02x}" fill-opacity="{:.4}""##, c.r, c.g, c.b, c.a as f32 / 255.0) + } +} +fn stroke_attrs(c: &ShapeColor) -> String { + if c.a == 255 { + format!(r##"stroke="#{:02x}{:02x}{:02x}""##, c.r, c.g, c.b) + } else { + format!(r##"stroke="#{:02x}{:02x}{:02x}" stroke-opacity="{:.4}""##, c.r, c.g, c.b, c.a as f32 / 255.0) + } +} +fn cap_str(cap: Cap) -> &'static str { + match cap { + Cap::Butt => "butt", + Cap::Round => "round", + Cap::Square => "square", + } +} +fn join_str(join: Join) -> &'static str { + match join { + Join::Miter => "miter", + Join::Round => "round", + Join::Bevel => "bevel", + } +} + +#[cfg(test)] +mod export_tests { + use super::*; + use crate::shape::{ShapeColor, StrokeStyle}; + use crate::vector_graph::{Direction, VectorGraph}; + use kurbo::{CubicBez, Point}; + + fn line(a: Point, b: Point) -> CubicBez { + // Degenerate cubic representing a straight segment (matches our model). + CubicBez::new(a, a.lerp(b, 1.0 / 3.0), a.lerp(b, 2.0 / 3.0), b) + } + + #[test] + fn solid_triangle_fill_and_stroke() { + let mut g = VectorGraph::new(); + let p0 = Point::new(10.0, 10.0); + let p1 = Point::new(90.0, 10.0); + let p2 = Point::new(50.0, 80.0); + let v0 = g.alloc_vertex(p0); + let v1 = g.alloc_vertex(p1); + let v2 = g.alloc_vertex(p2); + let stroke = Some(StrokeStyle { width: 2.0, ..Default::default() }); + let scol = Some(ShapeColor::rgb(0, 0, 0)); + let e0 = g.alloc_edge(line(p0, p1), v0, v1, stroke.clone(), scol); + let e1 = g.alloc_edge(line(p1, p2), v1, v2, stroke.clone(), scol); + let e2 = g.alloc_edge(line(p2, p0), v2, v0, stroke.clone(), scol); + g.alloc_fill( + vec![(e0, Direction::Forward), (e1, Direction::Forward), (e2, Direction::Forward)], + ShapeColor::rgb(255, 0, 0), + crate::shape::FillRule::NonZero, + ); + + let mut body = String::new(); + let mut defs = String::new(); + let mut n = 0; + vector_graph_to_svg(&g, &mut body, &mut defs, &mut n); + + assert!(body.contains(r##"fill="#ff0000""##), "fill color missing: {body}"); + assert!(body.contains(r#"fill-rule="nonzero""#), "fill-rule missing: {body}"); + assert!(body.contains(r#"fill="none""#), "stroke path missing: {body}"); + assert!(body.contains(r#"stroke-width="2.000""#), "stroke width missing: {body}"); + assert!(defs.is_empty(), "no gradients expected: {defs}"); + // 1 fill path + 3 stroked edges = 4 elements. + assert_eq!(body.matches(" self.audio_settings.format.extension(), ExportType::Image => self.image_settings.format.extension(), ExportType::Video => self.video_settings.codec.container_format(), + ExportType::Svg => "svg", } } @@ -198,6 +203,7 @@ impl ExportDialog { ExportType::Audio => "Export Audio", ExportType::Image => "Export Image", ExportType::Video => "Export Video", + ExportType::Svg => "Export SVG", }; let modal_response = egui::Modal::new(egui::Id::new("export_dialog_modal")) @@ -219,6 +225,7 @@ impl ExportDialog { (ExportType::Audio, "Audio"), (ExportType::Image, "Image"), (ExportType::Video, "Video"), + (ExportType::Svg, "SVG"), ] { if ui.selectable_value(&mut self.export_type, variant, label).clicked() { self.update_filename_extension(); @@ -235,6 +242,7 @@ impl ExportDialog { ExportType::Audio => self.render_audio_basic(ui), ExportType::Image => self.render_image_settings(ui), ExportType::Video => self.render_video_basic(ui), + ExportType::Svg => self.render_svg_settings(ui), } ui.add_space(12.0); @@ -253,6 +261,7 @@ impl ExportDialog { ExportType::Audio => self.render_audio_advanced(ui), ExportType::Image => self.render_image_advanced(ui), ExportType::Video => self.render_video_advanced(ui), + ExportType::Svg => {} // SVG has no advanced settings } } @@ -356,6 +365,20 @@ impl ExportDialog { } } + /// Render SVG export settings — just the frame time (reuses the image time field). + fn render_svg_settings(&mut self, ui: &mut egui::Ui) { + ui.horizontal(|ui| { + ui.label("Time:"); + ui.add(egui::DragValue::new(&mut self.image_settings.time) + .speed(0.01) + .range(0.0..=f64::MAX) + .suffix(" s")); + }); + ui.add_space(4.0); + ui.weak("Exports vector layers losslessly at this frame. Raster, video, and"); + ui.weak("effect layers are not included."); + } + /// Render advanced image export settings (time, resolution override). fn render_image_advanced(&mut self, ui: &mut egui::Ui) { // Time (which frame to export) @@ -595,7 +618,7 @@ impl ExportDialog { fn render_time_range(&mut self, ui: &mut egui::Ui) { let (start_time, end_time) = match self.export_type { ExportType::Audio => (&mut self.audio_settings.start_time, &mut self.audio_settings.end_time), - ExportType::Image => return, // image uses a single time field, not a range + ExportType::Image | ExportType::Svg => return, // single time field, not a range ExportType::Video => (&mut self.video_settings.start_time, &mut self.video_settings.end_time), }; @@ -669,6 +692,7 @@ impl ExportDialog { } Some(ExportResult::Image(self.image_settings.clone(), output_path)) } + ExportType::Svg => Some(ExportResult::Svg(self.image_settings.time, output_path)), ExportType::Audio => { // Validate audio settings if let Err(err) = self.audio_settings.validate() { diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 07fe8a0..7073e2d 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -60,6 +60,7 @@ mod test_mode; mod sample_import; mod sample_import_dialog; +mod svg_import; mod curve_editor; @@ -3238,7 +3239,11 @@ impl EditorApp { .and_then(|e| e.to_str()) .unwrap_or(""); - let imported_asset = match get_file_type(extension) { + // SVG imports as a new vector layer (not a placeable asset). + let imported_asset = if extension.eq_ignore_ascii_case("svg") { + self.import_svg_file(&path); + None + } else { match get_file_type(extension) { Some(FileType::Image) => { self.last_import_filter = ImportFilter::Images; self.import_image(&path) @@ -3255,11 +3260,12 @@ impl EditorApp { self.last_import_filter = ImportFilter::Midi; self.import_midi(&path) } + Some(FileType::Vector) => None, // handled by the svg intercept above None => { println!("Unsupported file type: {}", extension); None } - }; + } }; eprintln!("[TIMING] import took {:.1}ms", _import_timer.elapsed().as_secs_f64() * 1000.0); // Auto-place if this is "Import" (not "Import to Library") @@ -4512,6 +4518,53 @@ impl EditorApp { } /// Import an image file as an ImageAsset + /// Import an `.svg` file as a new vector layer (one static keyframe at the playhead). + fn import_svg_file(&mut self, path: &std::path::Path) { + let bytes = match std::fs::read(path) { + Ok(b) => b, + Err(e) => { + eprintln!("❌ Failed to read SVG {}: {}", path.display(), e); + return; + } + }; + + let graph = match svg_import::import_svg(&bytes) { + Ok(g) => g, + Err(e) => { + eprintln!("❌ {}", e); + return; + } + }; + + let name = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("SVG") + .to_string(); + + // Build a vector layer holding the imported graph as a keyframe at the current time. + let mut layer = lightningbeam_core::layer::VectorLayer::new(name); + let mut keyframe = lightningbeam_core::layer::ShapeKeyframe::new(self.playback_time); + keyframe.graph = graph; + layer.keyframes.push(keyframe); + + let editing_clip_id = self.editing_context.current_clip_id(); + let action = lightningbeam_core::actions::AddLayerAction::new( + lightningbeam_core::layer::AnyLayer::Vector(layer), + ) + .with_target_clip(editing_clip_id); + if let Err(e) = self.action_executor.execute(Box::new(action)) { + eprintln!("❌ Failed to add imported SVG layer: {}", e); + return; + } + + // Select the newly created layer. + let context_layers = self.action_executor.document().context_layers(editing_clip_id.as_ref()); + if let Some(last_layer) = context_layers.last() { + self.active_layer_id = Some(last_layer.id()); + } + self.last_import_filter = ImportFilter::Images; + } + fn import_image(&mut self, path: &std::path::Path) -> Option { use lightningbeam_core::clip::ImageAsset; self.note_possible_large_media(path); @@ -6132,6 +6185,19 @@ impl eframe::App for EditorApp { ); false // image export is silent (no progress dialog) } + ExportResult::Svg(time, output_path) => { + println!("🖋 [MAIN] Exporting SVG: {}", output_path.display()); + let svg = lightningbeam_core::svg_export::document_to_svg( + self.action_executor.document(), + time, + ); + if let Err(err) = std::fs::write(&output_path, svg) { + eprintln!("❌ Failed to write SVG: {}", err); + } else { + println!("✅ SVG written: {}", output_path.display()); + } + false // synchronous; no progress dialog + } ExportResult::AudioOnly(settings, output_path) => { println!("🎵 [MAIN] Starting audio-only export: {}", output_path.display()); diff --git a/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs b/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs new file mode 100644 index 0000000..c0cf4b1 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs @@ -0,0 +1,315 @@ +//! SVG import → `VectorGraph`. +//! +//! Parses an `.svg` with usvg (which resolves CSS, converts shapes/rects/circles to +//! paths, and computes absolute transforms), then bakes each path's absolute transform +//! into geometry and builds a single [`VectorGraph`] that becomes one new vector layer. +//! +//! Scope (matches the export pass): paths with solid/gradient fills and strokes. `` +//! and `` nodes are skipped, and nested groups are flattened (their transforms are +//! already baked into each path's `abs_transform`). +//! +//! Known limitation: imported edges are NOT intersection-split, so the paint-bucket tool +//! may need to re-process imported art. Display, transform, and round-trip are fine. + +use kurbo::{CubicBez, Point as KPoint}; +use lightningbeam_core::gradient::{GradientExtend, GradientStop, GradientType, ShapeGradient}; +use lightningbeam_core::shape::{Cap, FillRule, Join, ShapeColor, StrokeStyle}; +use lightningbeam_core::vector_graph::{Direction, EdgeId, VectorGraph, VertexId}; +use resvg::usvg; +use usvg::tiny_skia_path::{PathSegment, Point as SkPoint}; + +/// Parse SVG bytes into a single flattened [`VectorGraph`] in document (canvas) space. +pub fn import_svg(bytes: &[u8]) -> Result { + let tree = usvg::Tree::from_data(bytes, &usvg::Options::default()) + .map_err(|e| format!("Failed to parse SVG: {e}"))?; + let mut graph = VectorGraph::new(); + walk_group(tree.root(), &mut graph); + if graph.edges.is_empty() { + return Err("SVG contained no importable vector paths".to_string()); + } + Ok(graph) +} + +fn walk_group(group: &usvg::Group, graph: &mut VectorGraph) { + for node in group.children() { + match node { + usvg::Node::Group(g) => walk_group(g, graph), + usvg::Node::Path(p) => convert_path(p, graph), + usvg::Node::Image(_) | usvg::Node::Text(_) => {} // skipped this pass + } + } +} + +fn convert_path(path: &usvg::Path, graph: &mut VectorGraph) { + if !path.is_visible() { + return; + } + let ts = path.abs_transform(); + // Bake the absolute transform into the geometry so everything lives in canvas space. + let Some(data) = path.data().clone().transform(ts) else { + return; + }; + + // One stroke style/colour shared by every edge of this path. + let stroke = path.stroke().map(|s| stroke_to_style(s, ts)); + + // Walk the (transformed) segments, allocating vertices/edges and recording the + // boundary cycle. `EdgeId::NONE` separates subpaths (outer contour + holes). + let mut boundary: Vec<(EdgeId, Direction)> = Vec::new(); + let mut have_subpath = false; + let mut cur_v = VertexId(0); + let mut cur_p = SkPoint::from_xy(0.0, 0.0); + let mut start_v = VertexId(0); + let mut start_p = SkPoint::from_xy(0.0, 0.0); + + for seg in data.segments() { + match seg { + PathSegment::MoveTo(p) => { + if have_subpath { + boundary.push((EdgeId::NONE, Direction::Forward)); + } + let v = graph.alloc_vertex(kp(p)); + cur_v = v; + cur_p = p; + start_v = v; + start_p = p; + have_subpath = true; + } + PathSegment::LineTo(p) => { + let (c1, c2) = line_ctrls(cur_p, p); + cur_v = add_edge(graph, &mut boundary, cur_v, cur_p, c1, c2, p, &stroke); + cur_p = p; + } + PathSegment::QuadTo(c, p) => { + let (c1, c2) = quad_to_cubic(cur_p, c, p); + cur_v = add_edge(graph, &mut boundary, cur_v, cur_p, c1, c2, p, &stroke); + cur_p = p; + } + PathSegment::CubicTo(c1, c2, p) => { + cur_v = add_edge(graph, &mut boundary, cur_v, cur_p, c1, c2, p, &stroke); + cur_p = p; + } + PathSegment::Close => { + // Close back to the subpath start (reusing its vertex) unless already there. + if cur_p != start_p { + let (c1, c2) = line_ctrls(cur_p, start_p); + let curve = CubicBez::new(kp(cur_p), kp(c1), kp(c2), kp(start_p)); + let (style, color) = split_stroke(&stroke); + let e = graph.alloc_edge(curve, cur_v, start_v, style, color); + boundary.push((e, Direction::Forward)); + } + cur_v = start_v; + cur_p = start_p; + } + } + } + + // Fill (if any) references the whole boundary cycle. + if let Some(fill) = path.fill() { + if !boundary.is_empty() { + let rule = match fill.rule() { + usvg::FillRule::NonZero => FillRule::NonZero, + usvg::FillRule::EvenOdd => FillRule::EvenOdd, + }; + let fid = graph.alloc_fill(boundary, None, rule); + let slot = &mut graph.fills[fid.idx()]; + match fill.paint() { + usvg::Paint::Color(c) => { + slot.color = Some(ShapeColor::rgba(c.red, c.green, c.blue, opacity_u8(fill.opacity()))); + } + usvg::Paint::LinearGradient(g) => { + slot.gradient_fill = Some(linear_gradient(g, ts)); + } + usvg::Paint::RadialGradient(g) => { + slot.gradient_fill = Some(radial_gradient(g, ts)); + } + usvg::Paint::Pattern(_) => { + // Patterns aren't representable yet — neutral gray so the shape stays visible. + slot.color = Some(ShapeColor::rgba(128, 128, 128, opacity_u8(fill.opacity()))); + } + } + } + } +} + +/// Allocate the end vertex + a cubic edge from `av`/`ap` to `bp`, recording it on the boundary. +fn add_edge( + graph: &mut VectorGraph, + boundary: &mut Vec<(EdgeId, Direction)>, + av: VertexId, + ap: SkPoint, + c1: SkPoint, + c2: SkPoint, + bp: SkPoint, + stroke: &Option<(StrokeStyle, ShapeColor)>, +) -> VertexId { + let bv = graph.alloc_vertex(kp(bp)); + let curve = CubicBez::new(kp(ap), kp(c1), kp(c2), kp(bp)); + let (style, color) = split_stroke(stroke); + let e = graph.alloc_edge(curve, av, bv, style, color); + boundary.push((e, Direction::Forward)); + bv +} + +fn split_stroke(stroke: &Option<(StrokeStyle, ShapeColor)>) -> (Option, Option) { + match stroke { + Some((s, c)) => (Some(s.clone()), Some(*c)), + None => (None, None), + } +} + +fn stroke_to_style(s: &usvg::Stroke, ts: usvg::Transform) -> (StrokeStyle, ShapeColor) { + let scale = transform_scale(ts) as f64; + let style = StrokeStyle { + width: s.width().get() as f64 * scale, + cap: match s.linecap() { + usvg::LineCap::Butt => Cap::Butt, + usvg::LineCap::Round => Cap::Round, + usvg::LineCap::Square => Cap::Square, + }, + join: match s.linejoin() { + usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => Join::Miter, + usvg::LineJoin::Round => Join::Round, + usvg::LineJoin::Bevel => Join::Bevel, + }, + miter_limit: s.miterlimit().get() as f64, + }; + let color = match s.paint() { + usvg::Paint::Color(c) => ShapeColor::rgba(c.red, c.green, c.blue, opacity_u8(s.opacity())), + // Gradient/pattern strokes aren't representable per-edge — fall back to opaque black. + _ => ShapeColor::rgba(0, 0, 0, opacity_u8(s.opacity())), + }; + (style, color) +} + +/// Geometric-mean scale of the transform's linear part (for stroke-width baking). +fn transform_scale(ts: usvg::Transform) -> f32 { + (ts.sx * ts.sy - ts.kx * ts.ky).abs().sqrt() +} + +fn linear_gradient(g: &usvg::LinearGradient, abs: usvg::Transform) -> ShapeGradient { + let ct = abs.pre_concat(g.transform()); + let start = map_pt(ct, g.x1(), g.y1()); + let end = map_pt(ct, g.x2(), g.y2()); + let angle = (end.1 - start.1).atan2(end.0 - start.0).to_degrees() as f32; + ShapeGradient { + kind: GradientType::Linear, + stops: gradient_stops(g), + angle, + extend: spread(g), + start_world: Some(start), + end_world: Some(end), + } +} + +fn radial_gradient(g: &usvg::RadialGradient, abs: usvg::Transform) -> ShapeGradient { + let ct = abs.pre_concat(g.transform()); + // Our model stores center as start_world and a rim point (defining the radius) as end_world. + let center = map_pt(ct, g.cx(), g.cy()); + let rim = map_pt(ct, g.cx() + g.r().get(), g.cy()); + ShapeGradient { + kind: GradientType::Radial, + stops: gradient_stops(g), + angle: 0.0, + extend: spread(g), + start_world: Some(center), + end_world: Some(rim), + } +} + +fn gradient_stops(base: &usvg::BaseGradient) -> Vec { + base.stops() + .iter() + .map(|s| GradientStop { + position: s.offset().get(), + color: ShapeColor::rgba(s.color().red, s.color().green, s.color().blue, opacity_u8(s.opacity())), + }) + .collect() +} + +fn spread(base: &usvg::BaseGradient) -> GradientExtend { + match base.spread_method() { + usvg::SpreadMethod::Pad => GradientExtend::Pad, + usvg::SpreadMethod::Reflect => GradientExtend::Reflect, + usvg::SpreadMethod::Repeat => GradientExtend::Repeat, + } +} + +// ── small geometry helpers ────────────────────────────────────────────────── + +fn kp(p: SkPoint) -> KPoint { + KPoint::new(p.x as f64, p.y as f64) +} + +fn map_pt(ts: usvg::Transform, x: f32, y: f32) -> (f64, f64) { + let mut p = SkPoint::from_xy(x, y); + ts.map_point(&mut p); + (p.x as f64, p.y as f64) +} + +fn lerp(a: SkPoint, b: SkPoint, t: f32) -> SkPoint { + SkPoint::from_xy(a.x + (b.x - a.x) * t, a.y + (b.y - a.y) * t) +} + +/// Degenerate cubic control points for a straight segment (matches our edge model). +fn line_ctrls(a: SkPoint, b: SkPoint) -> (SkPoint, SkPoint) { + (lerp(a, b, 1.0 / 3.0), lerp(a, b, 2.0 / 3.0)) +} + +/// Elevate a quadratic Bézier to a cubic. +fn quad_to_cubic(a: SkPoint, c: SkPoint, b: SkPoint) -> (SkPoint, SkPoint) { + let c1 = SkPoint::from_xy(a.x + 2.0 / 3.0 * (c.x - a.x), a.y + 2.0 / 3.0 * (c.y - a.y)); + let c2 = SkPoint::from_xy(b.x + 2.0 / 3.0 * (c.x - b.x), b.y + 2.0 / 3.0 * (c.y - b.y)); + (c1, c2) +} + +fn opacity_u8(o: usvg::Opacity) -> u8 { + (o.get() * 255.0).round().clamp(0.0, 255.0) as u8 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn imports_solid_rect_fill() { + let svg = br##""##; + let g = import_svg(svg).expect("import"); + assert!(!g.edges.is_empty(), "expected edges from the rect"); + let fills: Vec<_> = g.fills.iter().filter(|f| !f.deleted).collect(); + assert_eq!(fills.len(), 1, "one fill expected"); + let c = fills[0].color.expect("solid color"); + assert_eq!((c.r, c.g, c.b), (255, 0, 0), "red fill"); + } + + #[test] + fn imports_stroke_only() { + let svg = br##""##; + let g = import_svg(svg).expect("import"); + let stroked = g.edges.iter().filter(|e| !e.deleted && e.stroke_color.is_some()).count(); + assert!(stroked >= 1, "expected at least one stroked edge"); + let c = g.edges.iter().find_map(|e| e.stroke_color).unwrap(); + assert_eq!((c.r, c.g, c.b), (0, 255, 0), "green stroke"); + } + + #[test] + fn imports_linear_gradient() { + let svg = br##" + + + + "##; + let g = import_svg(svg).expect("import"); + let fills: Vec<_> = g.fills.iter().filter(|f| !f.deleted).collect(); + assert_eq!(fills.len(), 1); + let grad = fills[0].gradient_fill.as_ref().expect("gradient"); + assert_eq!(grad.stops.len(), 2); + assert!(grad.start_world.is_some() && grad.end_world.is_some()); + } + + #[test] + fn empty_svg_errors() { + let svg = br#""#; + assert!(import_svg(svg).is_err()); + } +} From b301c63387e35e5f9bb0ccf6fb7323fc10a14498 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 17:47:32 -0400 Subject: [PATCH 29/31] Address code-review findings across export, decode, and data model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Export correctness: - Honor the user's color-range (Limited/Full) on the software encode path: thread full_range through gpu_yuv (new shader range uniform), the CPU swscale fallback (sws_setColorspaceDetails → BT.709 + range, fixing the BT.601 hue/level shift on odd-width exports), and the encoder color tags. - Reject HDR + WebM up front with a clear message and log the forced HEVC override instead of producing an unplayable file. - Delete dead render_frame_to_rgba_hdr (hardcoded Stretch; live HDR path already honors the fit mode). Decode/playback (video.rs): - Drain the decoder at EOF (send_eof + flush) so the final B-frame-delayed frames render instead of erroring; per-frame logic extracted to a helper. - Missing-PTS frames continue monotonically rather than snapping to ts=0. - Force exact thumbnail width so sub-128px sources aren't shown stretched. Resource leaks (gpu-video-encoder): - dmabuf import_raw: RAII guard frees the duped fd + partial VkImages/memory on every error path. - vaapi alloc: free device/frames-ctx/AVFrames on the unexpected-DRM path. Data model / robustness: - collapse_boundary_spikes requires a full curve reversal (all control points) so it no longer deletes a real lens/sliver and drops the fill. - Export audio spin-wait ignores a stale `finished` flag when a forward seek is pending (was rendering silence over real audio). - RasterDiff apply_before/after take current dims and skip on a post-resize mismatch. - beam_archive read_media_full caps the preallocation from untrusted total_len. UI/visual: - SVG export skips hidden layers/empty groups; import folds fill-opacity into gradients and surfaces failures as a notification. - Active raster-layer border uses playback_time + overlay_transform. - gpu_brush remove_layer_texture also evicts the stale low-res proxy. - ensure_raster_resident_for_undo registers faulted frames in the LRU so resident RAM stays bounded. Co-Authored-By: Claude Opus 4.8 --- daw-backend/src/audio/pool.rs | 15 +- .../gpu-video-encoder/src/dmabuf.rs | 58 +++ .../gpu-video-encoder/src/vaapi.rs | 12 +- .../src/actions/raster_diff.rs | 32 +- .../src/actions/raster_fill.rs | 4 +- .../src/actions/raster_stroke.rs | 4 +- .../lightningbeam-core/src/beam_archive.rs | 6 +- .../lightningbeam-core/src/svg_export.rs | 22 +- .../src/vector_graph/mod.rs | 30 +- .../lightningbeam-core/src/video.rs | 356 +++++++++++------- .../src/export/cpu_yuv_converter.rs | 27 +- .../src/export/gpu_yuv.rs | 76 +++- .../lightningbeam-editor/src/export/mod.rs | 25 +- .../src/export/readback_pipeline.rs | 4 +- .../src/export/video_exporter.rs | 233 +----------- .../lightningbeam-editor/src/gpu_brush.rs | 16 +- .../lightningbeam-editor/src/main.rs | 11 +- .../lightningbeam-editor/src/notifications.rs | 7 +- .../lightningbeam-editor/src/panes/stage.rs | 10 +- .../lightningbeam-editor/src/svg_import.rs | 19 +- 20 files changed, 527 insertions(+), 440 deletions(-) diff --git a/daw-backend/src/audio/pool.rs b/daw-backend/src/audio/pool.rs index cbe6a51..c130dd3 100644 --- a/daw-backend/src/audio/pool.rs +++ b/daw-backend/src/audio/pool.rs @@ -582,8 +582,21 @@ impl AudioClipPool { // or a container without exact stream duration). They will never // arrive — stop waiting and let the render fill silence, instead of // burning the 10s safety valve on every remaining chunk. + // + // BUT `finished` may be stale: we just moved the target with + // `set_target_frame`, and on a forward jump past the buffer the reader + // will reset()+seek() (clearing `finished`) on its next poll. The reader + // re-seeks when `target < buf_start || target > buf_end + sample_rate` + // (see disk_reader.rs), so only trust EOF when the target is inside the + // current window — otherwise a stale flag from the previous region would + // make us render silence over audio that's about to be decoded. if ra.is_finished() { - break; + let (buf_start, buf_end) = ra.snapshot(); + let sr = audio_file.sample_rate as u64; + let seek_pending = src_start < buf_start || src_start > buf_end + sr; + if !seek_pending { + break; + } } std::thread::sleep(std::time::Duration::from_micros(100)); wait_iters += 1; diff --git a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs index 4922f16..8b536ff 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs @@ -36,6 +36,47 @@ impl Drop for MemoryGuard { } } +/// Frees the duplicated dma-buf fd and any partially-created Vulkan objects when an import +/// errors out before ownership has been handed to wgpu/`MemoryGuard`. `commit()` disarms it on +/// the success path; `fd_consumed()` is called once `vkAllocateMemory` has taken the fd. +struct ImportGuard { + device: ash::Device, + fd: libc::c_int, + img_y: vk::Image, + img_uv: vk::Image, + memory: vk::DeviceMemory, + armed: bool, +} +impl ImportGuard { + fn fd_consumed(&mut self) { + self.fd = -1; // vkAllocateMemory owns the fd now; don't close it ourselves + } + fn commit(&mut self) { + self.armed = false; + } +} +impl Drop for ImportGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + unsafe { + if self.img_uv != vk::Image::null() { + self.device.destroy_image(self.img_uv, None); + } + if self.img_y != vk::Image::null() { + self.device.destroy_image(self.img_y, None); + } + if self.memory != vk::DeviceMemory::null() { + self.device.free_memory(self.memory, None); + } + if self.fd >= 0 { + libc::close(self.fd); + } + } + } +} + /// A VAAPI surface imported as two wgpu plane textures. The underlying Vulkan image/ /// memory are destroyed by wgpu (via drop callbacks) when these textures drop. pub struct ImportedNv12 { @@ -103,6 +144,16 @@ pub fn import_raw( if dup_fd < 0 { return Err("dup(dma-buf fd) failed".into()); } + // Owns the fd + any Vk objects created below until ownership transfers to wgpu; on any + // early `?`/return before that, its Drop frees them (was leaking on every failed import). + let mut guard = ImportGuard { + device: raw_device.clone(), + fd: dup_fd, + img_y: vk::Image::null(), + img_uv: vk::Image::null(), + memory: vk::DeviceMemory::null(), + armed: true, + }; // 16-bit-norm plane formats (P010) are NOT renderable, so the import is sample-only for // those (decode path). 8-bit planes keep COLOR_ATTACHMENT for the encoder's RGBA→NV12 write. @@ -155,7 +206,9 @@ pub fn import_raw( }; let img_y = make_image(vk_y, buf.width, buf.height, buf.y_pitch)?; + guard.img_y = img_y; let img_uv = make_image(vk_uv, buf.width / 2, buf.height / 2, buf.uv_pitch)?; + guard.img_uv = img_uv; let fd_dev = ash::khr::external_memory_fd::Device::new(instance, &raw_device); let mut fd_props = vk::MemoryFdPropertiesKHR::default(); @@ -180,6 +233,8 @@ pub fn import_raw( let memory = raw_device .allocate_memory(&alloc, None) .map_err(|e| format!("vkAllocateMemory(import dma-buf) failed: {e:?}"))?; + guard.fd_consumed(); // the import transferred fd ownership to Vulkan + guard.memory = memory; raw_device .bind_image_memory(img_y, memory, buf.y_offset) @@ -242,6 +297,9 @@ pub fn import_raw( }, ) }; + // Ownership of the images (→ texture drop callbacks) and memory (→ MemoryGuard) has now + // transferred to wgpu; disarm the cleanup guard so it doesn't double-free them. + guard.commit(); let y = wrap(img_y, wgpu_y, buf.width, buf.height); let uv = wrap(img_uv, wgpu_uv, buf.width / 2, buf.height / 2); drop(hal_device); diff --git a/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs b/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs index f7502e0..5eb0da2 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs @@ -157,10 +157,18 @@ impl MappedSurface { let desc = (*drm).data[0] as *const ff::AVDRMFrameDescriptor; // Expect 1 object, 2 layers (Y=R8, UV=GR88). if (*desc).nb_objects != 1 || (*desc).nb_layers != 2 { - return Err(format!( + let msg = format!( "unexpected DRM layout: {} objects, {} layers", (*desc).nb_objects, (*desc).nb_layers - )); + ); + // Free everything mapped/allocated above (this path was leaking the device, + // frames context, and both AVFrames on every odd-layout surface). + ff::av_frame_free(&mut (drm as *mut _)); + ff::av_frame_free(&mut (surf as *mut _)); + let mut fr = frames_ref; + ff::av_buffer_unref(&mut fr); + ff::av_buffer_unref(&mut hw_device); + return Err(msg); } let obj = &(*desc).objects[0]; let y = &(*desc).layers[0].planes[0]; diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs b/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs index 15a5f16..1911750 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs @@ -98,11 +98,16 @@ impl RasterDiff { self.before_region.len() + self.after_region.len() } - /// Restore the pre-edit pixels into `raw` (undo / first-execute rollback). - pub fn apply_before(&self, raw: &mut Vec) { + /// Restore the pre-edit pixels into `raw` (undo / first-execute rollback). `cur_w`/`cur_h` + /// are the keyframe's current dimensions; if they differ from the diff's captured size the + /// diff predates a resize and is skipped rather than applied at mismatched dims. + pub fn apply_before(&self, raw: &mut Vec, cur_w: u32, cur_h: u32) { if self.bbox.is_none() { return; // no change } + if cur_w != self.full_width || cur_h != self.full_height { + return; // diff predates a keyframe resize + } if self.before_blank { // The frame was blank before this edit (it was the first stroke); undoing // it returns to blank regardless of the current buffer. @@ -112,11 +117,16 @@ impl RasterDiff { self.stamp_resident(&self.before_region, raw); } - /// Apply the post-edit pixels into `raw` (commit / redo). - pub fn apply_after(&self, raw: &mut Vec) { + /// Apply the post-edit pixels into `raw` (commit / redo). `cur_w`/`cur_h` are the keyframe's + /// current dimensions; a mismatch with the captured size means the diff predates a resize, so + /// it's skipped rather than rebuilding the buffer at stale dimensions. + pub fn apply_after(&self, raw: &mut Vec, cur_w: u32, cur_h: u32) { if self.bbox.is_none() { return; // no change } + if cur_w != self.full_width || cur_h != self.full_height { + return; // diff predates a keyframe resize + } if self.before_blank { // Base was blank: build a full transparent buffer then stamp the bbox. The // commit/redo path frequently starts from empty `raw_pixels` here. @@ -175,9 +185,9 @@ mod tests { assert_eq!(diff.bbox, Some((3, 2, 2, 2))); let mut buf = after.clone(); - diff.apply_before(&mut buf); + diff.apply_before(&mut buf, w, h); assert_eq!(buf, before, "undo must reproduce the pre-edit buffer exactly"); - diff.apply_after(&mut buf); + diff.apply_after(&mut buf, w, h); assert_eq!(buf, after, "redo must reproduce the post-edit buffer exactly"); } @@ -195,15 +205,15 @@ mod tests { // First execute / redo from EMPTY raw_pixels (the real commit path): builds // the full buffer from transparent + the stroke. let mut buf: Vec = Vec::new(); - diff.apply_after(&mut buf); + diff.apply_after(&mut buf, w, h); assert_eq!(buf, after, "commit/redo must build the frame from a blank base"); // Undo the first stroke → back to blank (empty). - diff.apply_before(&mut buf); + diff.apply_before(&mut buf, w, h); assert!(buf.is_empty(), "undoing the first stroke restores the blank keyframe"); // Redo again from the now-empty buffer. - diff.apply_after(&mut buf); + diff.apply_after(&mut buf, w, h); assert_eq!(buf, after); } @@ -215,7 +225,7 @@ mod tests { assert_eq!(diff.bbox, None); assert_eq!(diff.byte_size(), 0); let mut b = buf.clone(); - diff.apply_before(&mut b); + diff.apply_before(&mut b, w, h); assert_eq!(b, buf); } @@ -226,7 +236,7 @@ mod tests { let after = solid(w, h, [1, 2, 3, 255]); let diff = RasterDiff::compute(&before, &after, w, h); let mut empty: Vec = Vec::new(); - diff.apply_before(&mut empty); // base not resident + diff.apply_before(&mut empty, w, h); // base not resident assert!(empty.is_empty(), "must not resize/corrupt a non-resident base"); } } diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs b/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs index 92c0dd0..3693d0d 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs @@ -53,7 +53,7 @@ impl Action for RasterFillAction { if let Some(full) = self.full_after.take() { kf.raw_pixels = full; } else { - self.diff.apply_after(&mut kf.raw_pixels); + self.diff.apply_after(&mut kf.raw_pixels, kf.width, kf.height); } kf.texture_dirty = true; kf.dirty = true; @@ -71,7 +71,7 @@ impl Action for RasterFillAction { let kf = raster .keyframe_at_mut(self.time) .ok_or_else(|| format!("No raster keyframe at/before t={}", self.time))?; - self.diff.apply_before(&mut kf.raw_pixels); + self.diff.apply_before(&mut kf.raw_pixels, kf.width, kf.height); kf.texture_dirty = true; kf.dirty = true; Ok(()) diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs b/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs index 06517b9..392fcb0 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs @@ -61,7 +61,7 @@ impl Action for RasterStrokeAction { kf.raw_pixels = full; } else { // Redo: replay via the diff onto the (resident) base. - self.diff.apply_after(&mut kf.raw_pixels); + self.diff.apply_after(&mut kf.raw_pixels, kf.width, kf.height); } kf.texture_dirty = true; kf.dirty = true; @@ -70,7 +70,7 @@ impl Action for RasterStrokeAction { fn rollback(&mut self, document: &mut Document) -> Result<(), String> { let kf = get_keyframe_mut(document, &self.layer_id, self.time, self.width, self.height)?; - self.diff.apply_before(&mut kf.raw_pixels); + self.diff.apply_before(&mut kf.raw_pixels, kf.width, kf.height); kf.texture_dirty = true; kf.dirty = true; Ok(()) diff --git a/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs index 39ef088..770b544 100644 --- a/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs +++ b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs @@ -374,7 +374,11 @@ impl BeamArchive { let rows = stmt .query_map([&id_bytes], |r| r.get::<_, Vec>(0)) .map_err(map_sql)?; - let mut out = Vec::with_capacity(info.total_len as usize); + // `total_len` is an untrusted DB field; cap the preallocation so a corrupt/oversized + // value can't trigger a multi-GB eager allocation (or, on 32-bit, a truncated `as usize`). + // The Vec still grows to fit the actual chunk bytes regardless. + const PREALLOC_CAP: u64 = 64 * 1024 * 1024; + let mut out = Vec::with_capacity(info.total_len.min(PREALLOC_CAP) as usize); for row in rows { out.extend_from_slice(&row.map_err(map_sql)?); } diff --git a/lightningbeam-ui/lightningbeam-core/src/svg_export.rs b/lightningbeam-ui/lightningbeam-core/src/svg_export.rs index ad36f80..008bc4e 100644 --- a/lightningbeam-ui/lightningbeam-core/src/svg_export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/svg_export.rs @@ -266,6 +266,9 @@ pub fn document_to_svg(document: &Document, time: f64) -> String { fn layer_to_svg(layer: &AnyLayer, time: f64, parent_opacity: f64, body: &mut String, defs: &mut String, grad_n: &mut usize) { match layer { AnyLayer::Vector(vl) => { + if !vl.layer.visible { + return; // hidden layers are not rendered, so don't export them + } let opacity = parent_opacity * vl.layer.opacity; if let Some(graph) = vl.tweened_graph_at(time) { let wrap = opacity < 0.999; @@ -281,12 +284,21 @@ fn layer_to_svg(layer: &AnyLayer, time: f64, parent_opacity: f64, body: &mut Str // exported — a refinement once loose-geometry export is verified. } AnyLayer::Group(g) => { - let opacity = parent_opacity * g.layer.opacity; - body.push_str(&format!(r#""#)); - for child in &g.children { - layer_to_svg(child, time, 1.0, body, defs, grad_n); + if !g.layer.visible { + return; + } + // Render children first; only emit the wrapper if it has exportable content + // (avoids empty groups when every child is a non-vector/hidden layer). + let mut inner = String::new(); + for child in &g.children { + layer_to_svg(child, time, 1.0, &mut inner, defs, grad_n); + } + if !inner.is_empty() { + let opacity = parent_opacity * g.layer.opacity; + body.push_str(&format!(r#""#)); + body.push_str(&inner); + body.push_str(""); } - body.push_str(""); } // Raster/Video/Audio/Effect have no lossless vector representation — skipped this pass. _ => {} diff --git a/lightningbeam-ui/lightningbeam-core/src/vector_graph/mod.rs b/lightningbeam-ui/lightningbeam-core/src/vector_graph/mod.rs index e5744a6..646fe31 100644 --- a/lightningbeam-ui/lightningbeam-core/src/vector_graph/mod.rs +++ b/lightningbeam-ui/lightningbeam-core/src/vector_graph/mod.rs @@ -1302,21 +1302,26 @@ impl VectorGraph { /// bouncing across near-coincident duplicate edges). These are zero-area and would make /// `boundary_to_bezpath` render a stray hair; collapsing them yields a simple loop. fn collapse_boundary_spikes(&self, face: &mut Vec<(EdgeId, Direction)>) { - let dstart = |entry: &(EdgeId, Direction)| -> Point { + // The four control points of an entry's curve in its traversal order. + let traversed = |entry: &(EdgeId, Direction)| -> [Point; 4] { let c = self.edges[entry.0.idx()].curve; match entry.1 { - Direction::Forward => c.p0, - Direction::Backward => c.p3, - } - }; - let dend = |entry: &(EdgeId, Direction)| -> Point { - let c = self.edges[entry.0.idx()].curve; - match entry.1 { - Direction::Forward => c.p3, - Direction::Backward => c.p0, + Direction::Forward => [c.p0, c.p1, c.p2, c.p3], + Direction::Backward => [c.p3, c.p2, c.p1, c.p0], } }; const EPS: f64 = 0.5; + // Entries i and j cancel only when j is the *exact reverse* of i — every control point of + // j matches the mirror of i. Testing endpoints alone would also collapse a genuine + // lens/sliver (two distinct edges that merely share near-coincident endpoints), silently + // deleting real boundary geometry and dropping the fill. + let reverses = |a: &(EdgeId, Direction), b: &(EdgeId, Direction)| -> bool { + let (ca, cb) = (traversed(a), traversed(b)); + (0..4).all(|k| { + let (p, q) = (ca[k], cb[3 - k]); + (p.x - q.x).hypot(p.y - q.y) < EPS + }) + }; loop { let n = face.len(); if n < 2 { @@ -1325,10 +1330,7 @@ impl VectorGraph { let mut collapsed = false; for i in 0..n { let j = (i + 1) % n; - // entries i and j cancel when j ends back at i's start. - let si = dstart(&face[i]); - let ej = dend(&face[j]); - if (si.x - ej.x).hypot(si.y - ej.y) < EPS { + if reverses(&face[i], &face[j]) { let (hi, lo) = if i > j { (i, j) } else { (j, i) }; face.remove(hi); face.remove(lo); diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 9b57350..1f631f9 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -120,6 +120,153 @@ enum DecodedFrame { Gpu(GpuVideoFrame), } +/// Result of processing one decoded frame against the requested target timestamp. +enum FrameOutcome { + /// Frame consumed; keep decoding. + Continue, + /// `current_frame_ts >= frame_ts` — caller should return the best frame found. + ReachedTarget, + /// Hardware surface import failed; caller should fall back to software. + HwImportFailed, +} + +/// Process one decoded `frame`: update the best-so-far (`best_*`) toward `frame_ts`, importing a +/// GPU surface (`gpu_out`) or swscaling to RGBA otherwise. Pulled out of the decode loop so the +/// same logic runs both for packet-fed frames and for frames flushed out of the codec at EOF. +#[allow(clippy::too_many_arguments)] +fn process_decoded_video_frame( + frame: &mut ffmpeg::util::frame::Video, + decoder: &ffmpeg::decoder::Video, + gpu_out: bool, + hw: bool, + out_w: u32, + out_h: u32, + frame_ts: i64, + importer: Option<&Arc>, + scaler: &mut Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>, + best_frame_data: &mut Option>, + best_gpu: &mut Option, + best_frame_ts: &mut Option, + last_decoded_ts: &mut i64, + scale_time_ms: &mut u128, +) -> Result { + use std::time::Instant; + + // A frame with no PTS continues monotonically from the last decoded position rather than + // snapping to 0 — treating "no timestamp" as ts=0 would look like a huge backward jump and + // corrupt the best-frame / seek logic on streams with missing PTS. + let current_frame_ts = frame.timestamp().unwrap_or(*last_decoded_ts + 1); + *last_decoded_ts = current_frame_ts; + + let is_better = match *best_frame_ts { + None => true, + Some(best_ts) => (current_frame_ts - frame_ts).abs() < (best_ts - frame_ts).abs(), + }; + + if is_better { + if gpu_out { + // VAAPI hw frames often don't carry the stream's colour tags, so the importer would + // mis-detect transfer/gamut. Copy the authoritative values from the codec context + // (parsed from the bitstream) onto the frame when it left them unspecified. + unsafe { + use ffmpeg::ffi::*; + let fp = frame.as_mut_ptr(); + let cp = decoder.as_ptr(); + if (*fp).color_trc == AVColorTransferCharacteristic::AVCOL_TRC_UNSPECIFIED { + (*fp).color_trc = (*cp).color_trc; + } + if (*fp).color_primaries == AVColorPrimaries::AVCOL_PRI_UNSPECIFIED { + (*fp).color_primaries = (*cp).color_primaries; + } + if (*fp).colorspace == AVColorSpace::AVCOL_SPC_UNSPECIFIED { + (*fp).colorspace = (*cp).colorspace; + } + if (*fp).color_range == AVColorRange::AVCOL_RANGE_UNSPECIFIED { + (*fp).color_range = (*cp).color_range; + } + } + let importer = importer.unwrap(); + match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } { + Some(gpu) => { + *best_gpu = Some(gpu); + *best_frame_ts = Some(current_frame_ts); + } + None => return Ok(FrameOutcome::HwImportFailed), + } + } else { + let t_scale_start = Instant::now(); + + // A hardware decoder produces VAAPI surfaces; a CPU consumer (export) downloads to + // system memory first, then swscales like the software path. + let downloaded; + let src: &ffmpeg::util::frame::Video = if hw { + let mut dl = ffmpeg::util::frame::Video::empty(); + let r = unsafe { + ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0) + }; + if r < 0 { + return Err(format!("av_hwframe_transfer_data failed: {r}")); + } + downloaded = dl; + &downloaded + } else { + &*frame + }; + + // Reuse the RGBA scaler across frames; rebuild only if the input format/size or the + // requested output size changes. + let need_new = match scaler { + Some((fmt, w, h, ow, oh, _)) => { + *fmt != src.format() || *w != src.width() || *h != src.height() + || *ow != out_w || *oh != out_h + } + None => true, + }; + if need_new { + let ctx = ffmpeg::software::scaling::context::Context::get( + src.format(), + src.width(), + src.height(), + ffmpeg::format::Pixel::RGBA, + out_w, + out_h, + ffmpeg::software::scaling::flag::Flags::BILINEAR, + ).map_err(|e| e.to_string())?; + *scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx))); + } + let scaler = &mut scaler.as_mut().unwrap().5.0; + + let mut rgb_frame = ffmpeg::util::frame::Video::empty(); + scaler.run(src, &mut rgb_frame) + .map_err(|e| e.to_string())?; + + // Remove stride padding to create tightly packed RGBA data + let width = out_w as usize; + let height = out_h as usize; + let stride = rgb_frame.stride(0); + let row_size = width * 4; // RGBA = 4 bytes per pixel + let source_data = rgb_frame.data(0); + + let mut packed_data = Vec::with_capacity(row_size * height); + for y in 0..height { + let row_start = y * stride; + let row_end = row_start + row_size; + packed_data.extend_from_slice(&source_data[row_start..row_end]); + } + + *scale_time_ms += t_scale_start.elapsed().as_millis(); + *best_frame_data = Some(packed_data); + *best_frame_ts = Some(current_frame_ts); + } + } + + if current_frame_ts >= frame_ts { + Ok(FrameOutcome::ReachedTarget) + } else { + Ok(FrameOutcome::Continue) + } +} + /// `get_format` callback for hardware decode: select VAAPI surfaces. With `hw_device_ctx` set, /// FFmpeg auto-allocates the frames context. unsafe extern "C" fn get_vaapi_format( @@ -459,159 +606,73 @@ impl VideoDecoder { let mut scale_time_ms = 0u128; let mut hw_import_failed = false; - 'decode: for (stream, packet) in input.packets() { - if stream.index() == self.stream_index { - decoder.send_packet(&packet) - .map_err(|e| e.to_string())?; + 'decode: { + for (stream, packet) in input.packets() { + if stream.index() == self.stream_index { + decoder.send_packet(&packet) + .map_err(|e| e.to_string())?; + let mut frame = ffmpeg::util::frame::Video::empty(); + while decoder.receive_frame(&mut frame).is_ok() { + decode_count += 1; + match process_decoded_video_frame( + &mut frame, decoder, gpu_out, hw, out_w, out_h, frame_ts, + self.importer.as_ref(), &mut self.scaler, + &mut best_frame_data, &mut best_gpu, &mut best_frame_ts, + &mut self.last_decoded_ts, &mut scale_time_ms, + )? { + FrameOutcome::Continue => {} + FrameOutcome::ReachedTarget => break 'decode, + FrameOutcome::HwImportFailed => { + self.hw_failed = true; + hw_import_failed = true; + break 'decode; + } + } + } + } + } + + // Flush: the codec may still hold buffered frames (B-frame reorder delay) past the + // last packet. Drain them so requesting the final frame(s) of a clip — scrubbing to + // the end or exporting the tail — doesn't fail with "Failed to decode frame". + if !hw_import_failed { + let _ = decoder.send_eof(); let mut frame = ffmpeg::util::frame::Video::empty(); while decoder.receive_frame(&mut frame).is_ok() { decode_count += 1; - let current_frame_ts = frame.timestamp().unwrap_or(0); - self.last_decoded_ts = current_frame_ts; // Update last decoded position - - // Check if this frame is closer to our target than the previous best - let is_better = match best_frame_ts { - None => true, - Some(best_ts) => { - (current_frame_ts - frame_ts).abs() < (best_ts - frame_ts).abs() + match process_decoded_video_frame( + &mut frame, decoder, gpu_out, hw, out_w, out_h, frame_ts, + self.importer.as_ref(), &mut self.scaler, + &mut best_frame_data, &mut best_gpu, &mut best_frame_ts, + &mut self.last_decoded_ts, &mut scale_time_ms, + )? { + FrameOutcome::Continue => {} + FrameOutcome::ReachedTarget => break 'decode, + FrameOutcome::HwImportFailed => { + self.hw_failed = true; + hw_import_failed = true; + break 'decode; } - }; - - if is_better { - if gpu_out { - // Hardware + GPU consumer: import the VAAPI surface as wgpu NV12 textures - // (no CPU copy). - // VAAPI hw frames often don't carry the stream's colour tags, so the - // importer (which only sees the frame) would mis-detect transfer/gamut. - // Copy the authoritative values from the codec context (parsed from the - // bitstream) onto the frame when it left them unspecified. - unsafe { - use ffmpeg::ffi::*; - let fp = frame.as_mut_ptr(); - let cp = decoder.as_ptr(); - if (*fp).color_trc == AVColorTransferCharacteristic::AVCOL_TRC_UNSPECIFIED { - (*fp).color_trc = (*cp).color_trc; - } - if (*fp).color_primaries == AVColorPrimaries::AVCOL_PRI_UNSPECIFIED { - (*fp).color_primaries = (*cp).color_primaries; - } - if (*fp).colorspace == AVColorSpace::AVCOL_SPC_UNSPECIFIED { - (*fp).colorspace = (*cp).colorspace; - } - if (*fp).color_range == AVColorRange::AVCOL_RANGE_UNSPECIFIED { - (*fp).color_range = (*cp).color_range; - } - } - let importer = self.importer.as_ref().unwrap(); - match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } { - Some(gpu) => { - best_gpu = Some(gpu); - best_frame_ts = Some(current_frame_ts); - } - None => { - // Import failed → fall back to software for this clip. - self.hw_failed = true; - hw_import_failed = true; - break 'decode; - } - } - } else { - let t_scale_start = Instant::now(); - - // A hardware decoder produces VAAPI surfaces; a CPU consumer (export) - // downloads to system memory first, then swscales like the software path. - let downloaded; - let src: &ffmpeg::util::frame::Video = if hw { - let mut dl = ffmpeg::util::frame::Video::empty(); - let r = unsafe { - ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0) - }; - if r < 0 { - return Err(format!("av_hwframe_transfer_data failed: {r}")); - } - downloaded = dl; - &downloaded - } else { - &frame - }; - - // Reuse the RGBA scaler across frames; rebuild only if the input - // format/size or the requested output size changes. - let need_new = match &self.scaler { - Some((fmt, w, h, ow, oh, _)) => { - *fmt != src.format() || *w != src.width() || *h != src.height() - || *ow != out_w || *oh != out_h - } - None => true, - }; - if need_new { - let ctx = ffmpeg::software::scaling::context::Context::get( - src.format(), - src.width(), - src.height(), - ffmpeg::format::Pixel::RGBA, - out_w, - out_h, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ).map_err(|e| e.to_string())?; - self.scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx))); - } - let scaler = &mut self.scaler.as_mut().unwrap().5.0; - - let mut rgb_frame = ffmpeg::util::frame::Video::empty(); - scaler.run(src, &mut rgb_frame) - .map_err(|e| e.to_string())?; - - // Remove stride padding to create tightly packed RGBA data - let width = out_w as usize; - let height = out_h as usize; - let stride = rgb_frame.stride(0); - let row_size = width * 4; // RGBA = 4 bytes per pixel - let source_data = rgb_frame.data(0); - - let mut packed_data = Vec::with_capacity(row_size * height); - for y in 0..height { - let row_start = y * stride; - let row_end = row_start + row_size; - packed_data.extend_from_slice(&source_data[row_start..row_end]); - } - - scale_time_ms += t_scale_start.elapsed().as_millis(); - best_frame_data = Some(packed_data); - best_frame_ts = Some(current_frame_ts); - } - } - - // If we've reached or passed the target timestamp, we can stop - if current_frame_ts >= frame_ts { - if video_debug() { - let total_time = t_start.elapsed().as_millis(); - let decode_time = t_decode_start.elapsed().as_millis(); - eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}", - timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" }); - } - if gpu_out { - if let Some(gpu) = best_gpu.take() { - return Ok(DecodedFrame::Gpu(gpu)); - } - } else if let Some(data) = best_frame_data { - self.frame_cache.put(cache_key, data.clone()); - return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h }); - } - break 'decode; } } } } - // Reached EOF without hitting the target, or HW import failed mid-stream. + if video_debug() { + let total_time = t_start.elapsed().as_millis(); + let decode_time = t_decode_start.elapsed().as_millis(); + eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}", + timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" }); + } + + // Reached the target, EOF, or HW import failed mid-stream. if hw_import_failed { self.decoder = None; // force a software rebuild next call (decoder borrow ended here) self.input = None; return Err("hardware frame import failed; retrying software".to_string()); } - // EOF: return the closest frame we found, if any. + // Return the closest frame we found, if any. if gpu_out { if let Some(gpu) = best_gpu.take() { return Ok(DecodedFrame::Gpu(gpu)); @@ -673,8 +734,23 @@ pub fn generate_keyframe_thumbnails( } // Decode at the thumbnail width (large height so width is the constraint), capped to native. // Thumbnail decoders are always software (no hardware importer). - if let Ok(DecodedFrame::Cpu { rgba, .. }) = decoder.get_frame(ks, thumb_width, 100_000, false) { - on_thumb(ks, Arc::new(rgba)); + if let Ok(DecodedFrame::Cpu { rgba, width, height }) = decoder.get_frame(ks, thumb_width, 100_000, false) { + // `capped_output` never upscales, so a source narrower than `thumb_width` decodes + // smaller — but `get_thumbnail_at` reconstructs height assuming an exact `thumb_width`. + // Force the exact width here (rare path) so that assumption holds and the thumbnail + // isn't shown stretched. + let data = if width == thumb_width || width == 0 || height == 0 { + rgba + } else { + let new_h = ((thumb_width as u64 * height as u64) / width as u64).max(1) as u32; + match image::RgbaImage::from_raw(width, height, rgba) { + Some(img) => image::imageops::resize( + &img, thumb_width, new_h, image::imageops::FilterType::Triangle, + ).into_raw(), + None => continue, + } + }; + on_thumb(ks, Arc::new(data)); } } Ok(()) diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs index 2ba087c..bfd4271 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs @@ -26,9 +26,9 @@ impl CpuYuvConverter { /// # Arguments /// * `width` - Frame width in pixels /// * `height` - Frame height in pixels - pub fn new(width: u32, height: u32) -> Result { + pub fn new(width: u32, height: u32, full_range: bool) -> Result { // BT.709 (HD) RGBA→YUV420p context, created once. - let scaler = ffmpeg::software::scaling::Context::get( + let mut scaler = ffmpeg::software::scaling::Context::get( ffmpeg::format::Pixel::RGBA, width, height, @@ -38,6 +38,23 @@ impl CpuYuvConverter { ffmpeg::software::scaling::Flags::BILINEAR, ) .map_err(|e| format!("Failed to create swscale context: {}", e))?; + + // swscale defaults to BT.601 + limited range; force BT.709 with the requested output + // range so this fallback matches the GPU path and the encoder's color tags + // (otherwise non-%8-width exports come out with shifted hue / wrong levels). There is + // no safe ffmpeg-next wrapper for sws_setColorspaceDetails, so this is the raw call. + unsafe { + let coeffs = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709 as i32); + let dst_range = if full_range { 1 } else { 0 }; + let one = 1 << 16; // 16.16 fixed-point 1.0 + ffmpeg::ffi::sws_setColorspaceDetails( + scaler.as_mut_ptr(), + coeffs, 1, // source table (RGB input is full-range) + coeffs, dst_range, // dest table = BT.709, dest range = requested + 0, one, one, // brightness, contrast, saturation (neutral) + ); + } + let rgba_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::RGBA, width, height); let yuv_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::YUV420P, width, height); Ok(Self { width, height, scaler, rgba_frame, yuv_frame }) @@ -90,13 +107,13 @@ mod tests { #[test] fn test_converter_creation() { - let converter = CpuYuvConverter::new(1920, 1080); + let converter = CpuYuvConverter::new(1920, 1080, true); assert!(converter.is_ok()); } #[test] fn test_conversion_output_sizes() { - let mut converter = CpuYuvConverter::new(1920, 1080).unwrap(); + let mut converter = CpuYuvConverter::new(1920, 1080, true).unwrap(); // Create dummy RGBA data (all black) let rgba_data = vec![0u8; 1920 * 1080 * 4]; @@ -117,7 +134,7 @@ mod tests { #[test] #[should_panic(expected = "RGBA data size mismatch")] fn test_wrong_input_size_panics() { - let mut converter = CpuYuvConverter::new(1920, 1080).unwrap(); + let mut converter = CpuYuvConverter::new(1920, 1080, true).unwrap(); // Wrong size input let rgba_data = vec![0u8; 1000]; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs b/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs index adcdb32..c5cc8a0 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs @@ -6,8 +6,8 @@ //! 8.3 MB RGBA) and — more importantly — the per-frame CPU `rgba_to_yuv420p` (swscale) //! is eliminated. //! -//! Color math is BT.709 **full-range** (JPEG range), matching the encoder color tags -//! set in `setup_video_encoder` (`Space::BT709` + `Range::JPEG`). +//! Color math is BT.709; the Y/chroma scale+offset (full vs limited range) is selected by +//! the `full_range` flag and must match the encoder color tags set in `setup_video_encoder`. //! //! Output buffer layout (tight, little-endian byte packing into `array`): //! - `[0, W*H)` Y plane, row stride `W` @@ -30,15 +30,36 @@ pub fn yuv420p_len(width: u32, height: u32) -> usize { y + 2 * c } +/// `(y_offset, y_scale, chroma_offset, chroma_scale)` as fractions of 255, selecting +/// limited (TV, 16–235 / 16–240) vs full (PC, 0–255) range. Mirrors `render_nv12`. +fn range_params(full_range: bool) -> [f32; 4] { + if full_range { + [0.0, 1.0, 128.0 / 255.0, 1.0] + } else { + [16.0 / 255.0, 219.0 / 255.0, 128.0 / 255.0, 224.0 / 255.0] + } +} + /// GPU compute pipeline: `Rgba8Unorm` texture → tight planar YUV420p storage buffer. pub struct GpuYuv { y_pipeline: wgpu::ComputePipeline, uv_pipeline: wgpu::ComputePipeline, bind_group_layout: wgpu::BindGroupLayout, + range_buffer: wgpu::Buffer, } impl GpuYuv { - pub fn new(device: &wgpu::Device) -> Self { + /// `full_range`: true → full/PC range (Y 0–255), false → limited/TV range (Y 16–235). + /// The encoder must tag the stream to match (`setup_video_encoder`'s `full_range`). + pub fn new(device: &wgpu::Device, full_range: bool) -> Self { + use wgpu::util::DeviceExt; + let params = range_params(full_range); + let range_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("gpu_yuv_range"), + contents: bytemuck::cast_slice(¶ms), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("gpu_yuv_bgl"), entries: &[ @@ -64,6 +85,17 @@ impl GpuYuv { }, count: None, }, + // 2: range params (y_offset, y_scale, chroma_offset, chroma_scale) + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); @@ -93,6 +125,7 @@ impl GpuYuv { y_pipeline: mk("y_main"), uv_pipeline: mk("uv_main"), bind_group_layout, + range_buffer, } } @@ -118,6 +151,7 @@ impl GpuYuv { entries: &[ wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(rgba_view) }, wgpu::BindGroupEntry { binding: 1, resource: yuv_buffer.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: self.range_buffer.as_entire_binding() }, ], }); @@ -142,12 +176,12 @@ impl GpuYuv { /// CPU reference for the exact math/layout the shader produces — used by unit tests so /// the packing and BT.709 coefficients stay verifiable without a GPU. -#[cfg(test)] -fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec { +fn cpu_reference(rgba: &[u8], width: u32, height: u32, full_range: bool) -> Vec { let w = width as usize; let h = height as usize; let cw = w / 2; let ch = h / 2; + let [yo, ys, co, cs] = range_params(full_range); let mut out = vec![0u8; yuv420p_len(width, height)]; let to_byte = |v: f32| (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8; let px = |x: usize, y: usize| { @@ -158,7 +192,8 @@ fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec { for y in 0..h { for x in 0..w { let p = px(x, y); - out[y * w + x] = to_byte(0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2]); + let yy = 0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2]; + out[y * w + x] = to_byte(yo + ys * yy); } } // U/V (2x2 average) @@ -172,19 +207,21 @@ fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec { acc[0] += p[0]; acc[1] += p[1]; acc[2] += p[2]; } let a = [acc[0] / 4.0, acc[1] / 4.0, acc[2] / 4.0]; - let u = -0.1146 * a[0] - 0.3854 * a[1] + 0.5000 * a[2] + 0.5; - let v = 0.5000 * a[0] - 0.4542 * a[1] - 0.0458 * a[2] + 0.5; - out[y_size + cy * cw + cx] = to_byte(u); - out[y_size + uv_size + cy * cw + cx] = to_byte(v); + let uc = -0.1146 * a[0] - 0.3854 * a[1] + 0.5000 * a[2]; + let vc = 0.5000 * a[0] - 0.4542 * a[1] - 0.0458 * a[2]; + out[y_size + cy * cw + cx] = to_byte(co + cs * uc); + out[y_size + uv_size + cy * cw + cx] = to_byte(co + cs * vc); } } out } const SHADER: &str = r#" -// RGBA -> tight planar YUV420p (BT.709 full-range), packed 4 bytes/u32. +// RGBA -> tight planar YUV420p (BT.709), packed 4 bytes/u32. +// rng = (y_offset, y_scale, chroma_offset, chroma_scale): selects limited vs full range. @group(0) @binding(0) var input_rgba: texture_2d; @group(0) @binding(1) var out_buf: array; +@group(0) @binding(2) var rng: vec4; fn to_byte(v: f32) -> u32 { return u32(clamp(v, 0.0, 1.0) * 255.0 + 0.5); } @@ -201,7 +238,7 @@ fn y_main(@builtin(global_invocation_id) gid: vec3) { for (var i = 0u; i < 4u; i = i + 1u) { let c = textureLoad(input_rgba, vec2(x4 + i, y), 0).rgb; let yy = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b; - packed = packed | (to_byte(yy) << (8u * i)); + packed = packed | (to_byte(rng.x + rng.y * yy) << (8u * i)); } out_buf[(y * w + x4) / 4u] = packed; } @@ -230,10 +267,11 @@ fn uv_main(@builtin(global_invocation_id) gid: vec3) { let p01 = textureLoad(input_rgba, vec2(sx, sy + 1u), 0).rgb; let p11 = textureLoad(input_rgba, vec2(sx + 1u, sy + 1u), 0).rgb; let a = (p00 + p10 + p01 + p11) * 0.25; - let u = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b + 0.5; - let v = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b + 0.5; - up = up | (to_byte(u) << (8u * i)); - vp = vp | (to_byte(v) << (8u * i)); + // Centered chroma in [-0.5, 0.5], then map to range via (offset + scale*coef). + let uc = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b; + let vc = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b; + up = up | (to_byte(rng.z + rng.w * uc) << (8u * i)); + vp = vp | (to_byte(rng.z + rng.w * vc) << (8u * i)); } out_buf[(y_size + cy * cw + cx4) / 4u] = up; out_buf[(y_size + uv_size + cy * cw + cx4) / 4u] = vp; @@ -264,14 +302,14 @@ mod tests { fn reference_known_colors() { // 8x2 solid white → Y≈255, U≈V≈128. Solid black → Y=0, U=V≈128. let white = vec![255u8; 8 * 2 * 4]; - let out = cpu_reference(&white, 8, 2); + let out = cpu_reference(&white, 8, 2, true); let (cw, ch) = (4usize, 1usize); let y_size = 8 * 2; for &y in &out[..y_size] { assert!(y >= 254, "white Y={y}"); } for &u in &out[y_size..y_size + cw * ch] { assert!((u as i32 - 128).abs() <= 1, "white U={u}"); } let black = vec![0u8; 8 * 2 * 4]; - let out = cpu_reference(&black, 8, 2); + let out = cpu_reference(&black, 8, 2, true); for &y in &out[..y_size] { assert_eq!(y, 0); } for &v in &out[y_size + cw * ch..] { assert!((v as i32 - 128).abs() <= 1, "black V={v}"); } } @@ -280,7 +318,7 @@ mod tests { fn reference_red_bt709() { // Solid red (255,0,0): Y=0.2126*255≈54; V high, U low (full range). let red: Vec = (0..8 * 2).flat_map(|_| [255u8, 0, 0, 255]).collect(); - let out = cpu_reference(&red, 8, 2); + let out = cpu_reference(&red, 8, 2, true); assert!((out[0] as i32 - 54).abs() <= 1, "red Y={}", out[0]); let y_size = 8 * 2; let u = out[y_size]; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 7b7a7fc..57aef6c 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -57,6 +57,9 @@ pub struct VideoExportState { hdr: lightningbeam_core::export::HdrExportMode, /// How the document is fit into the export frame (stretch/letterbox/crop). fit: lightningbeam_core::export::ExportFitMode, + /// SDR color range: true = full (PC, 0–255), false = limited (TV, 16–235). The YUV + /// converters and the encoder color tag must agree on this. + full_range: bool, /// Channel to send rendered frames to encoder thread frame_tx: Option>, /// HDR GPU resources for compositing pipeline (effects, color conversion) @@ -868,6 +871,7 @@ impl ExportOrchestrator { ) -> (std::thread::JoinHandle<()>, VideoExportState) { let hdr = settings.hdr; let fit = settings.fit; + let full_range = settings.color_range.is_full(); let handle = std::thread::spawn(move || { Self::run_video_encoder(settings, output_path, frame_rx, progress_tx, cancel_flag, total_frames); }); @@ -882,6 +886,7 @@ impl ExportOrchestrator { height, hdr, fit, + full_range, frame_tx: Some(frame_tx), gpu_resources: None, readback_pipeline: None, @@ -1333,8 +1338,8 @@ impl ExportOrchestrator { if !gpu_yuv_tight { println!("🎬 [VIDEO EXPORT] YUV planes are padded at {width}x{height}; using CPU YUV path"); } - state.readback_pipeline = Some(readback_pipeline::ReadbackPipeline::new(device, queue, width, height, gpu_yuv_tight)); - state.cpu_yuv_converter = Some(cpu_yuv_converter::CpuYuvConverter::new(width, height)?); + state.readback_pipeline = Some(readback_pipeline::ReadbackPipeline::new(device, queue, width, height, gpu_yuv_tight, state.full_range)); + state.cpu_yuv_converter = Some(cpu_yuv_converter::CpuYuvConverter::new(width, height, state.full_range)?); println!("🚀 [ASYNC PIPELINE] Triple-buffered pipeline initialized"); println!("🚀 [CPU YUV] swscale converter initialized"); } @@ -1638,6 +1643,21 @@ impl ExportOrchestrator { // Convert codec enum to FFmpeg codec ID. HDR requires 10-bit HEVC (Main10), so force HEVC // regardless of the chosen codec when an HDR mode is selected. let codec_id = if settings.hdr.is_hdr() { + // HEVC can only be muxed into MP4/MOV, not WebM — reject the incompatible combo up + // front with a clear message instead of letting the muxer fail cryptically. + if settings.codec.container_format() == "webm" { + return Err(format!( + "HDR export needs H.265/HEVC in an MP4 container, but {} uses WebM. \ + Pick H.265 (or H.264) for HDR.", + settings.codec.name() + )); + } + if !matches!(settings.codec, VideoCodec::H265) { + println!( + "⚠️ [ENCODER] HDR selected: overriding codec {} → H.265/HEVC (Main10)", + settings.codec.name() + ); + } ffmpeg_next::codec::Id::HEVC } else { match settings.codec { @@ -1690,6 +1710,7 @@ impl ExportOrchestrator { framerate, bitrate_kbps, settings.hdr, + settings.color_range.is_full(), )?; // Pixel format the encoder frames are built in (matches setup_video_encoder). diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs b/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs index 5997e5e..df552f3 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs @@ -91,12 +91,12 @@ impl ReadbackPipeline { /// `enable_gpu_yuv` should be `true` only when the caller has verified the encoder's /// `YUV420P` plane strides are tight (== width / width-2), so the packed GPU planes /// drop straight into the `AVFrame` without row misalignment. - pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, width: u32, height: u32, enable_gpu_yuv: bool) -> Self { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, width: u32, height: u32, enable_gpu_yuv: bool, full_range: bool) -> Self { let (readback_tx, readback_rx) = channel(); // GPU YUV conversion when enabled AND the dimensions fit the packed shader; else RGBA + CPU. let gpu_yuv = if enable_gpu_yuv && super::gpu_yuv::supports(width, height) { - Some(super::gpu_yuv::GpuYuv::new(device)) + Some(super::gpu_yuv::GpuYuv::new(device, full_range)) } else { None }; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 1c65ace..b27694f 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -560,6 +560,7 @@ pub fn setup_video_encoder( framerate: f64, bitrate_kbps: u32, hdr: lightningbeam_core::export::HdrExportMode, + full_range: bool, ) -> Result<(ffmpeg::encoder::Video, ffmpeg::Codec), String> { // Try to find codec by ID first println!("🔍 Looking for codec: {:?}", codec_id); @@ -641,7 +642,12 @@ pub fn setup_video_encoder( color_opts.set("profile", "main10"); } else { encoder.set_colorspace(ffmpeg::color::Space::BT709); - encoder.set_color_range(ffmpeg::color::Range::JPEG); // full range + // Range must match what the YUV converters (gpu_yuv / cpu_yuv) actually produce. + encoder.set_color_range(if full_range { + ffmpeg::color::Range::JPEG // full (PC, 0–255) + } else { + ffmpeg::color::Range::MPEG // limited (TV, 16–235) + }); color_opts.set("color_primaries", "bt709"); color_opts.set("color_trc", "bt709"); } @@ -1133,232 +1139,9 @@ fn upload_transient_texture( tex } -/// Render a document frame using the HDR compositing pipeline with effects -/// -/// This function uses the same rendering pipeline as the stage preview, -/// ensuring effects are applied correctly during export. -/// -/// # Arguments -/// * `document` - Document to render (current_time will be modified) -/// * `timestamp` - Time in seconds to render at -/// * `width` - Frame width in pixels -/// * `height` - Frame height in pixels -/// * `device` - wgpu device -/// * `queue` - wgpu queue -/// * `renderer` - Vello renderer -/// * `image_cache` - Image cache for rendering -/// * `video_manager` - Video manager for video clips -/// * `gpu_resources` - HDR GPU resources for compositing -/// -/// # Returns -/// Ok((y_plane, u_plane, v_plane)) with YUV420p planes on success, Err with message on failure -pub fn render_frame_to_rgba_hdr( - document: &mut Document, - timestamp: f64, - width: u32, - height: u32, - device: &wgpu::Device, - queue: &wgpu::Queue, - renderer: &mut vello::Renderer, - image_cache: &mut ImageCache, - video_manager: &Arc>, - gpu_resources: &mut ExportGpuResources, -) -> Result<(Vec, Vec, Vec), String> { - use vello::kurbo::Affine; - - // Set document time to the frame timestamp - document.current_time = timestamp; - - // Scale the document to the export resolution. The core renderer bakes this - // base transform into every layer (vector scenes, raster and video layer - // transforms), so the whole stage scales up/down to fill the output. When the - // export size matches the document this is the identity. - let base_transform = if document.width > 0.0 && document.height > 0.0 { - Affine::scale_non_uniform( - width as f64 / document.width, - height as f64 / document.height, - ) - } else { - Affine::IDENTITY - }; - - // Export composites on the encoder's own device, not the shared one, so it cannot use - // hardware-decoded GPU frames (textures can't cross devices). Force software frames; a - // hardware decoder downloads its surface to CPU instead. - if let Ok(mut vm) = video_manager.lock() { - vm.set_render_hardware_ok(false); - } - - // Render document for compositing (returns per-layer scenes) - let composite_result = render_document_for_compositing( - document, - base_transform, - image_cache, - video_manager, - None, // No webcam during export - None, // No floating selection during export - false, // No checkerboard in export - ); - - // Video export is never transparent. - composite_document_to_hdr(&composite_result, document, device, queue, renderer, gpu_resources, width, height, false)?; - - // Use persistent output texture (already created in ExportGpuResources) - let output_view = &gpu_resources.output_texture_view; - - // Convert HDR to sRGB for output - let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("export_linear_to_srgb_bind_group"), - layout: &gpu_resources.linear_to_srgb_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&gpu_resources.hdr_texture_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler(&gpu_resources.linear_to_srgb_sampler), - }, - ], - }); - - let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("export_linear_to_srgb_encoder"), - }); - - { - let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("export_linear_to_srgb_pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &output_view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), - store: wgpu::StoreOp::Store, - }, - depth_slice: None, - })], - depth_stencil_attachment: None, - occlusion_query_set: None, - timestamp_writes: None, - }); - - let final_pipeline = match document.hdr_output_mode { - lightningbeam_core::document::HdrOutputMode::HighlightRolloff => &gpu_resources.linear_to_srgb_pipeline_rolloff, - lightningbeam_core::document::HdrOutputMode::Clip => &gpu_resources.linear_to_srgb_pipeline, - }; - render_pass.set_pipeline(final_pipeline); - render_pass.set_bind_group(0, &bind_group, &[]); - render_pass.draw(0..4, 0..1); - } - - queue.submit(Some(encoder.finish())); - - // GPU YUV conversion: Convert RGBA output to YUV420p - let mut yuv_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("export_yuv_conversion_encoder"), - }); - - gpu_resources.yuv_converter.convert( - device, - &mut yuv_encoder, - output_view, - &gpu_resources.yuv_texture_view, - width, - height, - ); - - // Copy YUV texture to persistent staging buffer - let yuv_height = height + height / 2; // Y plane + U plane + V plane - yuv_encoder.copy_texture_to_buffer( - wgpu::TexelCopyTextureInfo { - texture: &gpu_resources.yuv_texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::TexelCopyBufferInfo { - buffer: &gpu_resources.staging_buffer, - layout: wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(width * 4), // Rgba8Unorm = 4 bytes per pixel - rows_per_image: Some(yuv_height), - }, - }, - wgpu::Extent3d { - width, - height: yuv_height, - depth_or_array_layers: 1, - }, - ); - - queue.submit(Some(yuv_encoder.finish())); - - // Map buffer and read YUV pixels (synchronous) - let buffer_slice = gpu_resources.staging_buffer.slice(..); - let (sender, receiver) = std::sync::mpsc::channel(); - buffer_slice.map_async(wgpu::MapMode::Read, move |result| { - sender.send(result).ok(); - }); - - let _ = device.poll(wgpu::PollType::wait_indefinitely()); - - receiver - .recv() - .map_err(|_| "Failed to receive buffer mapping result")? - .map_err(|e| format!("Failed to map buffer: {:?}", e))?; - - // Extract Y, U, V planes from packed YUV buffer - let data = buffer_slice.get_mapped_range(); - let width_usize = width as usize; - let height_usize = height as usize; - - // Y plane: rows 0 to height-1 (extract R channel from Rgba8Unorm) - let y_plane_size = width_usize * height_usize; - let mut y_plane = vec![0u8; y_plane_size]; - for y in 0..height_usize { - let src_row_offset = y * width_usize * 4; // 4 bytes per pixel (Rgba8Unorm) - let dst_row_offset = y * width_usize; - for x in 0..width_usize { - y_plane[dst_row_offset + x] = data[src_row_offset + x * 4]; // Extract R channel - } - } - - // U and V planes: rows height to height + height/2 - 1 (half resolution, side-by-side layout) - // U plane is in left half (columns 0 to width/2-1), V plane is in right half (columns width/2 to width-1) - let chroma_width = width_usize / 2; - let chroma_height = height_usize / 2; - let chroma_row_start = height_usize * width_usize * 4; // Start of chroma rows in bytes - - let mut u_plane = vec![0u8; chroma_width * chroma_height]; - let mut v_plane = vec![0u8; chroma_width * chroma_height]; - - for y in 0..chroma_height { - let row_offset = chroma_row_start + y * width_usize * 4; // Full width rows in chroma region - - // Extract U plane (left half: columns 0 to chroma_width-1) - let u_start = row_offset; - let dst_offset = y * chroma_width; - for x in 0..chroma_width { - u_plane[dst_offset + x] = data[u_start + x * 4]; // Extract R channel - } - - // Extract V plane (right half: columns width/2 to width/2+chroma_width-1) - let v_start = row_offset + chroma_width * 4; - for x in 0..chroma_width { - v_plane[dst_offset + x] = data[v_start + x * 4]; // Extract R channel - } - } - - drop(data); - gpu_resources.staging_buffer.unmap(); - - Ok((y_plane, u_plane, v_plane)) -} - /// Render frame to GPU RGBA texture (non-blocking, for async pipeline) /// -/// Similar to render_frame_to_rgba_hdr but renders to an external RGBA texture view +/// Renders to an external RGBA texture view /// (provided by ReadbackPipeline) and returns the command encoder WITHOUT blocking on readback. /// The caller (ReadbackPipeline) will submit the encoder and handle async readback. /// diff --git a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs index 80586d4..07ba0b1 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs @@ -1627,12 +1627,24 @@ impl GpuBrushEngine { self.proxy_layer_cache.get(kf_id) } - /// Remove the cached texture for a raster layer keyframe (e.g. when deleted). + /// Remove the cached texture for a raster layer keyframe (e.g. when deleted or edited). pub fn remove_layer_texture(&mut self, kf_id: &Uuid) { - if self.raster_layer_cache.remove(kf_id).is_some() { + let mut changed = self.raster_layer_cache.remove(kf_id).is_some(); + if changed { if let Some(pos) = self.raster_layer_lru.iter().position(|id| id == kf_id) { self.raster_layer_lru.remove(pos); } + } + // Also drop the low-res proxy: proxies are uploaded once and never refreshed, so a + // stale pre-edit proxy left here would be blitted (flashing old content) if the full-res + // texture is later evicted before the edited pixels page back in. + if self.proxy_layer_cache.remove(kf_id).is_some() { + if let Some(pos) = self.proxy_layer_lru.iter().position(|id| id == kf_id) { + self.proxy_layer_lru.remove(pos); + } + changed = true; + } + if changed { self.report_raster_cache_vram(); } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 7073e2d..09aff3f 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -2392,6 +2392,12 @@ impl EditorApp { kf.needs_fault_in = false; } } + // Track these resident pixels in the LRU so they count toward + // RASTER_RESIDENT_MAX and can be evicted later; without this, a frame + // faulted in for undo/redo that ends up clean would stay resident forever, + // letting resident RAM grow past the cap. + self.raster_resident_lru.retain(|id| *id != kf_id); + self.raster_resident_lru.push_back(kf_id); } } } @@ -4523,7 +4529,9 @@ impl EditorApp { let bytes = match std::fs::read(path) { Ok(b) => b, Err(e) => { - eprintln!("❌ Failed to read SVG {}: {}", path.display(), e); + let msg = format!("Failed to read SVG: {}", e); + eprintln!("❌ {} ({})", msg, path.display()); + notifications::notify_error("SVG Import Failed", &msg); return; } }; @@ -4532,6 +4540,7 @@ impl EditorApp { Ok(g) => g, Err(e) => { eprintln!("❌ {}", e); + notifications::notify_error("SVG Import Failed", &e); return; } }; diff --git a/lightningbeam-ui/lightningbeam-editor/src/notifications.rs b/lightningbeam-ui/lightningbeam-editor/src/notifications.rs index 2222d58..0489df2 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/notifications.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/notifications.rs @@ -32,6 +32,11 @@ pub fn notify_export_complete(output_path: &Path) { /// Show a desktop notification for an export error (fire-and-forget). pub fn notify_export_error(error_message: &str) { + notify_error("Export Failed", error_message); +} + +/// Show a desktop error notification with a custom title (fire-and-forget). +pub fn notify_error(title: &'static str, error_message: &str) { // Truncate very long error messages (on a char boundary). let truncated = if error_message.chars().count() > 100 { let prefix: String = error_message.chars().take(97).collect(); @@ -42,7 +47,7 @@ pub fn notify_export_error(error_message: &str) { std::thread::spawn(move || { if let Err(e) = Notification::new() - .summary("Export Failed") + .summary(title) .body(&truncated) .icon("dialog-error") // Standard error icon .timeout(10000) // 10 seconds for errors (longer to read) diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 46beb28..92d85be 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -2080,7 +2080,11 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // fills the black's gaps → interlocking black/yellow marching-ants. if let Some(active_id) = self.ctx.active_layer_id { if let Some(lightningbeam_core::layer::AnyLayer::Raster(rl)) = self.ctx.document.get_layer(&active_id) { - if let Some(kf) = rl.keyframe_at(self.ctx.document.current_time) { + // Use playback_time (clip-local when editing a movie clip) like every other + // keyframe lookup in prepare(), and overlay_transform so the outline tracks the + // layer's pixels through any clip-instance affine (it equals camera_transform + // outside clip-edit mode). + if let Some(kf) = rl.keyframe_at(self.ctx.playback_time) { let rect = vello::kurbo::Rect::new(0.0, 0.0, kf.width as f64, kf.height as f64); // Sizes are in document space; divide by zoom so they're ~constant on screen. let inv_zoom = 1.0 / (self.ctx.zoom as f64).max(1e-3); @@ -2089,14 +2093,14 @@ impl egui_wgpu::CallbackTrait for VelloCallback { let pattern = [dash, dash]; scene.stroke( &vello::kurbo::Stroke::new(stroke_w).with_dashes(0.0, pattern), - camera_transform, + overlay_transform, vello::peniko::Color::new([0.0, 0.0, 0.0, 1.0]), None, &rect, ); scene.stroke( &vello::kurbo::Stroke::new(stroke_w).with_dashes(dash, pattern), - camera_transform, + overlay_transform, vello::peniko::Color::new([1.0, 0.85, 0.0, 1.0]), None, &rect, diff --git a/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs b/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs index c0cf4b1..be793eb 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs @@ -118,10 +118,14 @@ fn convert_path(path: &usvg::Path, graph: &mut VectorGraph) { slot.color = Some(ShapeColor::rgba(c.red, c.green, c.blue, opacity_u8(fill.opacity()))); } usvg::Paint::LinearGradient(g) => { - slot.gradient_fill = Some(linear_gradient(g, ts)); + let mut grad = linear_gradient(g, ts); + apply_fill_opacity(&mut grad, fill.opacity()); + slot.gradient_fill = Some(grad); } usvg::Paint::RadialGradient(g) => { - slot.gradient_fill = Some(radial_gradient(g, ts)); + let mut grad = radial_gradient(g, ts); + apply_fill_opacity(&mut grad, fill.opacity()); + slot.gradient_fill = Some(grad); } usvg::Paint::Pattern(_) => { // Patterns aren't representable yet — neutral gray so the shape stays visible. @@ -217,6 +221,17 @@ fn radial_gradient(g: &usvg::RadialGradient, abs: usvg::Transform) -> ShapeGradi } } +/// Fold the path's `fill-opacity` into a gradient's stop alphas (SVG multiplies them). +fn apply_fill_opacity(grad: &mut ShapeGradient, op: usvg::Opacity) { + let f = op.get(); + if f >= 1.0 { + return; + } + for s in &mut grad.stops { + s.color.a = (s.color.a as f32 * f).round().clamp(0.0, 255.0) as u8; + } +} + fn gradient_stops(base: &usvg::BaseGradient) -> Vec { base.stops() .iter() From 97172892713acca5b28af2cc6da199247ff0931d Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 21:38:07 -0400 Subject: [PATCH 30/31] Implement layer visibility toggle; log export color range - Wire MenuAction::ToggleLayerVisibility (was a stub) to flip the active layer's `visible` flag via SetLayerPropertiesAction (undoable). This is what the SVG-export visibility fix depends on, and the "Hide/Show Layer" menu item now works. - Log the resolved color range in the zero-copy H.264 path to diagnose whether `full_range` reaches the encoder (the VAAPI driver may still omit the SPS VUI full_range flag). Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-editor/src/export/mod.rs | 2 ++ .../lightningbeam-editor/src/main.rs | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 57aef6c..d5b2851 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -921,6 +921,8 @@ impl ExportOrchestrator { let bitrate = settings.quality.bitrate_kbps(); let fr = framerate.round() as i32; let full = settings.color_range.is_full(); + println!("🎬 [EXPORT] zero-copy H.264 color range: {} (full_range={})", + settings.color_range.name(), full); let on_shared_device = shared_device.is_some(); // Prefer the shared device → decode→composite→encode stay GPU-resident on one device. // Without it, the encoder builds its own device (decode still downloads to CPU per Step 1's diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 09aff3f..435f7cf 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -3727,8 +3727,21 @@ impl EditorApp { // TODO: Implement delete layer } MenuAction::ToggleLayerVisibility => { - println!("Menu: Toggle Layer Visibility"); - // TODO: Implement toggle layer visibility + use lightningbeam_core::actions::{SetLayerPropertiesAction, set_layer_properties::LayerProperty}; + use lightningbeam_core::layer::LayerTrait; + if let Some(layer_id) = self.active_layer_id { + let cur = self.action_executor.document() + .get_layer(&layer_id) + .map(|l| l.visible()); + if let Some(cur) = cur { + let action = SetLayerPropertiesAction::new(layer_id, LayerProperty::Visible(!cur)); + if let Err(e) = self.action_executor.execute(Box::new(action)) { + eprintln!("Toggle layer visibility: {}", e); + } + } + } else { + println!("Toggle Layer Visibility: no active layer"); + } } MenuAction::ShowMasterTrack => { // Toggle show_master_track on all Timeline pane instances From 58a3c829d7e11466ac1e761816f8d998950fd5da Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 21:54:50 -0400 Subject: [PATCH 31/31] Bump version to 1.0.7-alpha --- Changelog.md | 24 +++++++++++++++++++ .../lightningbeam-editor/Cargo.toml | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 6cf0e17..9d4eaf4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,27 @@ +# 1.0.7-alpha: +Changes: +- HDR video support: PQ/HLG/BT.2020 video is now read correctly (decoded to scene-linear), with a per-document output mode (clip vs highlight rolloff) and 10-bit HDR export (HEVC Main10, PQ or HLG) +- Hardware-accelerated video decode (VAAPI) for both playback and export, including a GPU NV12 preview path; the editor now runs on a shared VAAPI-capable GPU device so decode → composite → encode stay GPU-resident +- SVG import and export for vector layers: export the current frame to .svg, and import .svg as a new vector layer (Ctrl+I) +- Export fit modes: choose Stretch, Letterbox, or Crop when the export resolution's aspect ratio differs from the document (video and image export) +- Videos imported directly and dragged in from the asset library now use the same placement, so they no longer end up with different aspect ratios +- Resizing the document now leaves raster layers untouched; the Info Panel shows the active raster layer's size and a "Layer to document size" button (scale or expand/crop) +- The active raster layer now shows a dashed outline on the canvas +- H.264 export gained a color-range option (Limited/TV or Full/PC) +- Hide/Show Layer now works + +Bugfixes: +- Fix sped-up and jerky 4K video playback (frame-index frame cache + request-based seeking) +- Fix washed-out HDR/10-bit video (propagate stream color tags to hardware frames, P010 import) +- Fix the final frame(s) of a clip occasionally failing to render at the end of the stream +- Fix the CPU export color path producing shifted colors on unusual resolutions (BT.709 + honor the chosen range) +- Fix fills occasionally vanishing after a paint-bucket or lasso cut +- Fix silent gaps in exported audio +- Fix black video thumbnails and decoder thrashing +- Fix file-descriptor and GPU-memory leaks on hardware-import error paths +- SVG export now omits hidden layers; SVG import respects gradient opacity +- Fix the macOS/Windows build (hardware export is Linux-only) + # 1.0.6-alpha: Changes: - Hardware-accelerated H.264 video export: each frame is rendered and encoded on the GPU (zero-copy VAAPI), roughly 2x faster, with automatic fallback to software encoding when hardware acceleration isn't available (Linux, Intel/AMD only for now) diff --git a/lightningbeam-ui/lightningbeam-editor/Cargo.toml b/lightningbeam-ui/lightningbeam-editor/Cargo.toml index 9ad8b55..c80d068 100644 --- a/lightningbeam-ui/lightningbeam-editor/Cargo.toml +++ b/lightningbeam-ui/lightningbeam-editor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightningbeam-editor" -version = "1.0.6-alpha" +version = "1.0.7-alpha" edition = "2021" description = "Multimedia editor for audio, video and 2D animation" license = "GPL-3.0-or-later"