diff --git a/Changelog.md b/Changelog.md index 6cf0e17..9d4eaf4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,27 @@ +# 1.0.7-alpha: +Changes: +- HDR video support: PQ/HLG/BT.2020 video is now read correctly (decoded to scene-linear), with a per-document output mode (clip vs highlight rolloff) and 10-bit HDR export (HEVC Main10, PQ or HLG) +- Hardware-accelerated video decode (VAAPI) for both playback and export, including a GPU NV12 preview path; the editor now runs on a shared VAAPI-capable GPU device so decode → composite → encode stay GPU-resident +- SVG import and export for vector layers: export the current frame to .svg, and import .svg as a new vector layer (Ctrl+I) +- Export fit modes: choose Stretch, Letterbox, or Crop when the export resolution's aspect ratio differs from the document (video and image export) +- Videos imported directly and dragged in from the asset library now use the same placement, so they no longer end up with different aspect ratios +- Resizing the document now leaves raster layers untouched; the Info Panel shows the active raster layer's size and a "Layer to document size" button (scale or expand/crop) +- The active raster layer now shows a dashed outline on the canvas +- H.264 export gained a color-range option (Limited/TV or Full/PC) +- Hide/Show Layer now works + +Bugfixes: +- Fix sped-up and jerky 4K video playback (frame-index frame cache + request-based seeking) +- Fix washed-out HDR/10-bit video (propagate stream color tags to hardware frames, P010 import) +- Fix the final frame(s) of a clip occasionally failing to render at the end of the stream +- Fix the CPU export color path producing shifted colors on unusual resolutions (BT.709 + honor the chosen range) +- Fix fills occasionally vanishing after a paint-bucket or lasso cut +- Fix silent gaps in exported audio +- Fix black video thumbnails and decoder thrashing +- Fix file-descriptor and GPU-memory leaks on hardware-import error paths +- SVG export now omits hidden layers; SVG import respects gradient opacity +- Fix the macOS/Windows build (hardware export is Linux-only) + # 1.0.6-alpha: Changes: - Hardware-accelerated H.264 video export: each frame is rendered and encoded on the GPU (zero-copy VAAPI), roughly 2x faster, with automatic fallback to software encoding when hardware acceleration isn't available (Linux, Intel/AMD only for now) diff --git a/daw-backend/src/audio/pool.rs b/daw-backend/src/audio/pool.rs index cbe6a51..c130dd3 100644 --- a/daw-backend/src/audio/pool.rs +++ b/daw-backend/src/audio/pool.rs @@ -582,8 +582,21 @@ impl AudioClipPool { // or a container without exact stream duration). They will never // arrive — stop waiting and let the render fill silence, instead of // burning the 10s safety valve on every remaining chunk. + // + // BUT `finished` may be stale: we just moved the target with + // `set_target_frame`, and on a forward jump past the buffer the reader + // will reset()+seek() (clearing `finished`) on its next poll. The reader + // re-seeks when `target < buf_start || target > buf_end + sample_rate` + // (see disk_reader.rs), so only trust EOF when the target is inside the + // current window — otherwise a stale flag from the previous region would + // make us render silence over audio that's about to be decoded. if ra.is_finished() { - break; + let (buf_start, buf_end) = ra.snapshot(); + let sr = audio_file.sample_rate as u64; + let seek_pending = src_start < buf_start || src_start > buf_end + sr; + if !seek_pending { + break; + } } std::thread::sleep(std::time::Duration::from_micros(100)); wait_iters += 1; diff --git a/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md b/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md new file mode 100644 index 0000000..0586549 --- /dev/null +++ b/lightningbeam-ui/GPU_VIDEO_DECODE_PLAN.md @@ -0,0 +1,102 @@ +# GPU-resident video decode + dynamic decode resolution + +## Context + +Profiling the zero-copy H.264 export (single Group[Video, Audio] clip, `LB_RENDER_PROFILE=1`) +broke the per-frame CPU "render" bucket down as: + +| Cost (ms/frame) | 1080p | 4K | What it is | +|-------------------------|-------|-------|-----------------------------------------------------| +| decode | 3.1 | 19.0 | software ffmpeg decode (`video.rs::get_frame`) | +| background re-render | 3.6 | 7.5 | static background pushed through Vello *every frame* | +| video upload + blit | 4.1 | 4.2 | per-frame transient texture alloc + `write_texture` | +| srgb | 0.4 | 0.4 | linear→sRGB pass | + +The video correctly takes the GPU Video-instance path (not Vello-baked) — `LB_LAYER_DEBUG=1` +shows `Video (1 instance)`. So the cost is **the video frame itself**: software decode, then an +8 MB `write_texture` upload of the decoded RGBA every frame. At 4K, software decode (19 ms) +dominates everything. + +### Two correctness problems found alongside the perf issue + +1. **Decode resolution is frozen to document size at import.** `load_video(clip, src, doc_w, doc_h)` + (`main.rs:4302`) sizes the decoder's swscale output to the document, capped to never upscale + (`video.rs:149`). Export *reuses that decoder*, so exporting **above** document resolution yields + video that was decoded to ≤document res and then GPU-**up**scaled — real source detail thrown away. +2. **It can't follow the consumer or a document resize.** Preview wants small/fast frames; export + wants full res; changing the document size should re-target the decode. None of that works with a + size frozen at import. + +## Goal + +Decouple **decode resolution** from import/document size: the renderer requests a frame *at a target +resolution*, and the decode path produces it. Hardware-decode H.264 (and later HEVC/AV1) into a GPU +surface and keep it GPU-resident through composite into the encoder — no CPU frame copy in either +direction. Software decode stays a **first-class** path (codecs/platforms without HW support), decoding +at the requested target res. This fixes the 4K decode wall, the 8 MB upload, *and* the resolution bugs. + +## Design principles + +- **Decode native, scale to the consumer's target.** + - *Hardware path:* decode into a native VAAPI surface → import as a wgpu texture (reuse the + `gpu-video-encoder` `dmabuf.rs` / `vk_device.rs` plumbing, read direction) → the GPU blit that + already composites the Video instance scales native→target for free. Handles any target res and + document resizes inherently; the cached frame is a native GPU texture. + - *Software path:* decode native → `swscale` to the requested target (the reusable scaler is keyed + on input format/size **and** output size — rebuilt when the target changes). Preview requests + preview res (cheap); export requests export res (full quality). +- **`VideoManager::get_frame` takes a target `(w, h)`** instead of relying on a frozen output size. + The frame cache is keyed to handle multiple live targets (preview + export) — either cache native + frames and scale on demand, or key by `(clip, ts, target)`; decide in Stage 2 by measuring cache + hit/scale tradeoff. +- **Software is not optional.** Hardware decode is an acceleration of the same `get_frame` contract, + selected per source when the codec/driver supports it; everything falls back to software cleanly. + +## Approach (staged; each stage compiles + is independently useful) + +### Stage 0 — independent quick wins (not blocked on decode) +- **Cache the static background** (`composite_document_to_hdr`): render once, reuse via a persistent + HDR texture (copy-in each frame) instead of a full Vello render + 2 passes/submits every frame. + Recovers ~3.6 ms (1080p) / ~7.5 ms (4K) per frame on *every* export. (In flight.) + +### Stage 1 — software: decode at the requested target res (testable; fixes the quality bug now) +- Change `VideoManager::get_frame(clip, ts)` → `get_frame(clip, ts, target_w, target_h)`; thread the + target from the renderer (preview = current doc/preview res, export = export res). Cap at native. +- Rework `VideoDecoder` so output size is per-request, not frozen at construction; cache the swscale + context per output size (already cached per stream — extend the key). Adjust the frame cache key. +- Result: software exports are full-quality at any export res, and document resizes re-target decode. + No hardware needed; this is the correctness fix for the codecs HW can't handle anyway. + +### Stage 2 — hardware decode primitive (headless-testable here, like the 8 encode tests) +- In `gpu-video-encoder` (rename → `gpu-video-codec`): `h264_vaapi`-style **decode** → VAAPI surface → + export DMA-BUF → import as a wgpu texture. Hardware test: decode a known file, verify dims/contents. + +### Stage 3 — wire hardware decode into `get_frame` (blind; user-verifies) +- When the source codec/driver is HW-decodable, `get_frame` returns a **GPU texture** (native res) + instead of `Arc>`; the compositor uses it directly (no `write_texture`), GPU-scaling to the + target. For the zero-copy export the frame never leaves the GPU: **decode → composite → encode** on + one device. Software path is the fallback for everything else. + +## Critical files +- `lightningbeam-core/src/video.rs` — `VideoDecoder` (per-request output size, scaler cache), + `VideoManager::get_frame` (target param, cache key). +- `lightningbeam-core/src/renderer.rs` — pass the render target res into the video-instance build. +- `lightningbeam-editor/src/export/video_exporter.rs` — background cache (Stage 0); consume a GPU + texture instead of uploading RGBA (Stage 3). +- `gpu-video-encoder/` (→ `gpu-video-codec`) — `dmabuf.rs`/`vk_device.rs` reused for the decode import. + +## Risks +- **Codec coverage** — only some codecs are HW-decodable per GPU/driver; software must stay correct + and well-tested. Selection must probe support per source, not assume. +- **Cache memory** — native-res GPU textures (esp. 4K) are large; the frame cache budget needs revisiting. +- **Colorspace/format** — VAAPI decode surfaces are NV12/tiled; the existing import handles NV12, but + 10-bit/HDR sources (P010) need format handling. +- **Preview vs export sharing** — two live targets (preview res + export res) from the same source; the + cache/scaler design must serve both without thrashing. + +## Verification +- Stage 0/1: visual — export above document res is now full-quality (not upscaled); profile shows + background ≈ 0 and (Stage 1) software export correct at the chosen res. +- Stage 2: headless hardware test in `gpu-video-codec` (decode → wgpu texture, ffprobe/byte checks). +- Stage 3 (user): 1080p + 4K H.264 export — decode/upload buckets collapse; software fallback for a + non-HW codec (e.g. ProRes) still produces correct full-res output. diff --git a/lightningbeam-ui/gpu-video-encoder/src/decoder.rs b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs new file mode 100644 index 0000000..884c66f --- /dev/null +++ b/lightningbeam-ui/gpu-video-encoder/src/decoder.rs @@ -0,0 +1,217 @@ +//! VAAPI hardware video decode → wgpu textures. The mirror of [`crate::encoder`]: the codec +//! decodes into a VAAPI NV12 surface, which is mapped to a DRM-PRIME DMA-BUF and imported as two +//! wgpu plane textures via [`crate::dmabuf::import_raw`] — the exact same path the encoder uses, +//! in the read direction. Stays GPU-resident: no CPU frame copy. + +use crate::dmabuf::{self, ImportedNv12, Nv12DmaBuf}; +use crate::vk_device::{self, DrmDevice}; +use ffmpeg_sys_next as ff; +use std::ffi::CString; +use std::path::Path; +use std::ptr; + +#[inline] +fn averror(e: i32) -> i32 { + -e +} + +/// `get_format` callback: pick VAAPI surfaces so the decoder outputs hardware frames. With +/// `hw_device_ctx` set, FFmpeg auto-allocates the matching frames context. +unsafe extern "C" fn get_vaapi_format( + _ctx: *mut ff::AVCodecContext, + mut fmts: *const ff::AVPixelFormat, +) -> ff::AVPixelFormat { + while *fmts != ff::AVPixelFormat::AV_PIX_FMT_NONE { + if *fmts == ff::AVPixelFormat::AV_PIX_FMT_VAAPI { + return ff::AVPixelFormat::AV_PIX_FMT_VAAPI; + } + fmts = fmts.add(1); + } + ff::AVPixelFormat::AV_PIX_FMT_NONE +} + +/// Hardware decoder for a single video file/stream. Frames come back as importable NV12 textures +/// on [`Self::device`]. +pub struct VaapiDecoder { + drm: DrmDevice, + hw_device: *mut ff::AVBufferRef, + fmt: *mut ff::AVFormatContext, + dec: *mut ff::AVCodecContext, + pkt: *mut ff::AVPacket, + frame: *mut ff::AVFrame, + stream_index: i32, + flushing: bool, +} + +// Owns its FFmpeg/Vulkan handles exclusively; only moved, never shared (same as the encoder). +unsafe impl Send for VaapiDecoder {} + +impl VaapiDecoder { + /// Open `input_path` and set up VAAPI hardware decoding of its best video stream. + pub fn new(input_path: &Path) -> Result { + let drm = vk_device::create()?; + unsafe { + let mut hw_device = crate::vaapi::create_device()?; + let cleanup_hw = |hw: *mut ff::AVBufferRef| { + let mut h = hw; + ff::av_buffer_unref(&mut h); + }; + + let path_c = CString::new(input_path.to_string_lossy().as_ref()).unwrap(); + let mut fmt: *mut ff::AVFormatContext = ptr::null_mut(); + if ff::avformat_open_input(&mut fmt, path_c.as_ptr(), ptr::null_mut(), ptr::null_mut()) < 0 { + cleanup_hw(hw_device); + return Err(format!("avformat_open_input {input_path:?} failed")); + } + if ff::avformat_find_stream_info(fmt, ptr::null_mut()) < 0 { + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("avformat_find_stream_info failed".into()); + } + + let mut decoder: *const ff::AVCodec = ptr::null(); + let stream_index = ff::av_find_best_stream( + fmt, + ff::AVMediaType::AVMEDIA_TYPE_VIDEO, + -1, + -1, + &mut decoder, + 0, + ); + if stream_index < 0 || decoder.is_null() { + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("no decodable video stream".into()); + } + + let dec = ff::avcodec_alloc_context3(decoder); + let stream = *(*fmt).streams.add(stream_index as usize); + if ff::avcodec_parameters_to_context(dec, (*stream).codecpar) < 0 { + ff::avcodec_free_context(&mut (dec as *mut _)); + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("avcodec_parameters_to_context failed".into()); + } + (*dec).hw_device_ctx = ff::av_buffer_ref(hw_device); + (*dec).get_format = Some(get_vaapi_format); + + if ff::avcodec_open2(dec, decoder, ptr::null_mut()) < 0 { + ff::avcodec_free_context(&mut (dec as *mut _)); + ff::avformat_close_input(&mut fmt); + cleanup_hw(hw_device); + return Err("avcodec_open2 (vaapi decode) failed".into()); + } + + // `mut` only to satisfy the move into the struct; the binding above is consumed. + let _ = &mut hw_device; + Ok(Self { + drm, + hw_device, + fmt, + dec, + pkt: ff::av_packet_alloc(), + frame: ff::av_frame_alloc(), + stream_index, + flushing: false, + }) + } + } + + /// The wgpu device the decoded textures live on (the DMA-BUF-import device). + pub fn device(&self) -> &wgpu::Device { + &self.drm.device + } + pub fn queue(&self) -> &wgpu::Queue { + &self.drm.queue + } + + /// Decode the next frame and import it as NV12 plane textures, or `Ok(None)` at end of stream. + pub fn next_frame(&mut self) -> Result, String> { + unsafe { + loop { + let r = ff::avcodec_receive_frame(self.dec, self.frame); + if r == 0 { + let imported = self.map_current(); + ff::av_frame_unref(self.frame); + return imported.map(Some); + } + if r == ff::AVERROR_EOF { + return Ok(None); + } + if r != averror(libc::EAGAIN) { + return Err(format!("avcodec_receive_frame failed: {r}")); + } + if self.flushing { + return Ok(None); // already drained the flush + } + + // Decoder wants more input: pump one packet (or signal EOF to flush). + let rp = ff::av_read_frame(self.fmt, self.pkt); + if rp < 0 { + self.flushing = true; + ff::avcodec_send_packet(self.dec, ptr::null()); + continue; + } + if (*self.pkt).stream_index == self.stream_index { + let rs = ff::avcodec_send_packet(self.dec, self.pkt); + ff::av_packet_unref(self.pkt); + if rs < 0 && rs != averror(libc::EAGAIN) { + return Err(format!("avcodec_send_packet failed: {rs}")); + } + } else { + ff::av_packet_unref(self.pkt); + } + } + } + } + + /// Map the just-decoded VAAPI surface (`self.frame`) to a DRM-PRIME DMA-BUF and import it. + unsafe fn map_current(&self) -> Result { + let drm_f = ff::av_frame_alloc(); + (*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + let flags = ff::AV_HWFRAME_MAP_DIRECT as i32 | ff::AV_HWFRAME_MAP_READ as i32; + if ff::av_hwframe_map(drm_f, self.frame, flags) < 0 { + ff::av_frame_free(&mut (drm_f as *mut _)); + return Err("av_hwframe_map failed".into()); + } + let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor; + let obj = &(*desc).objects[0]; + let width = (*self.frame).width as u32; + let height = (*self.frame).height as u32; + // NV12: Y then UV — either as two layers (one plane each) or one layer with two planes. + let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 { + (&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0]) + } else { + (&(*desc).layers[0].planes[0], &(*desc).layers[0].planes[1]) + }; + let buf = Nv12DmaBuf { + fd: obj.fd, + size: obj.size as u64, + modifier: obj.format_modifier, + width, + height, + y_offset: y_pl.offset as u64, + y_pitch: y_pl.pitch as u64, + uv_offset: uv_pl.offset as u64, + uv_pitch: uv_pl.pitch as u64, + ten_bit: false, + }; + let imported = dmabuf::import_raw(&self.drm.device, &self.drm.adapter, &buf); + ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan + imported + } +} + +impl Drop for VaapiDecoder { + fn drop(&mut self) { + unsafe { + ff::av_frame_free(&mut (self.frame as *mut _)); + ff::av_packet_free(&mut (self.pkt as *mut _)); + ff::avcodec_free_context(&mut (self.dec as *mut _)); + if !self.fmt.is_null() { + ff::avformat_close_input(&mut self.fmt); + } + ff::av_buffer_unref(&mut self.hw_device); + } + } +} diff --git a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs index a6bfc33..8b536ff 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs @@ -6,7 +6,7 @@ use crate::vaapi::MappedSurface; use crate::vk_device::DrmDevice; use ash::vk; -/// Plane layout for a single-object NV12 DMA-BUF (the common VAAPI case). +/// Plane layout for a single-object NV12/P010 DMA-BUF (the common VAAPI case). #[derive(Clone, Copy)] pub struct Nv12DmaBuf { pub fd: i32, @@ -18,6 +18,9 @@ pub struct Nv12DmaBuf { pub y_pitch: u64, pub uv_offset: u64, pub uv_pitch: u64, + /// True for 10/12/16-bit content (P010 etc.): planes are 16-bit (R16/Rg16) rather than 8-bit + /// (R8/Rg8). The sampled float is normalized either way, so the consumer needs no change. + pub ten_bit: bool, } /// Frees the shared imported `VkDeviceMemory` once both plane images are gone. Held by @@ -33,6 +36,47 @@ impl Drop for MemoryGuard { } } +/// Frees the duplicated dma-buf fd and any partially-created Vulkan objects when an import +/// errors out before ownership has been handed to wgpu/`MemoryGuard`. `commit()` disarms it on +/// the success path; `fd_consumed()` is called once `vkAllocateMemory` has taken the fd. +struct ImportGuard { + device: ash::Device, + fd: libc::c_int, + img_y: vk::Image, + img_uv: vk::Image, + memory: vk::DeviceMemory, + armed: bool, +} +impl ImportGuard { + fn fd_consumed(&mut self) { + self.fd = -1; // vkAllocateMemory owns the fd now; don't close it ourselves + } + fn commit(&mut self) { + self.armed = false; + } +} +impl Drop for ImportGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + unsafe { + if self.img_uv != vk::Image::null() { + self.device.destroy_image(self.img_uv, None); + } + if self.img_y != vk::Image::null() { + self.device.destroy_image(self.img_y, None); + } + if self.memory != vk::DeviceMemory::null() { + self.device.free_memory(self.memory, None); + } + if self.fd >= 0 { + libc::close(self.fd); + } + } + } +} + /// A VAAPI surface imported as two wgpu plane textures. The underlying Vulkan image/ /// memory are destroyed by wgpu (via drop callbacks) when these textures drop. pub struct ImportedNv12 { @@ -47,12 +91,17 @@ impl ImportedNv12 { pub fn uv(&self) -> &wgpu::Texture { &self.uv } + /// Consume into the `(Y, UV)` plane textures (for handing to a longer-lived owner). + pub fn into_planes(self) -> (wgpu::Texture, wgpu::Texture) { + (self.y, self.uv) + } } -/// Convenience: map a freshly-allocated `MappedSurface` and import it. +/// Convenience: map a freshly-allocated `MappedSurface` and import it onto `drm`. pub fn import(drm: &DrmDevice, surf: &MappedSurface) -> Result { import_raw( - drm, + &drm.device, + &drm.adapter, &Nv12DmaBuf { fd: surf.fd, size: surf.size, @@ -63,22 +112,61 @@ pub fn import(drm: &DrmDevice, surf: &MappedSurface) -> Result Result { +/// Import an NV12 DMA-BUF (described by `buf`) as two wgpu plane textures **on `device`**. The raw +/// Vulkan handles are extracted from `device`/`adapter` via `as_hal`, so this works with any +/// DMA-BUF-import-capable wgpu device — the encoder/decoder's own `DrmDevice` *or* the editor's +/// shared device. The fd is duplicated, so the caller keeps ownership of theirs. +pub fn import_raw( + device: &wgpu::Device, + adapter: &wgpu::Adapter, + buf: &Nv12DmaBuf, +) -> Result { + use wgpu_hal::vulkan::Api as Vk; unsafe { - let device = drm.raw_device.clone(); - let instance = &drm.raw_instance; + let hal_device = device + .as_hal::() + .ok_or("device is not Vulkan")?; + let raw_device = hal_device.raw_device().clone(); + let raw_instance = adapter + .as_hal::() + .ok_or("adapter is not Vulkan")? + .shared_instance() + .raw_instance() + .clone(); + let instance = &raw_instance; let dup_fd = libc::dup(buf.fd); if dup_fd < 0 { return Err("dup(dma-buf fd) failed".into()); } + // Owns the fd + any Vk objects created below until ownership transfers to wgpu; on any + // early `?`/return before that, its Drop frees them (was leaking on every failed import). + let mut guard = ImportGuard { + device: raw_device.clone(), + fd: dup_fd, + img_y: vk::Image::null(), + img_uv: vk::Image::null(), + memory: vk::DeviceMemory::null(), + armed: true, + }; + // 16-bit-norm plane formats (P010) are NOT renderable, so the import is sample-only for + // those (decode path). 8-bit planes keep COLOR_ATTACHMENT for the encoder's RGBA→NV12 write. + let vk_usage = if buf.ten_bit { + vk::ImageUsageFlags::SAMPLED + | vk::ImageUsageFlags::TRANSFER_SRC + | vk::ImageUsageFlags::TRANSFER_DST + } else { + vk::ImageUsageFlags::COLOR_ATTACHMENT + | vk::ImageUsageFlags::SAMPLED + | vk::ImageUsageFlags::TRANSFER_SRC + | vk::ImageUsageFlags::TRANSFER_DST + }; let make_image = |format: vk::Format, w: u32, h: u32, pitch: u64| -> Result { let mut ext = vk::ExternalMemoryImageCreateInfo::default() .handle_types(vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT); @@ -94,30 +182,41 @@ pub fn import_raw(drm: &DrmDevice, buf: &Nv12DmaBuf) -> Result Result() - .ok_or("device is not Vulkan")?; + // Match the Vulkan usage: 16-bit-norm planes (P010) are sample-only (not renderable). + let (hal_usage, wgpu_usage) = if buf.ten_bit { + ( + wgpu_types::TextureUses::RESOURCE | wgpu_types::TextureUses::COPY_SRC, + wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_SRC, + ) + } else { + ( + wgpu_types::TextureUses::COLOR_TARGET + | wgpu_types::TextureUses::RESOURCE + | wgpu_types::TextureUses::COPY_SRC, + wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, + ) + }; let wrap = |img: vk::Image, format: wgpu::TextureFormat, w: u32, h: u32| -> wgpu::Texture { // wgpu destroys the image (after wait-idle) when the texture drops; the // captured Arc frees the shared memory once both have run. - let dev = device.clone(); + let dev = raw_device.clone(); let guard = mem_guard.clone(); let cb: wgpu_hal::DropCallback = Box::new(move || { dev.destroy_image(img, None); @@ -165,12 +278,12 @@ pub fn import_raw(drm: &DrmDevice, buf: &Nv12DmaBuf) -> Result( + device.create_texture_from_hal::( hal_tex, &wgpu::TextureDescriptor { label: Some("vaapi-plane"), @@ -179,13 +292,16 @@ pub fn import_raw(drm: &DrmDevice, buf: &Nv12DmaBuf) -> Result i32 { } pub struct ZeroCopyEncoder { - drm: DrmDevice, + /// wgpu handles the NV12 render runs on — either an own `DrmDevice`'s (via `new`) or the + /// editor's shared device (via `new_on_device`). Cloned (Arc-backed) so the source can drop. + device: wgpu::Device, + queue: wgpu::Queue, + adapter: wgpu::Adapter, renderer: Rgba2Nv12, hw_device: *mut ff::AVBufferRef, frames_ref: *mut ff::AVBufferRef, @@ -42,15 +46,39 @@ unsafe impl Send for ZeroCopyEncoder {} impl ZeroCopyEncoder { /// Build a zero-copy `h264_vaapi` encoder writing to `output_path` (container inferred /// from the extension, e.g. `.mp4`). `Err` if VAAPI/the device is unavailable. + #[allow(clippy::too_many_arguments)] pub fn new( width: u32, height: u32, framerate: i32, bitrate_kbps: u32, output_path: &Path, + full_range: bool, ) -> Result { + // Build a dedicated DMA-BUF-import device and run the encoder on it. let drm = vk_device::create()?; - let renderer = Rgba2Nv12::new(&drm.device); + Self::new_on_device( + drm.device, drm.queue, drm.adapter, + width, height, framerate, bitrate_kbps, output_path, full_range, + ) + } + + /// Build the encoder running its NV12 render + DMA-BUF import on an existing wgpu device (the + /// editor's shared device), so decode→composite→encode stay GPU-resident on one device. The + /// device must have the DMA-BUF import extensions (a `DrmDevice` / `vk_device::create_windowed`). + #[allow(clippy::too_many_arguments)] + pub fn new_on_device( + device: wgpu::Device, + queue: wgpu::Queue, + adapter: wgpu::Adapter, + width: u32, + height: u32, + framerate: i32, + bitrate_kbps: u32, + output_path: &Path, + full_range: bool, + ) -> Result { + let renderer = Rgba2Nv12::new(&device, full_range); unsafe { let mut hw_device = crate::vaapi::create_device()?; let name = CString::new("h264_vaapi").unwrap(); @@ -66,6 +94,17 @@ impl ZeroCopyEncoder { (*enc).framerate = ff::AVRational { num: framerate, den: 1 }; (*enc).pix_fmt = ff::AVPixelFormat::AV_PIX_FMT_VAAPI; (*enc).bit_rate = (bitrate_kbps as i64) * 1000; + // Color signalling for the H.264 VUI. The Rgba2Nv12 shader produces BT.709 luma/chroma + // in the matching range; without these tags players assume limited range and a + // full-range stream looks dark + oversaturated. + (*enc).color_range = if full_range { + ff::AVColorRange::AVCOL_RANGE_JPEG + } else { + ff::AVColorRange::AVCOL_RANGE_MPEG + }; + (*enc).colorspace = ff::AVColorSpace::AVCOL_SPC_BT709; + (*enc).color_primaries = ff::AVColorPrimaries::AVCOL_PRI_BT709; + (*enc).color_trc = ff::AVColorTransferCharacteristic::AVCOL_TRC_BT709; let frames_ref = ff::av_hwframe_ctx_alloc(hw_device); { @@ -140,7 +179,9 @@ impl ZeroCopyEncoder { let stream_tb = (*stream).time_base; Ok(Self { - drm, + device, + queue, + adapter, renderer, hw_device, frames_ref, @@ -159,10 +200,10 @@ impl ZeroCopyEncoder { /// The wgpu device frames must be rendered on (so the RGBA texture is importable). pub fn device(&self) -> &wgpu::Device { - &self.drm.device + &self.device } pub fn queue(&self) -> &wgpu::Queue { - &self.drm.queue + &self.queue } /// Render `rgba` (an `Rgba8Unorm` texture on [`Self::device`], `TEXTURE_BINDING`) @@ -201,8 +242,9 @@ impl ZeroCopyEncoder { y_pitch: y.pitch as u64, uv_offset: uv.offset as u64, uv_pitch: uv.pitch as u64, + ten_bit: false, }; - let imported = match dmabuf::import_raw(&self.drm, &buf) { + let imported = match dmabuf::import_raw(&self.device, &self.adapter, &buf) { Ok(i) => i, Err(e) => { ff::av_frame_free(&mut (drm_f as *mut _)); @@ -219,10 +261,10 @@ impl ZeroCopyEncoder { let rgba_view = rgba.create_view(&Default::default()); let y_view = imp.y().create_view(&Default::default()); let uv_view = imp.uv().create_view(&Default::default()); - let mut cmd = self.drm.device.create_command_encoder(&Default::default()); - self.renderer.convert(&self.drm.device, &mut cmd, &rgba_view, &y_view, &uv_view); - self.drm.queue.submit(Some(cmd.finish())); - let _ = self.drm.device.poll(wgpu::PollType::wait_indefinitely()); + let mut cmd = self.device.create_command_encoder(&Default::default()); + self.renderer.convert(&self.device, &mut cmd, &rgba_view, &y_view, &uv_view); + self.queue.submit(Some(cmd.finish())); + let _ = self.device.poll(wgpu::PollType::wait_indefinitely()); // Encode the surface. (*surf).pts = self.pts; diff --git a/lightningbeam-ui/gpu-video-encoder/src/lib.rs b/lightningbeam-ui/gpu-video-encoder/src/lib.rs index 0d76fe4..b1c5c3b 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/lib.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/lib.rs @@ -30,6 +30,10 @@ pub mod dmabuf; #[cfg(target_os = "linux")] pub mod encoder; +/// VAAPI hardware decode → wgpu textures (Linux). +#[cfg(target_os = "linux")] +pub mod decoder; + #[cfg(test)] mod probe_tests { /// Confirm a headless GPU adapter is reachable (Vulkan on Linux/Intel). This gates diff --git a/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs b/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs index 7e3ccab..025435a 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/render_nv12.rs @@ -2,29 +2,61 @@ //! surface's plane textures (R8 Y, RG8 UV). Render targets (not compute storage) so it //! works with the DMA-BUF-imported plane images, which aren't storage-writable. //! -//! BT.709 full-range, matching `nv12::cpu_reference` and the encoder's color tags. +//! BT.709 matrix; the Y/chroma scale+offset are chosen by `full_range` — limited (TV, 16–235) +//! is the H.264 convention most players assume, full (PC, 0–255) needs the matching color tag. /// Converts a bound RGBA texture into a Y plane (R8) and a UV plane (RG8) via two passes. pub struct Rgba2Nv12 { y_pipeline: wgpu::RenderPipeline, uv_pipeline: wgpu::RenderPipeline, bgl: wgpu::BindGroupLayout, + params_buf: wgpu::Buffer, } impl Rgba2Nv12 { - pub fn new(device: &wgpu::Device) -> Self { + /// `full_range`: true → full/PC range (Y 0–255, no scaling); false → limited/TV range + /// (Y 16–235, chroma 16–240), the H.264 default. The encoder must tag the stream to match. + pub fn new(device: &wgpu::Device, full_range: bool) -> Self { + // (y_offset, y_scale, chroma_offset, chroma_scale). The shader applies these to the + // BT.709 luma/chroma dot products. + let params: [f32; 4] = if full_range { + [0.0, 1.0, 128.0 / 255.0, 1.0] + } else { + [16.0 / 255.0, 219.0 / 255.0, 128.0 / 255.0, 224.0 / 255.0] + }; + let params_bytes: Vec = params.iter().flat_map(|f| f.to_le_bytes()).collect(); + let params_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("rgba2nv12_params"), + size: params_bytes.len() as u64, + usage: wgpu::BufferUsages::UNIFORM, + mapped_at_creation: true, + }); + params_buf.slice(..).get_mapped_range_mut().copy_from_slice(¶ms_bytes); + params_buf.unmap(); let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("rgba2nv12_bgl"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - sample_type: wgpu::TextureSampleType::Float { filterable: false }, - view_dimension: wgpu::TextureViewDimension::D2, - multisampled: false, + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, }, - count: None, - }], + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], }); let layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("rgba2nv12_pl"), @@ -65,6 +97,7 @@ impl Rgba2Nv12 { y_pipeline: mk("y_fs", wgpu::TextureFormat::R8Unorm), uv_pipeline: mk("uv_fs", wgpu::TextureFormat::Rg8Unorm), bgl, + params_buf, } } @@ -80,10 +113,16 @@ impl Rgba2Nv12 { let bg = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("rgba2nv12_bg"), layout: &self.bgl, - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(rgba_view), - }], + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(rgba_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: self.params_buf.as_entire_binding(), + }, + ], }); for (pipeline, view) in [(&self.y_pipeline, y_view), (&self.uv_pipeline, uv_view)] { let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { @@ -110,6 +149,8 @@ impl Rgba2Nv12 { const SHADER: &str = r#" @group(0) @binding(0) var input_rgba: texture_2d; +// (y_offset, y_scale, chroma_offset, chroma_scale) — selects limited vs full range. +@group(0) @binding(1) var params: vec4; // Fullscreen triangle. @vertex @@ -127,7 +168,8 @@ fn load(p: vec2) -> vec3 { @fragment fn y_fs(@builtin(position) pos: vec4) -> @location(0) vec4 { let c = load(vec2(i32(pos.x), i32(pos.y))); - let y = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b; + let luma = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b; + let y = params.x + params.y * luma; return vec4(y, 0.0, 0.0, 1.0); } @@ -138,8 +180,10 @@ fn uv_fs(@builtin(position) pos: vec4) -> @location(0) vec4 { let sy = 2 * i32(pos.y); let a = (load(vec2(sx, sy)) + load(vec2(sx + 1, sy)) + load(vec2(sx, sy + 1)) + load(vec2(sx + 1, sy + 1))) * 0.25; - let u = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b + 0.5; - let v = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b + 0.5; + let cb = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b; + let cr = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b; + let u = params.z + params.w * cb; + let v = params.z + params.w * cr; return vec4(u, v, 0.0, 1.0); } "#; diff --git a/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs b/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs index f7502e0..5eb0da2 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/vaapi.rs @@ -157,10 +157,18 @@ impl MappedSurface { let desc = (*drm).data[0] as *const ff::AVDRMFrameDescriptor; // Expect 1 object, 2 layers (Y=R8, UV=GR88). if (*desc).nb_objects != 1 || (*desc).nb_layers != 2 { - return Err(format!( + let msg = format!( "unexpected DRM layout: {} objects, {} layers", (*desc).nb_objects, (*desc).nb_layers - )); + ); + // Free everything mapped/allocated above (this path was leaking the device, + // frames context, and both AVFrames on every odd-layout surface). + ff::av_frame_free(&mut (drm as *mut _)); + ff::av_frame_free(&mut (surf as *mut _)); + let mut fr = frames_ref; + ff::av_buffer_unref(&mut fr); + ff::av_buffer_unref(&mut hw_device); + return Err(msg); } let obj = &(*desc).objects[0]; let y = &(*desc).layers[0].planes[0]; diff --git a/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs b/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs index 889b57d..fcff428 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/vk_device.rs @@ -22,12 +22,20 @@ pub struct DrmDevice { pub raw_instance: ash::Instance, } -/// Create the device, or `Err` if Vulkan/the extension isn't available (caller falls back). +/// Create a headless DMA-BUF-import device (encoder/decoder), or `Err` if Vulkan/the extension +/// isn't available (caller falls back). pub fn create() -> Result { - unsafe { create_inner() } + unsafe { create_inner(false) } } -unsafe fn create_inner() -> Result { +/// Like [`create`] but also enables `VK_KHR_swapchain` so the device can present to a window — +/// for use as the editor's **shared** wgpu device (eframe + compositor + decode + encode all on +/// one device, so hardware-decoded DMA-BUF textures are usable by the preview compositor). +pub fn create_windowed() -> Result { + unsafe { create_inner(true) } +} + +unsafe fn create_inner(windowed: bool) -> Result { use wgpu_hal::vulkan::Api as Vk; // Bring the HAL Instance trait into scope for `init` / `enumerate_adapters`. use wgpu_hal::Instance as _; @@ -72,14 +80,19 @@ unsafe fn create_inner() -> Result { exposed.adapter.required_device_extensions(exposed.features); // Only the genuine extensions; external_memory / bind_memory2 / ycbcr / format_list // are core in Vulkan 1.1+ (this device is 1.3) so they need no enabling. - let extra: &[&'static CStr] = &[ + let mut extra: Vec<&'static CStr> = vec![ ash::ext::image_drm_format_modifier::NAME, ash::khr::external_memory_fd::NAME, ash::ext::external_memory_dma_buf::NAME, ash::ext::queue_family_foreign::NAME, ]; + // Presentation (windowed shared device only): the WSI surface instance extensions are already + // enabled by `Instance::init`; the device needs the swapchain extension to present. + if windowed { + extra.push(ash::khr::swapchain::NAME); + } for e in extra { - if !ext_names.contains(e) { + if !ext_names.contains(&e) { ext_names.push(e); } } @@ -130,7 +143,10 @@ unsafe fn create_inner() -> Result { open_device, &wgpu::DeviceDescriptor { label: Some("drm-import-device"), - required_features: wgpu::Features::empty(), + // R16/Rg16 plane textures for P010 (10-bit HDR) import need this; request it only + // when the adapter supports it (else 10-bit falls back to software decode). + required_features: wgpu_adapter.features() + & wgpu::Features::TEXTURE_FORMAT_16BIT_NORM, // Vello's compute pipelines need more than downlevel limits (e.g. // max_storage_buffers_per_shader_stage >= 5). This device only ever runs on a // real VAAPI-capable GPU, so request the adapter's full limits. diff --git a/lightningbeam-ui/gpu-video-encoder/tests/decode_roundtrip.rs b/lightningbeam-ui/gpu-video-encoder/tests/decode_roundtrip.rs new file mode 100644 index 0000000..81f9dc8 --- /dev/null +++ b/lightningbeam-ui/gpu-video-encoder/tests/decode_roundtrip.rs @@ -0,0 +1,91 @@ +//! Round-trip: encode solid frames with the zero-copy encoder, then hardware-decode them back +//! into a wgpu texture and read the Y plane. Verifies the VAAPI decode → DMA-BUF → wgpu import +//! path produces real pixels on the GPU. Skips when VAAPI is unavailable. + +#![cfg(target_os = "linux")] + +use gpu_video_encoder::decoder::VaapiDecoder; +use gpu_video_encoder::encoder::ZeroCopyEncoder; + +#[test] +fn vaapi_decode_roundtrip() { + // 256-wide so the R8 Y readback row (256 B) is already 256-aligned. + let (w, h) = (256u32, 256u32); + let out = std::env::temp_dir().join("gpu_video_encoder_decode_rt.mp4"); + let _ = std::fs::remove_file(&out); + + // --- Encode 10 frames of solid mid-gray. Full range → Y == luma ≈ 128. --- + { + let mut enc = match ZeroCopyEncoder::new(w, h, 30, 4000, &out, true) { + Ok(e) => e, + Err(e) => { + eprintln!("[decode-rt] encode unavailable, skipping: {e}"); + return; + } + }; + let device = enc.device(); + let src = device.create_texture(&wgpu::TextureDescriptor { + label: Some("gray"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let gray = vec![128u8; (w * h * 4) as usize]; + enc.queue().write_texture( + wgpu::TexelCopyTextureInfo { texture: &src, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + &gray, + wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w * 4), rows_per_image: Some(h) }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + for _ in 0..10 { + enc.encode_rgba(&src).expect("encode_rgba"); + } + enc.finish().expect("finish"); + } + + // --- Decode it back on the GPU. --- + let mut dec = match VaapiDecoder::new(&out) { + Ok(d) => d, + Err(e) => { + eprintln!("[decode-rt] decode unavailable, skipping: {e}"); + return; + } + }; + let frame = dec.next_frame().expect("next_frame").expect("expected at least one frame"); + assert_eq!(frame.y().width(), w, "decoded Y width"); + assert_eq!(frame.y().height(), h, "decoded Y height"); + + // Read back the Y plane (R8) and check it's ≈ the gray we encoded. + let device = dec.device(); + let buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("y_readback"), + size: (w * h) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mut cmd = device.create_command_encoder(&Default::default()); + cmd.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { texture: frame.y(), mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyBufferInfo { + buffer: &buf, + layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(w), rows_per_image: Some(h) }, + }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + dec.queue().submit(Some(cmd.finish())); + buf.slice(..).map_async(wgpu::MapMode::Read, |_| {}); + let _ = device.poll(wgpu::PollType::wait_indefinitely()); + + let data = buf.slice(..).get_mapped_range(); + let mean = data.iter().map(|&b| b as f64).sum::() / data.len() as f64; + eprintln!("[decode-rt] decoded {w}x{h}, mean Y = {mean:.1}"); + assert!( + (mean - 128.0).abs() < 12.0, + "mean Y {mean} not ≈ 128 — decode produced wrong pixels" + ); + eprintln!("[decode-rt] ✅ VAAPI decode → wgpu texture verified"); +} diff --git a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs index dd2152c..2234091 100644 --- a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs +++ b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy.rs @@ -54,7 +54,7 @@ fn zerocopy_real_frame_render() { wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, ); - let conv = render_nv12::Rgba2Nv12::new(&drm.device); + let conv = render_nv12::Rgba2Nv12::new(&drm.device, true); let src_view = src.create_view(&Default::default()); let y_view = imported.y().create_view(&Default::default()); let uv_view = imported.uv().create_view(&Default::default()); diff --git a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs index 69e8ca9..749bca8 100644 --- a/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs +++ b/lightningbeam-ui/gpu-video-encoder/tests/zerocopy_encode.rs @@ -10,7 +10,7 @@ fn zerocopy_encode_h264() { let (w, h) = (640u32, 480u32); let out = std::env::temp_dir().join("gpu_video_encoder_zerocopy.mp4"); let _ = std::fs::remove_file(&out); - let mut enc = match ZeroCopyEncoder::new(w, h, 30, 4000, &out) { + let mut enc = match ZeroCopyEncoder::new(w, h, 30, 4000, &out, false) { Ok(e) => e, Err(e) => { eprintln!("[zc-encode] unavailable, skipping: {e}"); diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs b/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs index ed209cf..29db4f5 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/mod.rs @@ -40,6 +40,7 @@ pub mod raster_diff; pub mod raster_stroke; pub mod raster_fill; pub mod add_raster_keyframe; +pub mod resize_raster_layer; pub mod move_layer; pub mod set_fill_paint; pub mod set_image_fill; @@ -77,6 +78,7 @@ pub use group_layers::GroupLayersAction; pub use raster_stroke::RasterStrokeAction; pub use raster_fill::RasterFillAction; pub use add_raster_keyframe::AddRasterKeyframeAction; +pub use resize_raster_layer::ResizeRasterLayerAction; pub use move_layer::MoveLayerAction; pub use set_fill_paint::SetFillPaintAction; pub use set_image_fill::SetImageFillAction; diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs b/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs index 15a5f16..1911750 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs @@ -98,11 +98,16 @@ impl RasterDiff { self.before_region.len() + self.after_region.len() } - /// Restore the pre-edit pixels into `raw` (undo / first-execute rollback). - pub fn apply_before(&self, raw: &mut Vec) { + /// Restore the pre-edit pixels into `raw` (undo / first-execute rollback). `cur_w`/`cur_h` + /// are the keyframe's current dimensions; if they differ from the diff's captured size the + /// diff predates a resize and is skipped rather than applied at mismatched dims. + pub fn apply_before(&self, raw: &mut Vec, cur_w: u32, cur_h: u32) { if self.bbox.is_none() { return; // no change } + if cur_w != self.full_width || cur_h != self.full_height { + return; // diff predates a keyframe resize + } if self.before_blank { // The frame was blank before this edit (it was the first stroke); undoing // it returns to blank regardless of the current buffer. @@ -112,11 +117,16 @@ impl RasterDiff { self.stamp_resident(&self.before_region, raw); } - /// Apply the post-edit pixels into `raw` (commit / redo). - pub fn apply_after(&self, raw: &mut Vec) { + /// Apply the post-edit pixels into `raw` (commit / redo). `cur_w`/`cur_h` are the keyframe's + /// current dimensions; a mismatch with the captured size means the diff predates a resize, so + /// it's skipped rather than rebuilding the buffer at stale dimensions. + pub fn apply_after(&self, raw: &mut Vec, cur_w: u32, cur_h: u32) { if self.bbox.is_none() { return; // no change } + if cur_w != self.full_width || cur_h != self.full_height { + return; // diff predates a keyframe resize + } if self.before_blank { // Base was blank: build a full transparent buffer then stamp the bbox. The // commit/redo path frequently starts from empty `raw_pixels` here. @@ -175,9 +185,9 @@ mod tests { assert_eq!(diff.bbox, Some((3, 2, 2, 2))); let mut buf = after.clone(); - diff.apply_before(&mut buf); + diff.apply_before(&mut buf, w, h); assert_eq!(buf, before, "undo must reproduce the pre-edit buffer exactly"); - diff.apply_after(&mut buf); + diff.apply_after(&mut buf, w, h); assert_eq!(buf, after, "redo must reproduce the post-edit buffer exactly"); } @@ -195,15 +205,15 @@ mod tests { // First execute / redo from EMPTY raw_pixels (the real commit path): builds // the full buffer from transparent + the stroke. let mut buf: Vec = Vec::new(); - diff.apply_after(&mut buf); + diff.apply_after(&mut buf, w, h); assert_eq!(buf, after, "commit/redo must build the frame from a blank base"); // Undo the first stroke → back to blank (empty). - diff.apply_before(&mut buf); + diff.apply_before(&mut buf, w, h); assert!(buf.is_empty(), "undoing the first stroke restores the blank keyframe"); // Redo again from the now-empty buffer. - diff.apply_after(&mut buf); + diff.apply_after(&mut buf, w, h); assert_eq!(buf, after); } @@ -215,7 +225,7 @@ mod tests { assert_eq!(diff.bbox, None); assert_eq!(diff.byte_size(), 0); let mut b = buf.clone(); - diff.apply_before(&mut b); + diff.apply_before(&mut b, w, h); assert_eq!(b, buf); } @@ -226,7 +236,7 @@ mod tests { let after = solid(w, h, [1, 2, 3, 255]); let diff = RasterDiff::compute(&before, &after, w, h); let mut empty: Vec = Vec::new(); - diff.apply_before(&mut empty); // base not resident + diff.apply_before(&mut empty, w, h); // base not resident assert!(empty.is_empty(), "must not resize/corrupt a non-resident base"); } } diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs b/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs index 92c0dd0..3693d0d 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs @@ -53,7 +53,7 @@ impl Action for RasterFillAction { if let Some(full) = self.full_after.take() { kf.raw_pixels = full; } else { - self.diff.apply_after(&mut kf.raw_pixels); + self.diff.apply_after(&mut kf.raw_pixels, kf.width, kf.height); } kf.texture_dirty = true; kf.dirty = true; @@ -71,7 +71,7 @@ impl Action for RasterFillAction { let kf = raster .keyframe_at_mut(self.time) .ok_or_else(|| format!("No raster keyframe at/before t={}", self.time))?; - self.diff.apply_before(&mut kf.raw_pixels); + self.diff.apply_before(&mut kf.raw_pixels, kf.width, kf.height); kf.texture_dirty = true; kf.dirty = true; Ok(()) diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs b/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs index 06517b9..392fcb0 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs @@ -61,7 +61,7 @@ impl Action for RasterStrokeAction { kf.raw_pixels = full; } else { // Redo: replay via the diff onto the (resident) base. - self.diff.apply_after(&mut kf.raw_pixels); + self.diff.apply_after(&mut kf.raw_pixels, kf.width, kf.height); } kf.texture_dirty = true; kf.dirty = true; @@ -70,7 +70,7 @@ impl Action for RasterStrokeAction { fn rollback(&mut self, document: &mut Document) -> Result<(), String> { let kf = get_keyframe_mut(document, &self.layer_id, self.time, self.width, self.height)?; - self.diff.apply_before(&mut kf.raw_pixels); + self.diff.apply_before(&mut kf.raw_pixels, kf.width, kf.height); kf.texture_dirty = true; kf.dirty = true; Ok(()) diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/resize_raster_layer.rs b/lightningbeam-ui/lightningbeam-core/src/actions/resize_raster_layer.rs new file mode 100644 index 0000000..7fa5420 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-core/src/actions/resize_raster_layer.rs @@ -0,0 +1,90 @@ +//! Resize every keyframe of a raster layer to a new canvas size (Scale or Canvas mode), with undo. +//! +//! Used by the info panel's "Layer to document size" button. Pixels must be resident (the editor +//! faults them in first) so the resample/copy is exact and a later page-in won't mismatch the size. + +use crate::action::Action; +use crate::document::Document; +use crate::layer::AnyLayer; +use crate::raster_layer::RasterResizeMode; +use crate::raster_store::RasterStore; +use uuid::Uuid; + +/// Per-keyframe state captured for undo. +struct OldKeyframe { + id: Uuid, + width: u32, + height: u32, + pixels: Vec, +} + +pub struct ResizeRasterLayerAction { + layer_id: Uuid, + new_w: u32, + new_h: u32, + mode: RasterResizeMode, + /// Read-only page-in for keyframes whose pixels aren't resident. (No incremental write exists, so + /// resized keyframes stay resident + dirty and persist on the next full save.) + store: RasterStore, + /// Captured on first execute for rollback. + old: Option>, +} + +impl ResizeRasterLayerAction { + pub fn new(layer_id: Uuid, new_w: u32, new_h: u32, mode: RasterResizeMode, store: RasterStore) -> Self { + Self { layer_id, new_w, new_h, mode, store, old: None } + } +} + +impl Action for ResizeRasterLayerAction { + fn execute(&mut self, document: &mut Document) -> Result<(), String> { + let Some(AnyLayer::Raster(rl)) = document.get_layer_mut(&self.layer_id) else { + return Err("ResizeRasterLayerAction: layer is not a raster layer".into()); + }; + let capture = self.old.is_none(); + let mut old = Vec::new(); + for kf in rl.keyframes.iter_mut() { + // Page the keyframe in one at a time so we never hold the whole layer in memory at once. + if kf.raw_pixels.is_empty() { + if let Some(px) = self.store.load_pixels(kf.id) { + kf.raw_pixels = px; + kf.needs_fault_in = false; + } + } + if capture { + old.push(OldKeyframe { id: kf.id, width: kf.width, height: kf.height, pixels: kf.raw_pixels.clone() }); + } + kf.resize_to(self.new_w, self.new_h, self.mode); + } + if capture { + self.old = Some(old); + } + Ok(()) + } + + fn rollback(&mut self, document: &mut Document) -> Result<(), String> { + let Some(AnyLayer::Raster(rl)) = document.get_layer_mut(&self.layer_id) else { + return Err("ResizeRasterLayerAction: layer is not a raster layer".into()); + }; + if let Some(old) = &self.old { + for o in old { + if let Some(kf) = rl.keyframes.iter_mut().find(|kf| kf.id == o.id) { + kf.width = o.width; + kf.height = o.height; + kf.raw_pixels = o.pixels.clone(); + kf.proxy = None; + kf.texture_dirty = true; + kf.dirty = true; + } + } + } + Ok(()) + } + + fn description(&self) -> String { + match self.mode { + RasterResizeMode::Scale => "Scale raster layer".into(), + RasterResizeMode::Canvas => "Resize raster canvas".into(), + } + } +} diff --git a/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs b/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs index 48c5e38..fc038b3 100644 --- a/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs +++ b/lightningbeam-ui/lightningbeam-core/src/actions/set_document_properties.rs @@ -4,7 +4,7 @@ //! with undo/redo support. use crate::action::Action; -use crate::document::Document; +use crate::document::{Document, HdrOutputMode}; use crate::shape::ShapeColor; /// Individual property change for a document @@ -15,6 +15,7 @@ pub enum DocumentPropertyChange { Duration(f64), Framerate(f64), BackgroundColor(ShapeColor), + HdrOutputMode(HdrOutputMode), } /// Stored old value for undo (either f64 or color) @@ -22,6 +23,7 @@ pub enum DocumentPropertyChange { enum OldValue { F64(f64), Color(ShapeColor), + Hdr(HdrOutputMode), } /// Action that sets a property on the document @@ -72,6 +74,14 @@ impl SetDocumentPropertiesAction { old_value: None, } } + + /// Create a new action to set the HDR→SDR output mode + pub fn set_hdr_output_mode(mode: HdrOutputMode) -> Self { + Self { + property: DocumentPropertyChange::HdrOutputMode(mode), + old_value: None, + } + } } impl Action for SetDocumentPropertiesAction { @@ -83,6 +93,7 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(_) => OldValue::F64(document.duration), DocumentPropertyChange::Framerate(_) => OldValue::F64(document.framerate), DocumentPropertyChange::BackgroundColor(_) => OldValue::Color(document.background_color), + DocumentPropertyChange::HdrOutputMode(_) => OldValue::Hdr(document.hdr_output_mode), }); } @@ -92,6 +103,7 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(v) => document.duration = *v, DocumentPropertyChange::Framerate(v) => document.framerate = *v, DocumentPropertyChange::BackgroundColor(c) => document.background_color = *c, + DocumentPropertyChange::HdrOutputMode(m) => document.hdr_output_mode = *m, } Ok(()) } @@ -106,11 +118,15 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(_) => document.duration = v, DocumentPropertyChange::Framerate(_) => document.framerate = v, DocumentPropertyChange::BackgroundColor(_) => {} + DocumentPropertyChange::HdrOutputMode(_) => {} } } Some(OldValue::Color(c)) => { document.background_color = *c; } + Some(OldValue::Hdr(m)) => { + document.hdr_output_mode = *m; + } None => {} } Ok(()) @@ -123,6 +139,7 @@ impl Action for SetDocumentPropertiesAction { DocumentPropertyChange::Duration(_) => "duration", DocumentPropertyChange::Framerate(_) => "framerate", DocumentPropertyChange::BackgroundColor(_) => "background color", + DocumentPropertyChange::HdrOutputMode(_) => "HDR output mode", }; format!("Set {}", property_name) } diff --git a/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs index 39ef088..770b544 100644 --- a/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs +++ b/lightningbeam-ui/lightningbeam-core/src/beam_archive.rs @@ -374,7 +374,11 @@ impl BeamArchive { let rows = stmt .query_map([&id_bytes], |r| r.get::<_, Vec>(0)) .map_err(map_sql)?; - let mut out = Vec::with_capacity(info.total_len as usize); + // `total_len` is an untrusted DB field; cap the preallocation so a corrupt/oversized + // value can't trigger a multi-GB eager allocation (or, on 32-bit, a truncated `as usize`). + // The Vec still grows to fit the actual chunk bytes regardless. + const PREALLOC_CAP: u64 = 64 * 1024 * 1024; + let mut out = Vec::with_capacity(info.total_len.min(PREALLOC_CAP) as usize); for row in rows { out.extend_from_slice(&row.map_err(map_sql)?); } diff --git a/lightningbeam-ui/lightningbeam-core/src/document.rs b/lightningbeam-ui/lightningbeam-core/src/document.rs index 9fd4168..62eba1c 100644 --- a/lightningbeam-ui/lightningbeam-core/src/document.rs +++ b/lightningbeam-ui/lightningbeam-core/src/document.rs @@ -145,6 +145,26 @@ pub enum TimelineMode { Frames, } +/// How super-white (HDR) values are mapped to the SDR display/export output at the final +/// linear→sRGB encode. SDR content sits in [0,1] and is unaffected by either mode. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum HdrOutputMode { + /// Hard-clip values above 1.0 (the historical behaviour). SDR-exact; HDR highlights blow out. + #[default] + Clip, + /// Roll highlights above a knee smoothly toward 1.0 to recover detail. Slightly dims near-white. + HighlightRolloff, +} + +impl HdrOutputMode { + pub fn name(&self) -> &'static str { + match self { + HdrOutputMode::Clip => "Clip", + HdrOutputMode::HighlightRolloff => "Highlight rolloff", + } + } +} + /// Asset category for folder tree access #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AssetCategory { @@ -244,6 +264,10 @@ pub struct Document { #[serde(default)] pub timeline_mode: TimelineMode, + /// How HDR (super-white) values are mapped to the SDR output at the final encode. + #[serde(default)] + pub hdr_output_mode: HdrOutputMode, + /// Current UI layout state (serialized for save/load) #[serde(default, skip_serializing_if = "Option::is_none")] pub ui_layout: Option, @@ -278,6 +302,7 @@ impl Default for Document { ml }, duration: 10.0, + hdr_output_mode: HdrOutputMode::default(), root: GraphicsObject::default(), vector_clips: HashMap::new(), video_clips: HashMap::new(), diff --git a/lightningbeam-ui/lightningbeam-core/src/export.rs b/lightningbeam-ui/lightningbeam-core/src/export.rs index 4bef658..6e62d50 100644 --- a/lightningbeam-ui/lightningbeam-core/src/export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/export.rs @@ -265,6 +265,84 @@ impl VideoCodec { } } +/// YUV color range for the encoded video (currently H.264). Limited/TV (16–235) is what nearly +/// every player assumes; Full/PC (0–255) keeps a bit more precision but only looks right in players +/// that honor the full-range tag — so Limited is the safe default. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ColorRange { + Limited, + Full, +} + +impl Default for ColorRange { + fn default() -> Self { ColorRange::Limited } +} + +impl ColorRange { + pub fn is_full(&self) -> bool { matches!(self, ColorRange::Full) } + pub fn name(&self) -> &'static str { + match self { + ColorRange::Limited => "Limited (TV, 16–235)", + ColorRange::Full => "Full (PC, 0–255)", + } + } +} + +/// HDR output mode for video export. SDR encodes BT.709 8-bit as before; the HDR modes encode +/// 10-bit BT.2020 with the PQ (HDR10) or HLG transfer, preserving super-white highlights from the +/// linear compositor. HDR requires a 10-bit codec (HEVC Main10) — the exporter forces H.265. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum HdrExportMode { + #[default] + Sdr, + /// PQ (SMPTE ST 2084) — HDR10. Graphics white at 203 nits (matches the compositor convention). + Pq, + /// HLG (ARIB STD-B67) — broadcast HDR, also displayable as SDR. + Hlg, +} + +impl HdrExportMode { + pub fn is_hdr(&self) -> bool { !matches!(self, HdrExportMode::Sdr) } + pub fn name(&self) -> &'static str { + match self { + HdrExportMode::Sdr => "SDR (BT.709, 8-bit)", + HdrExportMode::Pq => "HDR10 / PQ (BT.2020, 10-bit)", + HdrExportMode::Hlg => "HLG (BT.2020, 10-bit)", + } + } + /// FFmpeg transfer-characteristic name for the color tags. + pub fn transfer_name(&self) -> &'static str { + match self { + HdrExportMode::Sdr => "bt709", + HdrExportMode::Pq => "smpte2084", + HdrExportMode::Hlg => "arib-std-b67", + } + } +} + +/// How the document is fit into the export frame when the export resolution's aspect ratio differs +/// from the document's. Applied as the export `base_transform` (document space → export pixels). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum ExportFitMode { + /// Scale each axis independently to fill the frame — distorts when aspects differ. + Stretch, + /// Scale uniformly to fit, centered, with black bars (letterbox/pillarbox). Preserves aspect. + #[default] + Letterbox, + /// Scale uniformly to fill, centered, cropping the overflow. Preserves aspect, no bars. + Crop, +} + +impl ExportFitMode { + pub fn name(&self) -> &'static str { + match self { + ExportFitMode::Stretch => "Stretch (distort to fill)", + ExportFitMode::Letterbox => "Letterbox (fit, black bars)", + ExportFitMode::Crop => "Crop (fill, trim edges)", + } + } +} + /// Video quality presets #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum VideoQuality { @@ -322,6 +400,18 @@ pub struct VideoExportSettings { /// Video quality pub quality: VideoQuality, + /// YUV color range (H.264 only; ignored by other codecs). + #[serde(default)] + pub color_range: ColorRange, + + /// HDR output mode. HDR forces 10-bit HEVC (BT.2020 + PQ/HLG); SDR is the default. + #[serde(default)] + pub hdr: HdrExportMode, + + /// How the document is fit into the export frame when aspect ratios differ (default Letterbox). + #[serde(default)] + pub fit: ExportFitMode, + /// Audio settings (None = no audio) pub audio: Option, @@ -340,6 +430,9 @@ impl Default for VideoExportSettings { height: None, framerate: 60.0, quality: VideoQuality::High, + color_range: ColorRange::Limited, + hdr: HdrExportMode::Sdr, + fit: ExportFitMode::Letterbox, audio: Some(AudioExportSettings::high_quality_aac()), start_time: 0.0, end_time: 60.0, @@ -431,11 +524,14 @@ pub struct ImageExportSettings { /// When false, the image is composited onto an opaque background before encoding. /// Only meaningful for formats that support alpha (PNG, WebP). pub allow_transparency: bool, + /// How the document is fit into the output frame when aspect ratios differ (default Letterbox). + #[serde(default)] + pub fit: ExportFitMode, } impl Default for ImageExportSettings { fn default() -> Self { - Self { format: ImageFormat::Png, time: 0.0, width: None, height: None, quality: 90, allow_transparency: false } + Self { format: ImageFormat::Png, time: 0.0, width: None, height: None, quality: 90, allow_transparency: false, fit: ExportFitMode::Letterbox } } } diff --git a/lightningbeam-ui/lightningbeam-core/src/file_types.rs b/lightningbeam-ui/lightningbeam-core/src/file_types.rs index 464ff36..01b837c 100644 --- a/lightningbeam-ui/lightningbeam-core/src/file_types.rs +++ b/lightningbeam-ui/lightningbeam-core/src/file_types.rs @@ -15,7 +15,9 @@ pub const VIDEO_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv", "webm", "m4v /// Supported MIDI file extensions pub const MIDI_EXTENSIONS: &[&str] = &["mid", "midi"]; -// Note: SVG import deferred to future task +/// Supported vector file extensions (imported as a new vector layer, not an asset) +pub const VECTOR_EXTENSIONS: &[&str] = &["svg"]; + // Note: .beam project files handled separately in file save/load feature /// File type categories for import routing @@ -25,6 +27,7 @@ pub enum FileType { Audio, Video, Midi, + Vector, } /// Detect file type from extension string @@ -53,6 +56,9 @@ pub fn get_file_type(extension: &str) -> Option { if MIDI_EXTENSIONS.contains(&ext.as_str()) { return Some(FileType::Midi); } + if VECTOR_EXTENSIONS.contains(&ext.as_str()) { + return Some(FileType::Vector); + } None } @@ -65,6 +71,7 @@ pub fn all_supported_extensions() -> Vec<&'static str> { all.extend_from_slice(AUDIO_EXTENSIONS); all.extend_from_slice(VIDEO_EXTENSIONS); all.extend_from_slice(MIDI_EXTENSIONS); + all.extend_from_slice(VECTOR_EXTENSIONS); all } @@ -90,7 +97,8 @@ mod tests { assert_eq!(get_file_type("midi"), Some(FileType::Midi)); assert_eq!(get_file_type("unknown"), None); - assert_eq!(get_file_type("svg"), None); // SVG deferred + assert_eq!(get_file_type("svg"), Some(FileType::Vector)); + assert_eq!(get_file_type("SVG"), Some(FileType::Vector)); } #[test] diff --git a/lightningbeam-ui/lightningbeam-core/src/object.rs b/lightningbeam-ui/lightningbeam-core/src/object.rs index 601bede..2e63008 100644 --- a/lightningbeam-ui/lightningbeam-core/src/object.rs +++ b/lightningbeam-ui/lightningbeam-core/src/object.rs @@ -47,6 +47,21 @@ impl Transform { Self::default() } + /// Set scale + position so `content_w × content_h` is fit **uniformly and centered** within + /// `doc_w × doc_h`, preserving aspect ratio (letterbox/pillarbox). The single shared way to + /// place a media clip (video/image) on the document — both import paths use this so a clip + /// looks identical however it was added. No-op for non-positive content dimensions. + pub fn fit_centered(&mut self, content_w: f64, content_h: f64, doc_w: f64, doc_h: f64) { + if content_w <= 0.0 || content_h <= 0.0 { + return; + } + let scale = (doc_w / content_w).min(doc_h / content_h); + self.scale_x = scale; + self.scale_y = scale; + self.x = (doc_w - content_w * scale) / 2.0; + self.y = (doc_h - content_h * scale) / 2.0; + } + /// Create a transform with position pub fn with_position(x: f64, y: f64) -> Self { Self { diff --git a/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs b/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs index 6a14d11..d158fd9 100644 --- a/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/raster_layer.rs @@ -169,6 +169,57 @@ impl RasterKeyframe { proxy: None, } } + + /// Change the canvas to `(new_w, new_h)`. `Scale` resamples the content (Lanczos3) to fill the + /// new size; `Canvas` keeps the content at native resolution anchored top-left, padding with + /// transparent (expand) or trimming (crop). No-op if the size is unchanged. If pixels aren't + /// resident the buffer is left empty and only the declared size changes — the caller must fault + /// pixels in first (a later load would otherwise mismatch the new size). + pub fn resize_to(&mut self, new_w: u32, new_h: u32, mode: RasterResizeMode) { + if new_w == 0 || new_h == 0 || (self.width == new_w && self.height == new_h) { + return; + } + // Resample/recanvas the buffer only when pixels are resident. A blank keyframe (no content + // and no store row) just takes the new declared size — there's nothing to corrupt. Paged-out + // keyframes are loaded by the caller (ResizeRasterLayerAction) before this runs. + if !self.raw_pixels.is_empty() { + let old = std::mem::take(&mut self.raw_pixels); + self.raw_pixels = match mode { + RasterResizeMode::Scale => match image::RgbaImage::from_raw(self.width, self.height, old) { + Some(img) => image::imageops::resize(&img, new_w, new_h, image::imageops::FilterType::Lanczos3).into_raw(), + None => vec![0u8; (new_w as usize) * (new_h as usize) * 4], + }, + RasterResizeMode::Canvas => { + // Copy the old pixels into a transparent new buffer, anchored top-left (matching + // the raster's (0,0) document anchor); right/bottom is padded or trimmed. + let mut buf = vec![0u8; (new_w as usize) * (new_h as usize) * 4]; + let copy_w = self.width.min(new_w) as usize; + let copy_h = self.height.min(new_h) as usize; + let (ow, nw) = (self.width as usize, new_w as usize); + for y in 0..copy_h { + let src = y * ow * 4; + let dst = y * nw * 4; + buf[dst..dst + copy_w * 4].copy_from_slice(&old[src..src + copy_w * 4]); + } + buf + } + }; + self.proxy = None; // invalidate any downsampled proxy + } + self.width = new_w; + self.height = new_h; + self.texture_dirty = true; + self.dirty = true; + } +} + +/// How a raster canvas resize treats existing pixels. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum RasterResizeMode { + /// Resample the content to fill the new size (changes pixel resolution). + Scale, + /// Keep content at native resolution, anchored top-left; pad/trim the canvas. + Canvas, } /// A pixel-buffer painting layer diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index 4ee00bd..63ffcb3 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -221,8 +221,11 @@ fn decode_image_brush(data: &[u8]) -> Option { /// A single decoded video frame ready for GPU upload, with its document-space transform. pub struct VideoRenderInstance { - /// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). + /// sRGB RGBA8 pixel data (straight alpha — as decoded by ffmpeg). Empty when `gpu` is `Some`. pub rgba_data: Arc>, + /// Hardware-decoded frame as GPU NV12 textures (preview path). When `Some`, the compositor + /// samples it directly (NV12→RGB) instead of uploading `rgba_data`. + pub gpu: Option, pub width: u32, pub height: u32, /// Affine transform that maps from video-pixel space to document space. @@ -366,6 +369,20 @@ pub struct CompositeRenderResult { /// and effects in the GPU compositor. /// /// Layers are returned in bottom-to-top order for compositing. +/// Decode-target resolution for video clips = the output (export/preview) resolution, derived from +/// the document→output `base_transform` scale. The decoder caps this to the source's native size, +/// so it means "decode at the size we'll actually display, never upscaling": full detail when +/// exporting above document res (instead of upscaling a document-res frame), and cheap small frames +/// for the canvas. Stable per render pass, so it doesn't thrash the decoder's scaler/cache. +fn video_decode_target(document: &Document, base_transform: Affine) -> (u32, u32) { + let c = base_transform.as_coeffs(); // [a, b, c, d, e, f] + let sx = (c[0] * c[0] + c[1] * c[1]).sqrt(); + let sy = (c[2] * c[2] + c[3] * c[3]).sqrt(); + let w = (document.width * sx).ceil().max(1.0) as u32; + let h = (document.height * sy).ceil().max(1.0) as u32; + (w, h) +} + pub fn render_document_for_compositing( document: &Document, base_transform: Affine, @@ -545,12 +562,13 @@ pub fn render_layer_isolated( let layer_opacity = layer.opacity(); let mut video_mgr = video_manager.lock().unwrap(); let mut instances = Vec::new(); + let (target_w, target_h) = video_decode_target(document, base_transform); let tempo_map = document.tempo_map(); for clip_instance in &video_layer.clip_instances { let Some(video_clip) = document.video_clips.get(&clip_instance.clip_id) else { continue }; let Some(clip_time) = clip_instance.remap_time(time, video_clip.duration, tempo_map) else { continue }; - let Some(frame) = video_mgr.get_frame(&clip_instance.clip_id, clip_time) else { continue }; + let Some(frame) = video_mgr.get_frame(&clip_instance.clip_id, clip_time, target_w, target_h) else { continue }; // Evaluate animated transform properties. let anim = &video_layer.layer.animation_data; @@ -594,6 +612,7 @@ pub fn render_layer_isolated( }; instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: frame.gpu.clone(), width: frame.width, height: frame.height, transform: base_transform * clip_transform * frame_to_clip, @@ -614,6 +633,7 @@ pub fn render_layer_isolated( * Affine::scale(scale); instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: None, // webcam frames are always CPU width: frame.width, height: frame.height, transform: cam_transform, @@ -1095,8 +1115,9 @@ fn render_video_layer( continue; // Clip instance not active at this time }; - // Get video frame from VideoManager - let Some(frame) = video_manager.get_frame(&clip_instance.clip_id, clip_time) else { + // Get video frame from VideoManager at the output (export/preview) resolution. + let (target_w, target_h) = video_decode_target(document, base_transform); + let Some(frame) = video_manager.get_frame(&clip_instance.clip_id, clip_time, target_w, target_h) else { continue; // Frame not available }; @@ -1244,6 +1265,7 @@ fn render_video_layer( if let Some(ex) = extract.as_deref_mut() { ex.instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: frame.gpu.clone(), width: frame.width, height: frame.height, transform: instance_transform * brush_transform, @@ -1298,6 +1320,7 @@ fn render_video_layer( if let Some(ex) = extract.as_deref_mut() { ex.instances.push(VideoRenderInstance { rgba_data: frame.rgba_data.clone(), + gpu: None, // webcam frames are always CPU width: frame.width, height: frame.height, transform: preview_transform, @@ -1321,7 +1344,7 @@ fn render_video_layer( /// The axis is centred on the bbox midpoint and oriented at `angle_deg` degrees /// (0 = left→right, 90 = top→bottom). The axis extends ± half the bbox diagonal /// so the gradient covers the entire shape regardless of angle. -fn gradient_bbox_endpoints(angle_deg: f32, bbox: kurbo::Rect) -> (kurbo::Point, kurbo::Point) { +pub(crate) fn gradient_bbox_endpoints(angle_deg: f32, bbox: kurbo::Rect) -> (kurbo::Point, kurbo::Point) { let cx = bbox.center().x; let cy = bbox.center().y; let dx = bbox.width(); diff --git a/lightningbeam-ui/lightningbeam-core/src/svg_export.rs b/lightningbeam-ui/lightningbeam-core/src/svg_export.rs index 527aac3..008bc4e 100644 --- a/lightningbeam-ui/lightningbeam-core/src/svg_export.rs +++ b/lightningbeam-ui/lightningbeam-core/src/svg_export.rs @@ -218,3 +218,274 @@ fn cubic_to_svg_path(curve: &CubicBez) -> String { curve.p3.x, curve.p3.y, ) } + +// =========================================================================== +// Document / VectorGraph → SVG (the current model). The functions above target +// the legacy DCEL and are kept only for the clipboard stub. +// =========================================================================== + +use crate::document::Document; +use crate::gradient::{GradientExtend, GradientType, ShapeGradient}; +use crate::layer::AnyLayer; +use crate::shape::{Cap, FillRule, Join, ShapeColor}; +use crate::vector_graph::{FillId, VectorGraph}; +use kurbo::{BezPath, PathEl, Rect, Shape}; + +/// Serialize the document's **vector** content to a standalone SVG string, at document time `time`. +/// Vector layers (and groups of them) only — raster/video/audio/effect layers are skipped (a later +/// pass can rasterize them to ``). Animation is a single static frame at `time`. +pub fn document_to_svg(document: &Document, time: f64) -> String { + let (w, h) = (document.width, document.height); + let mut defs = String::new(); + let mut body = String::new(); + let mut grad_n = 0usize; + + // Opaque background rect (skip if the document background is transparent). + let bg = document.background_color; + if bg.a > 0 { + body.push_str(&format!( + r#""#, + fill_attrs(&bg) + )); + } + + for layer in &document.root.children { + layer_to_svg(layer, time, 1.0, &mut body, &mut defs, &mut grad_n); + } + + format!( + concat!( + r#"{}{}"# + ), + w, h, w, h, defs, body + ) +} + +/// Append one layer's SVG. Recurses into groups (``); other non-vector layer types are skipped. +fn layer_to_svg(layer: &AnyLayer, time: f64, parent_opacity: f64, body: &mut String, defs: &mut String, grad_n: &mut usize) { + match layer { + AnyLayer::Vector(vl) => { + if !vl.layer.visible { + return; // hidden layers are not rendered, so don't export them + } + let opacity = parent_opacity * vl.layer.opacity; + if let Some(graph) = vl.tweened_graph_at(time) { + let wrap = opacity < 0.999; + if wrap { + body.push_str(&format!(r#""#)); + } + vector_graph_to_svg(&graph, body, defs, grad_n); + if wrap { + body.push_str(""); + } + } + // NOTE: placed clip instances (nested clips with their own transform) are not yet + // exported — a refinement once loose-geometry export is verified. + } + AnyLayer::Group(g) => { + if !g.layer.visible { + return; + } + // Render children first; only emit the wrapper if it has exportable content + // (avoids empty groups when every child is a non-vector/hidden layer). + let mut inner = String::new(); + for child in &g.children { + layer_to_svg(child, time, 1.0, &mut inner, defs, grad_n); + } + if !inner.is_empty() { + let opacity = parent_opacity * g.layer.opacity; + body.push_str(&format!(r#""#)); + body.push_str(&inner); + body.push_str(""); + } + } + // Raster/Video/Audio/Effect have no lossless vector representation — skipped this pass. + _ => {} + } +} + +/// Emit a vector graph's fills (``) and stroked edges (``) into `body`, +/// accumulating any gradients into `defs`. Geometry is in document space (no per-layer transform). +fn vector_graph_to_svg(graph: &VectorGraph, body: &mut String, defs: &mut String, grad_n: &mut usize) { + // Fills first (drawn under strokes, matching the renderer). + for (i, fill) in graph.fills.iter().enumerate() { + if fill.deleted { + continue; + } + let path = graph.fill_to_bezpath(FillId(i as u32)); + let d = bezpath_to_d(&path); + if d.is_empty() { + continue; + } + let rule = match fill.fill_rule { + FillRule::NonZero => "nonzero", + FillRule::EvenOdd => "evenodd", + }; + + if let Some(grad) = &fill.gradient_fill { + let id = format!("grad{}", *grad_n); + *grad_n += 1; + defs.push_str(&gradient_to_svg(grad, &id, path.bounding_box())); + body.push_str(&format!(r#""#)); + } else if fill.image_fill.is_some() { + // Image fills need / + asset embedding — skipped this (vector-only) pass. + continue; + } else if let Some(c) = &fill.color { + body.push_str(&format!(r#""#, fill_attrs(c))); + } + } + + // Strokes: one per stroked edge (each edge may carry its own style). + for edge in &graph.edges { + if edge.deleted { + continue; + } + if let (Some(style), Some(color)) = (&edge.stroke_style, &edge.stroke_color) { + let d = cubic_to_svg_path(&edge.curve); + body.push_str(&format!( + r#""#, + stroke_attrs(color), style.width, cap_str(style.cap), join_str(style.join), style.miter_limit + )); + } + } +} + +/// `` / `` definition matching the renderer's start/end semantics. +fn gradient_to_svg(grad: &ShapeGradient, id: &str, bbox: Rect) -> String { + use kurbo::Point; + // Mirror renderer.rs: explicit world endpoints if present (radial reflects the edge through the + // center so midpoint(start,end) == center), else derive from angle + bbox. + let (start, end) = match (grad.start_world, grad.end_world) { + (Some((sx, sy)), Some((ex, ey))) => match grad.kind { + GradientType::Linear => (Point::new(sx, sy), Point::new(ex, ey)), + GradientType::Radial => (Point::new(2.0 * sx - ex, 2.0 * sy - ey), Point::new(ex, ey)), + }, + _ => crate::renderer::gradient_bbox_endpoints(grad.angle, bbox), + }; + + let stops: String = grad + .stops + .iter() + .map(|s| { + format!( + r##""##, + s.position, s.color.r, s.color.g, s.color.b, s.color.a as f32 / 255.0 + ) + }) + .collect(); + let spread = match grad.extend { + GradientExtend::Pad => "pad", + GradientExtend::Reflect => "reflect", + GradientExtend::Repeat => "repeat", + }; + + match grad.kind { + GradientType::Linear => format!( + r#"{stops}"#, + start.x, start.y, end.x, end.y + ), + GradientType::Radial => { + let (cx, cy) = ((start.x + end.x) * 0.5, (start.y + end.y) * 0.5); + let r = (((end.x - start.x).powi(2) + (end.y - start.y).powi(2)).sqrt()) * 0.5; + format!( + r#"{stops}"# + ) + } + } +} + +/// kurbo `BezPath` → SVG path-data string (`M/L/Q/C/Z`). +fn bezpath_to_d(path: &BezPath) -> String { + let mut d = String::new(); + for el in path.elements() { + match el { + PathEl::MoveTo(p) => d.push_str(&format!("M{:.3} {:.3} ", p.x, p.y)), + PathEl::LineTo(p) => d.push_str(&format!("L{:.3} {:.3} ", p.x, p.y)), + PathEl::QuadTo(p1, p) => d.push_str(&format!("Q{:.3} {:.3} {:.3} {:.3} ", p1.x, p1.y, p.x, p.y)), + PathEl::CurveTo(p1, p2, p) => d.push_str(&format!( + "C{:.3} {:.3} {:.3} {:.3} {:.3} {:.3} ", + p1.x, p1.y, p2.x, p2.y, p.x, p.y + )), + PathEl::ClosePath => d.push_str("Z "), + } + } + d.trim_end().to_string() +} + +// sRGB color → SVG attributes. Hex color + a separate `*-opacity` for max compatibility (Inkscape). +fn fill_attrs(c: &ShapeColor) -> String { + if c.a == 255 { + format!(r##"fill="#{:02x}{:02x}{:02x}""##, c.r, c.g, c.b) + } else { + format!(r##"fill="#{:02x}{:02x}{:02x}" fill-opacity="{:.4}""##, c.r, c.g, c.b, c.a as f32 / 255.0) + } +} +fn stroke_attrs(c: &ShapeColor) -> String { + if c.a == 255 { + format!(r##"stroke="#{:02x}{:02x}{:02x}""##, c.r, c.g, c.b) + } else { + format!(r##"stroke="#{:02x}{:02x}{:02x}" stroke-opacity="{:.4}""##, c.r, c.g, c.b, c.a as f32 / 255.0) + } +} +fn cap_str(cap: Cap) -> &'static str { + match cap { + Cap::Butt => "butt", + Cap::Round => "round", + Cap::Square => "square", + } +} +fn join_str(join: Join) -> &'static str { + match join { + Join::Miter => "miter", + Join::Round => "round", + Join::Bevel => "bevel", + } +} + +#[cfg(test)] +mod export_tests { + use super::*; + use crate::shape::{ShapeColor, StrokeStyle}; + use crate::vector_graph::{Direction, VectorGraph}; + use kurbo::{CubicBez, Point}; + + fn line(a: Point, b: Point) -> CubicBez { + // Degenerate cubic representing a straight segment (matches our model). + CubicBez::new(a, a.lerp(b, 1.0 / 3.0), a.lerp(b, 2.0 / 3.0), b) + } + + #[test] + fn solid_triangle_fill_and_stroke() { + let mut g = VectorGraph::new(); + let p0 = Point::new(10.0, 10.0); + let p1 = Point::new(90.0, 10.0); + let p2 = Point::new(50.0, 80.0); + let v0 = g.alloc_vertex(p0); + let v1 = g.alloc_vertex(p1); + let v2 = g.alloc_vertex(p2); + let stroke = Some(StrokeStyle { width: 2.0, ..Default::default() }); + let scol = Some(ShapeColor::rgb(0, 0, 0)); + let e0 = g.alloc_edge(line(p0, p1), v0, v1, stroke.clone(), scol); + let e1 = g.alloc_edge(line(p1, p2), v1, v2, stroke.clone(), scol); + let e2 = g.alloc_edge(line(p2, p0), v2, v0, stroke.clone(), scol); + g.alloc_fill( + vec![(e0, Direction::Forward), (e1, Direction::Forward), (e2, Direction::Forward)], + ShapeColor::rgb(255, 0, 0), + crate::shape::FillRule::NonZero, + ); + + let mut body = String::new(); + let mut defs = String::new(); + let mut n = 0; + vector_graph_to_svg(&g, &mut body, &mut defs, &mut n); + + assert!(body.contains(r##"fill="#ff0000""##), "fill color missing: {body}"); + assert!(body.contains(r#"fill-rule="nonzero""#), "fill-rule missing: {body}"); + assert!(body.contains(r#"fill="none""#), "stroke path missing: {body}"); + assert!(body.contains(r#"stroke-width="2.000""#), "stroke width missing: {body}"); + assert!(defs.is_empty(), "no gradients expected: {defs}"); + // 1 fill path + 3 stroked edges = 4 elements. + assert_eq!(body.matches(") { - let dstart = |entry: &(EdgeId, Direction)| -> Point { + // The four control points of an entry's curve in its traversal order. + let traversed = |entry: &(EdgeId, Direction)| -> [Point; 4] { let c = self.edges[entry.0.idx()].curve; match entry.1 { - Direction::Forward => c.p0, - Direction::Backward => c.p3, - } - }; - let dend = |entry: &(EdgeId, Direction)| -> Point { - let c = self.edges[entry.0.idx()].curve; - match entry.1 { - Direction::Forward => c.p3, - Direction::Backward => c.p0, + Direction::Forward => [c.p0, c.p1, c.p2, c.p3], + Direction::Backward => [c.p3, c.p2, c.p1, c.p0], } }; const EPS: f64 = 0.5; + // Entries i and j cancel only when j is the *exact reverse* of i — every control point of + // j matches the mirror of i. Testing endpoints alone would also collapse a genuine + // lens/sliver (two distinct edges that merely share near-coincident endpoints), silently + // deleting real boundary geometry and dropping the fill. + let reverses = |a: &(EdgeId, Direction), b: &(EdgeId, Direction)| -> bool { + let (ca, cb) = (traversed(a), traversed(b)); + (0..4).all(|k| { + let (p, q) = (ca[k], cb[3 - k]); + (p.x - q.x).hypot(p.y - q.y) < EPS + }) + }; loop { let n = face.len(); if n < 2 { @@ -1325,10 +1330,7 @@ impl VectorGraph { let mut collapsed = false; for i in 0..n { let j = (i + 1) % n; - // entries i and j cancel when j ends back at i's start. - let si = dstart(&face[i]); - let ej = dend(&face[j]); - if (si.x - ej.x).hypot(si.y - ej.y) < EPS { + if reverses(&face[i], &face[j]) { let (hi, lo) = if i > j { (i, j) } else { (j, i) }; face.remove(hi); face.remove(lo); diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 801132e..1f631f9 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -84,19 +84,215 @@ pub struct VideoMetadata { /// Video decoder with LRU frame caching pub struct VideoDecoder { source: VideoSource, - _width: u32, // Original video width - _height: u32, // Original video height - output_width: u32, // Scaled output width - output_height: u32, // Scaled output height + native_width: u32, // Original (decoded) video width + native_height: u32, // Original (decoded) video height fps: f64, _duration: f64, time_base: f64, stream_index: usize, - frame_cache: LruCache>, // timestamp -> RGBA data + // Decoded RGBA keyed by (frame timestamp, output width, output height): the same source + // frame may be requested at different sizes (preview res vs export res). + frame_cache: LruCache<(i64, u32, u32), Vec>, input: Option, decoder: Option, last_decoded_ts: i64, // Track the last decoded frame timestamp + /// The `frame_ts` requested by the previous `get_frame` call. Used to detect a genuine backward + /// jump (scrub/loop) from the request stream, which — unlike the decoded PTS — is strictly + /// monotonic during forward playback, so it never falses on the request-vs-PTS grid mismatch. + last_requested_ts: i64, keyframe_positions: Vec, // Index of keyframe timestamps for fast seeking + /// Reused RGBA scaler, keyed by `(input format, input w, input h, output w, output h)`. + /// Building an swscale context isn't free; a stream's frames share one input format/size and a + /// consumer keeps one output size, so it's built once and rebuilt only when either changes. + scaler: Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>, + /// When set (and `hw_failed` is false), decode on the GPU: attach `hw_device` as the decoder's + /// `hw_device_ctx`, decode into VAAPI surfaces, and hand each surface to `importer` to import as + /// wgpu NV12 textures (no CPU copy). `None`/failure → the software swscale path. + hw_device: Option, + importer: Option>, + /// Set if hardware decode init failed for this clip — fall back to software permanently. + hw_failed: bool, +} + +/// A decoded frame: CPU RGBA (software) or GPU NV12 textures (hardware). +enum DecodedFrame { + Cpu { rgba: Vec, width: u32, height: u32 }, + Gpu(GpuVideoFrame), +} + +/// Result of processing one decoded frame against the requested target timestamp. +enum FrameOutcome { + /// Frame consumed; keep decoding. + Continue, + /// `current_frame_ts >= frame_ts` — caller should return the best frame found. + ReachedTarget, + /// Hardware surface import failed; caller should fall back to software. + HwImportFailed, +} + +/// Process one decoded `frame`: update the best-so-far (`best_*`) toward `frame_ts`, importing a +/// GPU surface (`gpu_out`) or swscaling to RGBA otherwise. Pulled out of the decode loop so the +/// same logic runs both for packet-fed frames and for frames flushed out of the codec at EOF. +#[allow(clippy::too_many_arguments)] +fn process_decoded_video_frame( + frame: &mut ffmpeg::util::frame::Video, + decoder: &ffmpeg::decoder::Video, + gpu_out: bool, + hw: bool, + out_w: u32, + out_h: u32, + frame_ts: i64, + importer: Option<&Arc>, + scaler: &mut Option<(ffmpeg::format::Pixel, u32, u32, u32, u32, SendScaler)>, + best_frame_data: &mut Option>, + best_gpu: &mut Option, + best_frame_ts: &mut Option, + last_decoded_ts: &mut i64, + scale_time_ms: &mut u128, +) -> Result { + use std::time::Instant; + + // A frame with no PTS continues monotonically from the last decoded position rather than + // snapping to 0 — treating "no timestamp" as ts=0 would look like a huge backward jump and + // corrupt the best-frame / seek logic on streams with missing PTS. + let current_frame_ts = frame.timestamp().unwrap_or(*last_decoded_ts + 1); + *last_decoded_ts = current_frame_ts; + + let is_better = match *best_frame_ts { + None => true, + Some(best_ts) => (current_frame_ts - frame_ts).abs() < (best_ts - frame_ts).abs(), + }; + + if is_better { + if gpu_out { + // VAAPI hw frames often don't carry the stream's colour tags, so the importer would + // mis-detect transfer/gamut. Copy the authoritative values from the codec context + // (parsed from the bitstream) onto the frame when it left them unspecified. + unsafe { + use ffmpeg::ffi::*; + let fp = frame.as_mut_ptr(); + let cp = decoder.as_ptr(); + if (*fp).color_trc == AVColorTransferCharacteristic::AVCOL_TRC_UNSPECIFIED { + (*fp).color_trc = (*cp).color_trc; + } + if (*fp).color_primaries == AVColorPrimaries::AVCOL_PRI_UNSPECIFIED { + (*fp).color_primaries = (*cp).color_primaries; + } + if (*fp).colorspace == AVColorSpace::AVCOL_SPC_UNSPECIFIED { + (*fp).colorspace = (*cp).colorspace; + } + if (*fp).color_range == AVColorRange::AVCOL_RANGE_UNSPECIFIED { + (*fp).color_range = (*cp).color_range; + } + } + let importer = importer.unwrap(); + match unsafe { importer.import(frame.as_mut_ptr() as *mut std::ffi::c_void) } { + Some(gpu) => { + *best_gpu = Some(gpu); + *best_frame_ts = Some(current_frame_ts); + } + None => return Ok(FrameOutcome::HwImportFailed), + } + } else { + let t_scale_start = Instant::now(); + + // A hardware decoder produces VAAPI surfaces; a CPU consumer (export) downloads to + // system memory first, then swscales like the software path. + let downloaded; + let src: &ffmpeg::util::frame::Video = if hw { + let mut dl = ffmpeg::util::frame::Video::empty(); + let r = unsafe { + ffmpeg::ffi::av_hwframe_transfer_data(dl.as_mut_ptr(), frame.as_ptr(), 0) + }; + if r < 0 { + return Err(format!("av_hwframe_transfer_data failed: {r}")); + } + downloaded = dl; + &downloaded + } else { + &*frame + }; + + // Reuse the RGBA scaler across frames; rebuild only if the input format/size or the + // requested output size changes. + let need_new = match scaler { + Some((fmt, w, h, ow, oh, _)) => { + *fmt != src.format() || *w != src.width() || *h != src.height() + || *ow != out_w || *oh != out_h + } + None => true, + }; + if need_new { + let ctx = ffmpeg::software::scaling::context::Context::get( + src.format(), + src.width(), + src.height(), + ffmpeg::format::Pixel::RGBA, + out_w, + out_h, + ffmpeg::software::scaling::flag::Flags::BILINEAR, + ).map_err(|e| e.to_string())?; + *scaler = Some((src.format(), src.width(), src.height(), out_w, out_h, SendScaler(ctx))); + } + let scaler = &mut scaler.as_mut().unwrap().5.0; + + let mut rgb_frame = ffmpeg::util::frame::Video::empty(); + scaler.run(src, &mut rgb_frame) + .map_err(|e| e.to_string())?; + + // Remove stride padding to create tightly packed RGBA data + let width = out_w as usize; + let height = out_h as usize; + let stride = rgb_frame.stride(0); + let row_size = width * 4; // RGBA = 4 bytes per pixel + let source_data = rgb_frame.data(0); + + let mut packed_data = Vec::with_capacity(row_size * height); + for y in 0..height { + let row_start = y * stride; + let row_end = row_start + row_size; + packed_data.extend_from_slice(&source_data[row_start..row_end]); + } + + *scale_time_ms += t_scale_start.elapsed().as_millis(); + *best_frame_data = Some(packed_data); + *best_frame_ts = Some(current_frame_ts); + } + } + + if current_frame_ts >= frame_ts { + Ok(FrameOutcome::ReachedTarget) + } else { + Ok(FrameOutcome::Continue) + } +} + +/// `get_format` callback for hardware decode: select VAAPI surfaces. With `hw_device_ctx` set, +/// FFmpeg auto-allocates the frames context. +unsafe extern "C" fn get_vaapi_format( + _ctx: *mut ffmpeg::ffi::AVCodecContext, + mut fmts: *const ffmpeg::ffi::AVPixelFormat, +) -> ffmpeg::ffi::AVPixelFormat { + while *fmts != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE { + if *fmts == ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI { + return ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_VAAPI; + } + fmts = fmts.add(1); + } + ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE +} + +/// `SwsContext` is `!Send` in ffmpeg-next, but a `VideoDecoder` (like its decoder/input) is only +/// ever accessed under the `VideoManager` mutex — never concurrently — so moving it between +/// threads is sound. The decoder/input fields rely on the same invariant. +struct SendScaler(ffmpeg::software::scaling::context::Context); +unsafe impl Send for SendScaler {} + +/// Per-frame video decode tracing, gated behind `LB_VIDEO_DEBUG` (checked once). Off by +/// default — at export frame rates these prints are a lot of locked stderr writes. +fn video_debug() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("LB_VIDEO_DEBUG").is_ok()) } impl VideoDecoder { @@ -129,14 +325,12 @@ impl VideoDecoder { let height = decoder.height(); let time_base = f64::from(video_stream.time_base()); - // Calculate output dimensions (scale down if larger than max) - let (output_width, output_height) = if let (Some(max_w), Some(max_h)) = (max_width, max_height) { - // Calculate scale to fit within max dimensions while preserving aspect ratio - let scale = (max_w as f32 / width as f32).min(max_h as f32 / height as f32).min(1.0); - ((width as f32 * scale) as u32, (height as f32 * scale) as u32) - } else { - (width, height) - }; + // Output dimensions are now chosen per `get_frame` call (the caller's target res, capped to + // native) rather than frozen here — so the same clip can be decoded at preview res for the + // canvas and at full export res, and exporting above document res no longer upscales. + // `max_width`/`max_height` are retained as an upper bound for callers that want a fixed cap + // (e.g. thumbnails pass their thumb width per call instead). + let _ = (max_width, max_height); // Try to get duration from stream, fallback to container let duration = if video_stream.duration() > 0 { @@ -168,10 +362,8 @@ impl VideoDecoder { Ok(Self { source, - _width: width, - _height: height, - output_width, - output_height, + native_width: width, + native_height: height, fps, _duration: duration, time_base, @@ -182,10 +374,31 @@ impl VideoDecoder { input: None, decoder: None, last_decoded_ts: -1, + last_requested_ts: i64::MIN, keyframe_positions, + scaler: None, + hw_device: None, + importer: None, + hw_failed: false, }) } + /// Configure hardware (VAAPI) decode for this clip. The next decoder open attaches `hw_device` + /// and decodes into VAAPI surfaces imported by `importer`. Resets any prior decoder so the new + /// mode takes effect on the next `get_frame`. + fn set_hardware(&mut self, hw_device: HwDeviceHandle, importer: Arc) { + self.hw_device = Some(hw_device); + self.importer = Some(importer); + self.hw_failed = false; + self.decoder = None; // force a rebuild with hw_device_ctx + self.input = None; + } + + /// Whether this decoder will hardware-decode (configured + not failed). + fn hw_active(&self) -> bool { + self.hw_device.is_some() && self.importer.is_some() && !self.hw_failed + } + /// The source this decoder reads from (file path or packed container blob). pub fn source(&self) -> VideoSource { self.source.clone() @@ -201,19 +414,22 @@ impl VideoDecoder { self.keyframe_positions = positions; } - /// Get the output width (scaled dimensions) - pub fn get_output_width(&self) -> u32 { - self.output_width + /// The output size for a requested target: the target capped to native resolution, preserving + /// aspect ratio (never upscale beyond native — there's no detail to invent). + fn capped_output(&self, target_w: u32, target_h: u32) -> (u32, u32) { + let (nw, nh) = (self.native_width as f32, self.native_height as f32); + if nw <= 0.0 || nh <= 0.0 { return (self.native_width.max(1), self.native_height.max(1)); } + let scale = (target_w as f32 / nw).min(target_h as f32 / nh).min(1.0); + (((nw * scale) as u32).max(1), ((nh * scale) as u32).max(1)) } - /// Get the output height (scaled dimensions) - pub fn get_output_height(&self) -> u32 { - self.output_height - } - - /// Decode a frame at the specified timestamp (public wrapper) + /// Decode a frame at the specified timestamp, at native resolution (public wrapper). pub fn decode_frame(&mut self, timestamp: f64) -> Result, String> { - self.get_frame(timestamp) + // Software-only helper; request CPU output. + match self.get_frame(timestamp, self.native_width, self.native_height, false)? { + DecodedFrame::Cpu { rgba, .. } => Ok(rgba), + DecodedFrame::Gpu(_) => Err("decode_frame: unexpected GPU frame".into()), + } } /// Build an index of all keyframe positions in the video by scanning packets @@ -255,11 +471,21 @@ impl VideoDecoder { } } - /// Get a decoded frame at the specified timestamp - fn get_frame(&mut self, timestamp: f64) -> Result, String> { + /// Decode the frame at `timestamp`, scaled to `capped_output(target_w, target_h)`. Returns GPU + /// NV12 textures when hardware-decoding and `want_gpu` (the consumer is on the shared device, + /// i.e. the preview); otherwise CPU RGBA. A hardware decoder serving a CPU consumer (export) + /// downloads the surface via `av_hwframe_transfer_data` then swscales. The `VideoManager` caches + /// the result, so the inner RGBA cache here is for CPU output only. + fn get_frame(&mut self, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Result { use std::time::Instant; let t_start = Instant::now(); + // `hw` = decoder is opened in hardware mode (produces VAAPI surfaces). + // `gpu_out` = return GPU textures (hw + the consumer can use them). + let hw = self.hw_active(); + let gpu_out = hw && want_gpu; + let (out_w, out_h) = self.capped_output(target_w, target_h); + // Round timestamp to nearest frame boundary to improve cache hits // This ensures that timestamps like 1.0001s and 0.9999s both map to frame 1.0s let frame_duration = 1.0 / self.fps; @@ -267,19 +493,41 @@ impl VideoDecoder { // Convert timestamp to frame timestamp let frame_ts = (rounded_timestamp / self.time_base) as i64; + let cache_key = (frame_ts, out_w, out_h); - // Check cache - if let Some(cached_frame) = self.frame_cache.get(&frame_ts) { - eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); - return Ok(cached_frame.clone()); + // Check the inner RGBA cache (CPU output only; GPU frames are cached by VideoManager). + if !gpu_out { + if let Some(cached_frame) = self.frame_cache.get(&cache_key) { + if video_debug() { + eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + } + return Ok(DecodedFrame::Cpu { rgba: cached_frame.clone(), width: out_w, height: out_h }); + } } - // Determine if we need to seek - // Seek if: no decoder open, going backwards, or jumping forward more than 2 seconds + // Determine if we need to seek. Seek if: no decoder open, the *request* jumped backward + // (scrub/step-back/loop), or the request is more than 2s ahead of the decoder's position. + // + // Detect "backward" from the request stream, NOT the decoded frame PTS. The requested + // presentation time is rounded to a frame and converted through floats + // (`round(ts/fd)*fd / time_base`), which routinely lands a frame *behind* the exact PTS of + // the frame just produced — so `frame_ts < last_decoded_ts` falsely fires during smooth + // forward playback (every ~10 frames → a 40ms seek + whole-GOP re-decode = the 4K jerk). + // The request `frame_ts` is strictly monotonic while playing forward, so comparing against + // the previous request never falses; a real backward scrub still decreases it. let need_seek = self.decoder.is_none() - || frame_ts < self.last_decoded_ts + || frame_ts < self.last_requested_ts || frame_ts > self.last_decoded_ts + (2.0 / self.time_base) as i64; + if need_seek && video_debug() { + let reason = if self.decoder.is_none() { "no decoder" } + else if frame_ts < self.last_requested_ts { "backward" } + else { "forward>2s" }; + eprintln!("[Video Seek?] need_seek={reason} frame_ts={frame_ts} last_requested_ts={} last_decoded_ts={}", + self.last_requested_ts, self.last_decoded_ts); + } + self.last_requested_ts = frame_ts; + if need_seek { let t_seek_start = Instant::now(); @@ -292,8 +540,10 @@ impl VideoDecoder { let keyframe_seconds = keyframe_ts_stream as f64 * self.time_base; let keyframe_ts_av = (keyframe_seconds * 1_000_000.0) as i64; // AV_TIME_BASE = 1000000 - eprintln!("[Video Seek] Target: {} | Keyframe(stream): {} | Keyframe(AV): {} | Index size: {}", - frame_ts, keyframe_ts_stream, keyframe_ts_av, self.keyframe_positions.len()); + if video_debug() { + eprintln!("[Video Seek] Target: {} | Keyframe(stream): {} | Keyframe(AV): {} | Index size: {}", + frame_ts, keyframe_ts_stream, keyframe_ts_av, self.keyframe_positions.len()); + } // Reopen input (a fresh BlobReader for packed sources). let mut owned = self.source.open() @@ -306,15 +556,37 @@ impl VideoDecoder { input.seek(keyframe_ts_av, keyframe_ts_av..(keyframe_ts_av + 1)) .map_err(|e| format!("Seek failed: {}", e))?; - eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis()); + if video_debug() { + eprintln!("[Video Timing] Seek call took {}ms", t_seek_start.elapsed().as_millis()); + } let context_decoder = ffmpeg::codec::context::Context::from_parameters( input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() ).map_err(|e| e.to_string())?; - let decoder = context_decoder.decoder().video() - .map_err(|e| e.to_string())?; - self.decoder = Some(decoder); + let mut dec_ctx = context_decoder.decoder(); + if hw { + // Attach the VAAPI device + format selector before opening so the decoder + // produces hardware surfaces. + unsafe { + let ctx = dec_ctx.as_mut_ptr(); + let hwdev = self.hw_device.unwrap().0 as *mut ffmpeg::ffi::AVBufferRef; + (*ctx).hw_device_ctx = ffmpeg::ffi::av_buffer_ref(hwdev); + (*ctx).get_format = Some(get_vaapi_format); + } + } + match dec_ctx.video() { + Ok(decoder) => self.decoder = Some(decoder), + Err(e) if hw => { + // Hardware decode unavailable for this clip — fall back to software. This + // frame fails; the next call rebuilds a software decoder. + eprintln!("[Video] hardware decode unavailable ({e}); falling back to software"); + self.hw_failed = true; + self.decoder = None; + return Err(format!("hw decode init failed: {e}")); + } + Err(e) => return Err(e.to_string()), + } } self.input = Some(owned); // Set last_decoded_ts to just before the seek target so forward playback works @@ -327,82 +599,87 @@ impl VideoDecoder { // Decode frames until we find the one closest to our target timestamp let mut best_frame_data: Option> = None; + let mut best_gpu: Option = None; let mut best_frame_ts: Option = None; let t_decode_start = Instant::now(); let mut decode_count = 0; let mut scale_time_ms = 0u128; + let mut hw_import_failed = false; - for (stream, packet) in input.packets() { - if stream.index() == self.stream_index { - decoder.send_packet(&packet) - .map_err(|e| e.to_string())?; + 'decode: { + for (stream, packet) in input.packets() { + if stream.index() == self.stream_index { + decoder.send_packet(&packet) + .map_err(|e| e.to_string())?; - let mut frame = ffmpeg::util::frame::Video::empty(); - while decoder.receive_frame(&mut frame).is_ok() { - decode_count += 1; - let current_frame_ts = frame.timestamp().unwrap_or(0); - self.last_decoded_ts = current_frame_ts; // Update last decoded position - - // Check if this frame is closer to our target than the previous best - let is_better = match best_frame_ts { - None => true, - Some(best_ts) => { - (current_frame_ts - frame_ts).abs() < (best_ts - frame_ts).abs() + let mut frame = ffmpeg::util::frame::Video::empty(); + while decoder.receive_frame(&mut frame).is_ok() { + decode_count += 1; + match process_decoded_video_frame( + &mut frame, decoder, gpu_out, hw, out_w, out_h, frame_ts, + self.importer.as_ref(), &mut self.scaler, + &mut best_frame_data, &mut best_gpu, &mut best_frame_ts, + &mut self.last_decoded_ts, &mut scale_time_ms, + )? { + FrameOutcome::Continue => {} + FrameOutcome::ReachedTarget => break 'decode, + FrameOutcome::HwImportFailed => { + self.hw_failed = true; + hw_import_failed = true; + break 'decode; + } } - }; - - if is_better { - let t_scale_start = Instant::now(); - - // Convert to RGBA and scale to output size - let mut scaler = ffmpeg::software::scaling::context::Context::get( - frame.format(), - frame.width(), - frame.height(), - ffmpeg::format::Pixel::RGBA, - self.output_width, - self.output_height, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ).map_err(|e| e.to_string())?; - - let mut rgb_frame = ffmpeg::util::frame::Video::empty(); - scaler.run(&frame, &mut rgb_frame) - .map_err(|e| e.to_string())?; - - // Remove stride padding to create tightly packed RGBA data - let width = self.output_width as usize; - let height = self.output_height as usize; - let stride = rgb_frame.stride(0); - let row_size = width * 4; // RGBA = 4 bytes per pixel - let source_data = rgb_frame.data(0); - - let mut packed_data = Vec::with_capacity(row_size * height); - for y in 0..height { - let row_start = y * stride; - let row_end = row_start + row_size; - packed_data.extend_from_slice(&source_data[row_start..row_end]); - } - - scale_time_ms += t_scale_start.elapsed().as_millis(); - best_frame_data = Some(packed_data); - best_frame_ts = Some(current_frame_ts); - } - - // If we've reached or passed the target timestamp, we can stop - if current_frame_ts >= frame_ts { - // Found our frame, cache and return it - if let Some(data) = best_frame_data { - let total_time = t_start.elapsed().as_millis(); - let decode_time = t_decode_start.elapsed().as_millis(); - eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", - timestamp, decode_count, decode_time, scale_time_ms, total_time); - self.frame_cache.put(frame_ts, data.clone()); - return Ok(data); - } - break; } } } + + // Flush: the codec may still hold buffered frames (B-frame reorder delay) past the + // last packet. Drain them so requesting the final frame(s) of a clip — scrubbing to + // the end or exporting the tail — doesn't fail with "Failed to decode frame". + if !hw_import_failed { + let _ = decoder.send_eof(); + let mut frame = ffmpeg::util::frame::Video::empty(); + while decoder.receive_frame(&mut frame).is_ok() { + decode_count += 1; + match process_decoded_video_frame( + &mut frame, decoder, gpu_out, hw, out_w, out_h, frame_ts, + self.importer.as_ref(), &mut self.scaler, + &mut best_frame_data, &mut best_gpu, &mut best_frame_ts, + &mut self.last_decoded_ts, &mut scale_time_ms, + )? { + FrameOutcome::Continue => {} + FrameOutcome::ReachedTarget => break 'decode, + FrameOutcome::HwImportFailed => { + self.hw_failed = true; + hw_import_failed = true; + break 'decode; + } + } + } + } + } + + if video_debug() { + let total_time = t_start.elapsed().as_millis(); + let decode_time = t_decode_start.elapsed().as_millis(); + eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms | {}", + timestamp, decode_count, decode_time, scale_time_ms, total_time, if hw { "hw" } else { "sw" }); + } + + // Reached the target, EOF, or HW import failed mid-stream. + if hw_import_failed { + self.decoder = None; // force a software rebuild next call (decoder borrow ended here) + self.input = None; + return Err("hardware frame import failed; retrying software".to_string()); + } + // Return the closest frame we found, if any. + if gpu_out { + if let Some(gpu) = best_gpu.take() { + return Ok(DecodedFrame::Gpu(gpu)); + } + } else if let Some(data) = best_frame_data { + self.frame_cache.put(cache_key, data.clone()); + return Ok(DecodedFrame::Cpu { rgba: data, width: out_w, height: out_h }); } eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp); @@ -455,8 +732,25 @@ pub fn generate_keyframe_thumbnails( if should_skip(ks) { continue; } - if let Ok(rgba) = decoder.get_frame(ks) { - on_thumb(ks, Arc::new(rgba)); + // Decode at the thumbnail width (large height so width is the constraint), capped to native. + // Thumbnail decoders are always software (no hardware importer). + if let Ok(DecodedFrame::Cpu { rgba, width, height }) = decoder.get_frame(ks, thumb_width, 100_000, false) { + // `capped_output` never upscales, so a source narrower than `thumb_width` decodes + // smaller — but `get_thumbnail_at` reconstructs height assuming an exact `thumb_width`. + // Force the exact width here (rare path) so that assumption holds and the thumbnail + // isn't shown stretched. + let data = if width == thumb_width || width == 0 || height == 0 { + rgba + } else { + let new_h = ((thumb_width as u64 * height as u64) / width as u64).max(1) as u32; + match image::RgbaImage::from_raw(width, height, rgba) { + Some(img) => image::imageops::resize( + &img, thumb_width, new_h, image::imageops::FilterType::Triangle, + ).into_raw(), + None => continue, + } + }; + on_thumb(ks, Arc::new(data)); } } Ok(()) @@ -520,10 +814,98 @@ pub fn probe_video(source: &VideoSource) -> Result { pub struct VideoFrame { pub width: u32, pub height: u32, + /// CPU-decoded sRGB RGBA8 (software path). Empty when `gpu` is `Some`. pub rgba_data: Arc>, + /// Hardware-decoded frame living on the GPU (NV12 plane textures). When `Some`, the compositor + /// samples it directly and `rgba_data` is empty. + pub gpu: Option, pub timestamp: f64, } +/// A hardware-decoded video frame on the GPU: two NV12 plane textures (Y = R8, UV = RG8) imported +/// from a VAAPI DMA-BUF on the editor's shared wgpu device. The compositor samples these directly +/// (NV12→RGB), no CPU copy. +#[derive(Clone, Debug)] +pub struct GpuVideoFrame { + pub y: Arc, + pub uv: Arc, + pub width: u32, + pub height: u32, + /// Source YUV range: true = full/PC (0–255), false = limited/TV (16–235). Drives the NV12→RGB + /// offset/scale in the compositor. + pub full_range: bool, + /// Y'CbCr→R'G'B' matrix coefficients derived from the frame's colorspace (BT.709/601/2020), + /// so SD (BT.601) and HD/UHD clips each convert correctly: `[Cr→R, Cb→G, Cr→G, Cb→B]`. + /// R = Y + c[0]·Cr, G = Y + c[1]·Cb + c[2]·Cr, B = Y + c[3]·Cb + pub coeffs: [f32; 4], + /// Opto-electronic transfer of the encoded R'G'B' — the compositor applies the matching EOTF to + /// reach scene-linear (graphics white = 1.0). HDR (PQ/HLG) values exceed 1.0. + pub transfer: VideoTransfer, + /// Colour primaries; BT.2020 is gamut-mapped to the compositor's BT.709 space in linear light. + pub primaries: VideoPrimaries, +} + +/// Transfer characteristic of a decoded video frame (selects the EOTF in the NV12→linear pass). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VideoTransfer { + /// SDR gamma (BT.709/sRGB/601/gamma22) — approximated by the sRGB EOTF. + Gamma, + /// SMPTE ST 2084 (PQ) — absolute, normalized so 203 nits (graphics white) = 1.0. + Pq, + /// ARIB STD-B67 (HLG) — scene-referred, normalized so reference white ≈ 1.0. + Hlg, +} + +/// Colour primaries of a decoded video frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VideoPrimaries { + /// BT.709 / sRGB (also used for BT.601, whose primaries differ only slightly). + Bt709, + /// BT.2020 (wide gamut) — converted to BT.709 in linear light by the compositor. + Bt2020, +} + +/// Y'CbCr→R'G'B' matrix coefficients (`[Cr→R, Cb→G, Cr→G, Cb→B]`) from the luma weights `kr`/`kb` +/// (`kg = 1−kr−kb`). BT.709 → `[1.5748, −0.1873, −0.4681, 1.8556]`. +pub fn ycbcr_coeffs(kr: f32, kb: f32) -> [f32; 4] { + let kg = 1.0 - kr - kb; + [ + 2.0 * (1.0 - kr), + -2.0 * kb * (1.0 - kb) / kg, + -2.0 * kr * (1.0 - kr) / kg, + 2.0 * (1.0 - kb), + ] +} + +/// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't +/// reference the GPU crate's ffmpeg-sys types) into [`GpuVideoFrame`] textures on the shared device. +/// Implemented by the editor; `gpu-video-encoder` does the actual DMA-BUF import. +pub trait HwVideoImporter: Send + Sync { + /// # Safety + /// `av_frame` must be a valid `*mut ffmpeg_sys_next::AVFrame` holding a VAAPI surface. + unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option; +} + +/// Opaque handle to the FFmpeg VAAPI hardware device (`*mut AVBufferRef`), created by the editor and +/// handed to core so decoders can attach it as `hw_device_ctx`. Core never frees it (the editor owns +/// it for the app's lifetime). +#[derive(Clone, Copy)] +pub struct HwDeviceHandle(pub *mut std::ffi::c_void); +// SAFETY: the pointer is an AVBufferRef whose refcount is managed by FFmpeg; we only `av_buffer_ref` +// it (atomic) and never free it, so sharing the handle across threads is sound. +unsafe impl Send for HwDeviceHandle {} +unsafe impl Sync for HwDeviceHandle {} + +/// Approximate resident bytes of a cached frame for the byte budget: the CPU RGBA buffer, or for a +/// GPU (NV12) frame ~`w*h*3/2` of VRAM, so GPU frames stay bounded too. +fn frame_cache_bytes(frame: &VideoFrame) -> usize { + if frame.gpu.is_some() { + (frame.width as usize * frame.height as usize * 3) / 2 + } else { + frame.rgba_data.len() + } +} + /// Manages video decoders and frame caching for multiple video clips pub struct VideoManager { /// Pool of video decoders, one per clip @@ -533,7 +915,7 @@ pub struct VideoManager { /// zero-copy rendering. Bounded by a **byte budget** (not a frame count, which /// would be unsafe across resolutions — a 4K frame is ~33MB vs ~2MB at 800x600) /// so playback of arbitrarily long video never grows unbounded. - frame_cache: LruCache<(Uuid, i64), Arc>, + frame_cache: LruCache<(Uuid, i64, u32, u32, bool), Arc>, /// Running total of bytes held in `frame_cache` (sum of each frame's RGBA len), /// kept in sync on insert/evict/remove so eviction is O(1) per frame. frame_cache_bytes: usize, @@ -549,6 +931,15 @@ pub struct VideoManager { /// Maximum number of frames to cache per decoder cache_size: usize, + + /// Hardware (VAAPI) decode, injected by the editor once the shared device is up. When set, each + /// decoder attaches the VAAPI device and imports frames as GPU textures via `hw_importer`. + hw_device: Option, + hw_importer: Option>, + /// Whether the current render pass can consume GPU textures (preview = true; export = false, + /// since it composites on a different device → a hardware decoder downloads to CPU instead). + /// Set by the render caller before each pass. + render_hardware_ok: bool, } /// Byte budget for [`VideoManager::frame_cache`] (decoded full-resolution frames). @@ -571,9 +962,34 @@ impl VideoManager { thumbnail_cache: HashMap::new(), thumbnails_complete: std::collections::HashSet::new(), cache_size, + hw_device: None, + hw_importer: None, + render_hardware_ok: true, } } + /// Set whether the upcoming render pass can consume GPU video textures (preview = true; export = + /// false). Call before `render_document_for_compositing`. + pub fn set_render_hardware_ok(&mut self, ok: bool) { + self.render_hardware_ok = ok; + } + + /// Enable hardware (VAAPI) decode for all clips. Injected by the editor once the shared wgpu + /// device is active; `hw_device` is the FFmpeg VAAPI device and `importer` imports decoded + /// surfaces as GPU textures on that device. Applies to existing and future decoders. Clears the + /// frame cache (cached CPU frames would otherwise hide the new GPU frames). + pub fn set_hardware_decode(&mut self, hw_device: HwDeviceHandle, importer: Arc) { + self.hw_device = Some(hw_device); + self.hw_importer = Some(Arc::clone(&importer)); + for dec in self.decoders.values() { + if let Ok(mut d) = dec.lock() { + d.set_hardware(hw_device, Arc::clone(&importer)); + } + } + self.frame_cache.clear(); + self.frame_cache_bytes = 0; + } + /// Load a video file and create a decoder for it /// /// `target_width` and `target_height` specify the maximum dimensions @@ -593,7 +1009,7 @@ impl VideoManager { let metadata = probe_video(&source)?; // Create decoder with target dimensions, without building keyframe index - let decoder = VideoDecoder::new( + let mut decoder = VideoDecoder::new( source, self.cache_size, Some(target_width), @@ -601,6 +1017,11 @@ impl VideoManager { false, // Don't build keyframe index synchronously )?; + // Inherit hardware decode if the manager has it configured. + if let (Some(hw), Some(imp)) = (self.hw_device, &self.hw_importer) { + decoder.set_hardware(hw, Arc::clone(imp)); + } + // Store decoder in pool self.decoders.insert(clip_id, Arc::new(Mutex::new(decoder))); @@ -609,35 +1030,67 @@ impl VideoManager { /// Get a decoded frame for a specific clip at a specific timestamp /// - /// Returns None if the clip is not loaded or decoding fails. - /// Frames are cached for performance. - pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64) -> Option> { - // Convert timestamp to milliseconds for cache key - let timestamp_ms = (timestamp * 1000.0) as i64; - let cache_key = (*clip_id, timestamp_ms); + /// Returns None if the clip is not loaded or decoding fails. Frames are cached. + /// Whether a hardware decoder returns a GPU texture or downloads to CPU RGBA depends on + /// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the + /// preview, false for export, which composites on a different device). + pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { + // Whether this pass wants (and can produce) a GPU frame. Gated on HW being configured at all + // so that with software-only decode preview and export share one cache entry (no double-cache). + let want_gpu = self.render_hardware_ok && self.hw_device.is_some(); + self.get_frame_inner(clip_id, timestamp, target_w, target_h, want_gpu) + } + + /// Like [`get_frame`](Self::get_frame) but always returns a CPU (RGBA) frame, ignoring the + /// render-pass hardware flag. For consumers that need pixel bytes (thumbnails, image readback) + /// regardless of whether a render pass last enabled GPU frames. + pub fn get_frame_cpu(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { + self.get_frame_inner(clip_id, timestamp, target_w, target_h, false) + } + + fn get_frame_inner(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32, want_gpu: bool) -> Option> { + // Get the decoder for this clip. Clone the Arc so we don't hold a borrow of + // `self.decoders` across the `&mut self` cache insert below. + let decoder_arc = Arc::clone(self.decoders.get(clip_id)?); + + // Key the cache on the video FRAME INDEX, not milliseconds. A UI that refreshes finer than + // the video frame rate (e.g. a 60 Hz canvas on a 30 fps clip) would otherwise miss the cache + // on every sub-frame request and decode the NEXT frame each time — advancing the video faster + // than the clock (it plays sped-up). Rounding to round(ts·fps) maps all requests within one + // frame to a single entry, so each frame is decoded exactly once. + let fps = decoder_arc.lock().ok()?.fps; + let frame_index = if fps > 0.0 { (timestamp * fps).round() as i64 } else { (timestamp * 1000.0) as i64 }; + // The key also includes (target size, want_gpu): preview (GPU, preview res) and export + // (CPU, export res) request the same clip/time and must not collide or cross representation. + let cache_key = (*clip_id, frame_index, target_w, target_h, want_gpu); // Check frame cache first if let Some(cached_frame) = self.frame_cache.get(&cache_key) { return Some(Arc::clone(cached_frame)); } - // Get decoder for this clip. Clone the Arc so we don't hold a borrow of - // `self.decoders` across the `&mut self` cache insert below. - let decoder_arc = Arc::clone(self.decoders.get(clip_id)?); let mut decoder = decoder_arc.lock().ok()?; - // Decode the frame - let rgba_data = decoder.get_frame(timestamp).ok()?; - let width = decoder.output_width; - let height = decoder.output_height; + // Decode the frame at the requested target (capped to native by the decoder). + let decoded = decoder.get_frame(timestamp, target_w, target_h, want_gpu).ok()?; drop(decoder); // release the lock before touching `self` - // Create VideoFrame and cache it - let frame = Arc::new(VideoFrame { - width, - height, - rgba_data: Arc::new(rgba_data), - timestamp, + // Create VideoFrame and cache it. + let frame = Arc::new(match decoded { + DecodedFrame::Cpu { rgba, width, height } => VideoFrame { + width, + height, + rgba_data: Arc::new(rgba), + gpu: None, + timestamp, + }, + DecodedFrame::Gpu(gpu) => VideoFrame { + width: gpu.width, + height: gpu.height, + rgba_data: Arc::new(Vec::new()), + gpu: Some(gpu), + timestamp, + }, }); self.cache_frame(cache_key, Arc::clone(&frame)); @@ -647,16 +1100,16 @@ impl VideoManager { /// Insert a frame into the byte-budgeted cache, evicting least-recently-used /// frames until the total is within [`FRAME_CACHE_BYTE_BUDGET`]. - fn cache_frame(&mut self, key: (Uuid, i64), frame: Arc) { - let bytes = frame.rgba_data.len(); + fn cache_frame(&mut self, key: (Uuid, i64, u32, u32, bool), frame: Arc) { + let bytes = frame_cache_bytes(&frame); if let Some(old) = self.frame_cache.put(key, frame) { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(old.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&old)); } self.frame_cache_bytes += bytes; // Keep at least one frame resident even if it alone exceeds the budget. while self.frame_cache_bytes > FRAME_CACHE_BYTE_BUDGET && self.frame_cache.len() > 1 { if let Some((_, evicted)) = self.frame_cache.pop_lru() { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(evicted.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&evicted)); } else { break; } @@ -763,15 +1216,15 @@ impl VideoManager { // Remove all cached frames for this clip (LruCache has no retain; collect // matching keys, then pop each, keeping the byte total in sync). - let keys: Vec<(Uuid, i64)> = self + let keys: Vec<(Uuid, i64, u32, u32, bool)> = self .frame_cache .iter() - .filter(|((id, _), _)| id == clip_id) + .filter(|((id, _, _, _, _), _)| id == clip_id) .map(|(k, _)| *k) .collect(); for key in keys { if let Some(frame) = self.frame_cache.pop(&key) { - self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame.rgba_data.len()); + self.frame_cache_bytes = self.frame_cache_bytes.saturating_sub(frame_cache_bytes(&frame)); } } diff --git a/lightningbeam-ui/lightningbeam-editor/Cargo.toml b/lightningbeam-ui/lightningbeam-editor/Cargo.toml index 9ad8b55..c80d068 100644 --- a/lightningbeam-ui/lightningbeam-editor/Cargo.toml +++ b/lightningbeam-ui/lightningbeam-editor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lightningbeam-editor" -version = "1.0.6-alpha" +version = "1.0.7-alpha" edition = "2021" description = "Multimedia editor for audio, video and 2D animation" license = "GPL-3.0-or-later" diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs index 2ba087c..bfd4271 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/cpu_yuv_converter.rs @@ -26,9 +26,9 @@ impl CpuYuvConverter { /// # Arguments /// * `width` - Frame width in pixels /// * `height` - Frame height in pixels - pub fn new(width: u32, height: u32) -> Result { + pub fn new(width: u32, height: u32, full_range: bool) -> Result { // BT.709 (HD) RGBA→YUV420p context, created once. - let scaler = ffmpeg::software::scaling::Context::get( + let mut scaler = ffmpeg::software::scaling::Context::get( ffmpeg::format::Pixel::RGBA, width, height, @@ -38,6 +38,23 @@ impl CpuYuvConverter { ffmpeg::software::scaling::Flags::BILINEAR, ) .map_err(|e| format!("Failed to create swscale context: {}", e))?; + + // swscale defaults to BT.601 + limited range; force BT.709 with the requested output + // range so this fallback matches the GPU path and the encoder's color tags + // (otherwise non-%8-width exports come out with shifted hue / wrong levels). There is + // no safe ffmpeg-next wrapper for sws_setColorspaceDetails, so this is the raw call. + unsafe { + let coeffs = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709 as i32); + let dst_range = if full_range { 1 } else { 0 }; + let one = 1 << 16; // 16.16 fixed-point 1.0 + ffmpeg::ffi::sws_setColorspaceDetails( + scaler.as_mut_ptr(), + coeffs, 1, // source table (RGB input is full-range) + coeffs, dst_range, // dest table = BT.709, dest range = requested + 0, one, one, // brightness, contrast, saturation (neutral) + ); + } + let rgba_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::RGBA, width, height); let yuv_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::YUV420P, width, height); Ok(Self { width, height, scaler, rgba_frame, yuv_frame }) @@ -90,13 +107,13 @@ mod tests { #[test] fn test_converter_creation() { - let converter = CpuYuvConverter::new(1920, 1080); + let converter = CpuYuvConverter::new(1920, 1080, true); assert!(converter.is_ok()); } #[test] fn test_conversion_output_sizes() { - let mut converter = CpuYuvConverter::new(1920, 1080).unwrap(); + let mut converter = CpuYuvConverter::new(1920, 1080, true).unwrap(); // Create dummy RGBA data (all black) let rgba_data = vec![0u8; 1920 * 1080 * 4]; @@ -117,7 +134,7 @@ mod tests { #[test] #[should_panic(expected = "RGBA data size mismatch")] fn test_wrong_input_size_panics() { - let mut converter = CpuYuvConverter::new(1920, 1080).unwrap(); + let mut converter = CpuYuvConverter::new(1920, 1080, true).unwrap(); // Wrong size input let rgba_data = vec![0u8; 1000]; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs index 11eb248..aa5dddb 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/dialog.rs @@ -6,7 +6,7 @@ use eframe::egui; use lightningbeam_core::export::{ AudioExportSettings, AudioFormat, ImageExportSettings, ImageFormat, - VideoExportSettings, VideoCodec, VideoQuality, + VideoExportSettings, VideoCodec, VideoQuality, ColorRange, }; use std::path::PathBuf; @@ -25,6 +25,8 @@ pub enum ExportType { Audio, Image, Video, + /// Vector-only SVG of the current frame (lossless; raster/video layers skipped). + Svg, } /// Export result from dialog @@ -34,6 +36,8 @@ pub enum ExportResult { Image(ImageExportSettings, PathBuf), VideoOnly(VideoExportSettings, PathBuf), VideoWithAudio(VideoExportSettings, AudioExportSettings, PathBuf), + /// SVG of vector layers at the given document time. + Svg(f64, PathBuf), } /// Export dialog state @@ -156,6 +160,7 @@ impl ExportDialog { ExportType::Audio => self.audio_settings.format.extension(), ExportType::Image => self.image_settings.format.extension(), ExportType::Video => self.video_settings.codec.container_format(), + ExportType::Svg => "svg", } } @@ -198,6 +203,7 @@ impl ExportDialog { ExportType::Audio => "Export Audio", ExportType::Image => "Export Image", ExportType::Video => "Export Video", + ExportType::Svg => "Export SVG", }; let modal_response = egui::Modal::new(egui::Id::new("export_dialog_modal")) @@ -219,6 +225,7 @@ impl ExportDialog { (ExportType::Audio, "Audio"), (ExportType::Image, "Image"), (ExportType::Video, "Video"), + (ExportType::Svg, "SVG"), ] { if ui.selectable_value(&mut self.export_type, variant, label).clicked() { self.update_filename_extension(); @@ -235,6 +242,7 @@ impl ExportDialog { ExportType::Audio => self.render_audio_basic(ui), ExportType::Image => self.render_image_settings(ui), ExportType::Video => self.render_video_basic(ui), + ExportType::Svg => self.render_svg_settings(ui), } ui.add_space(12.0); @@ -253,6 +261,7 @@ impl ExportDialog { ExportType::Audio => self.render_audio_advanced(ui), ExportType::Image => self.render_image_advanced(ui), ExportType::Video => self.render_video_advanced(ui), + ExportType::Svg => {} // SVG has no advanced settings } } @@ -356,6 +365,20 @@ impl ExportDialog { } } + /// Render SVG export settings — just the frame time (reuses the image time field). + fn render_svg_settings(&mut self, ui: &mut egui::Ui) { + ui.horizontal(|ui| { + ui.label("Time:"); + ui.add(egui::DragValue::new(&mut self.image_settings.time) + .speed(0.01) + .range(0.0..=f64::MAX) + .suffix(" s")); + }); + ui.add_space(4.0); + ui.weak("Exports vector layers losslessly at this frame. Raster, video, and"); + ui.weak("effect layers are not included."); + } + /// Render advanced image export settings (time, resolution override). fn render_image_advanced(&mut self, ui: &mut egui::Ui) { // Time (which frame to export) @@ -378,6 +401,19 @@ impl ExportDialog { if changed_h { self.image_settings.height = if h == 0 { None } else { Some(h) }; } ui.weak("(0 = document size)"); }); + + // Fit mode — how the document maps into the output frame when aspect ratios differ. + ui.horizontal(|ui| { + use lightningbeam_core::export::ExportFitMode; + ui.label("Fit:"); + egui::ComboBox::from_id_salt("image_fit_mode") + .selected_text(self.image_settings.fit.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.image_settings.fit, ExportFitMode::Letterbox, ExportFitMode::Letterbox.name()); + ui.selectable_value(&mut self.image_settings.fit, ExportFitMode::Crop, ExportFitMode::Crop.name()); + ui.selectable_value(&mut self.image_settings.fit, ExportFitMode::Stretch, ExportFitMode::Stretch.name()); + }); + }); } /// Render advanced audio settings (sample rate, channels, bit depth, bitrate, time range) @@ -504,6 +540,19 @@ impl ExportDialog { } }); + // Fit mode — how the document maps into the export frame when the aspect ratios differ. + ui.horizontal(|ui| { + use lightningbeam_core::export::ExportFitMode; + ui.label("Fit:"); + egui::ComboBox::from_id_salt("video_fit_mode") + .selected_text(self.video_settings.fit.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.video_settings.fit, ExportFitMode::Letterbox, ExportFitMode::Letterbox.name()); + ui.selectable_value(&mut self.video_settings.fit, ExportFitMode::Crop, ExportFitMode::Crop.name()); + ui.selectable_value(&mut self.video_settings.fit, ExportFitMode::Stretch, ExportFitMode::Stretch.name()); + }); + }); + ui.horizontal(|ui| { ui.label("FPS:"); egui::ComboBox::from_id_salt("framerate") @@ -527,6 +576,36 @@ impl ExportDialog { }); }); + // Color range applies to H.264 (the VAAPI zero-copy encoder honors it). Limited/TV is the + // compatible default; Full/PC only looks right in players that read the full-range tag. + if matches!(self.video_settings.codec, VideoCodec::H264) { + ui.horizontal(|ui| { + ui.label("Color range:"); + egui::ComboBox::from_id_salt("video_color_range") + .selected_text(self.video_settings.color_range.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.video_settings.color_range, ColorRange::Limited, ColorRange::Limited.name()); + ui.selectable_value(&mut self.video_settings.color_range, ColorRange::Full, ColorRange::Full.name()); + }); + }); + } + + // HDR output: 10-bit BT.2020 PQ/HLG (HEVC). Forces H.265; software path (no zero-copy). + ui.horizontal(|ui| { + use lightningbeam_core::export::HdrExportMode; + ui.label("Dynamic range:"); + egui::ComboBox::from_id_salt("video_hdr_mode") + .selected_text(self.video_settings.hdr.name()) + .show_ui(ui, |ui| { + ui.selectable_value(&mut self.video_settings.hdr, HdrExportMode::Sdr, HdrExportMode::Sdr.name()); + ui.selectable_value(&mut self.video_settings.hdr, HdrExportMode::Pq, HdrExportMode::Pq.name()); + ui.selectable_value(&mut self.video_settings.hdr, HdrExportMode::Hlg, HdrExportMode::Hlg.name()); + }); + }); + if self.video_settings.hdr.is_hdr() { + ui.label(egui::RichText::new("HDR exports as 10-bit HEVC (H.265), BT.2020.").weak().small()); + } + ui.checkbox(&mut self.include_audio, "Include Audio"); ui.add_space(8.0); @@ -539,7 +618,7 @@ impl ExportDialog { fn render_time_range(&mut self, ui: &mut egui::Ui) { let (start_time, end_time) = match self.export_type { ExportType::Audio => (&mut self.audio_settings.start_time, &mut self.audio_settings.end_time), - ExportType::Image => return, // image uses a single time field, not a range + ExportType::Image | ExportType::Svg => return, // single time field, not a range ExportType::Video => (&mut self.video_settings.start_time, &mut self.video_settings.end_time), }; @@ -613,6 +692,7 @@ impl ExportDialog { } Some(ExportResult::Image(self.image_settings.clone(), output_path)) } + ExportType::Svg => Some(ExportResult::Svg(self.image_settings.time, output_path)), ExportType::Audio => { // Validate audio settings if let Err(err) = self.audio_settings.validate() { diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs b/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs index adcdb32..c5cc8a0 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/gpu_yuv.rs @@ -6,8 +6,8 @@ //! 8.3 MB RGBA) and — more importantly — the per-frame CPU `rgba_to_yuv420p` (swscale) //! is eliminated. //! -//! Color math is BT.709 **full-range** (JPEG range), matching the encoder color tags -//! set in `setup_video_encoder` (`Space::BT709` + `Range::JPEG`). +//! Color math is BT.709; the Y/chroma scale+offset (full vs limited range) is selected by +//! the `full_range` flag and must match the encoder color tags set in `setup_video_encoder`. //! //! Output buffer layout (tight, little-endian byte packing into `array`): //! - `[0, W*H)` Y plane, row stride `W` @@ -30,15 +30,36 @@ pub fn yuv420p_len(width: u32, height: u32) -> usize { y + 2 * c } +/// `(y_offset, y_scale, chroma_offset, chroma_scale)` as fractions of 255, selecting +/// limited (TV, 16–235 / 16–240) vs full (PC, 0–255) range. Mirrors `render_nv12`. +fn range_params(full_range: bool) -> [f32; 4] { + if full_range { + [0.0, 1.0, 128.0 / 255.0, 1.0] + } else { + [16.0 / 255.0, 219.0 / 255.0, 128.0 / 255.0, 224.0 / 255.0] + } +} + /// GPU compute pipeline: `Rgba8Unorm` texture → tight planar YUV420p storage buffer. pub struct GpuYuv { y_pipeline: wgpu::ComputePipeline, uv_pipeline: wgpu::ComputePipeline, bind_group_layout: wgpu::BindGroupLayout, + range_buffer: wgpu::Buffer, } impl GpuYuv { - pub fn new(device: &wgpu::Device) -> Self { + /// `full_range`: true → full/PC range (Y 0–255), false → limited/TV range (Y 16–235). + /// The encoder must tag the stream to match (`setup_video_encoder`'s `full_range`). + pub fn new(device: &wgpu::Device, full_range: bool) -> Self { + use wgpu::util::DeviceExt; + let params = range_params(full_range); + let range_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("gpu_yuv_range"), + contents: bytemuck::cast_slice(¶ms), + usage: wgpu::BufferUsages::UNIFORM, + }); + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("gpu_yuv_bgl"), entries: &[ @@ -64,6 +85,17 @@ impl GpuYuv { }, count: None, }, + // 2: range params (y_offset, y_scale, chroma_offset, chroma_scale) + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); @@ -93,6 +125,7 @@ impl GpuYuv { y_pipeline: mk("y_main"), uv_pipeline: mk("uv_main"), bind_group_layout, + range_buffer, } } @@ -118,6 +151,7 @@ impl GpuYuv { entries: &[ wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(rgba_view) }, wgpu::BindGroupEntry { binding: 1, resource: yuv_buffer.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: self.range_buffer.as_entire_binding() }, ], }); @@ -142,12 +176,12 @@ impl GpuYuv { /// CPU reference for the exact math/layout the shader produces — used by unit tests so /// the packing and BT.709 coefficients stay verifiable without a GPU. -#[cfg(test)] -fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec { +fn cpu_reference(rgba: &[u8], width: u32, height: u32, full_range: bool) -> Vec { let w = width as usize; let h = height as usize; let cw = w / 2; let ch = h / 2; + let [yo, ys, co, cs] = range_params(full_range); let mut out = vec![0u8; yuv420p_len(width, height)]; let to_byte = |v: f32| (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8; let px = |x: usize, y: usize| { @@ -158,7 +192,8 @@ fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec { for y in 0..h { for x in 0..w { let p = px(x, y); - out[y * w + x] = to_byte(0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2]); + let yy = 0.2126 * p[0] + 0.7152 * p[1] + 0.0722 * p[2]; + out[y * w + x] = to_byte(yo + ys * yy); } } // U/V (2x2 average) @@ -172,19 +207,21 @@ fn cpu_reference(rgba: &[u8], width: u32, height: u32) -> Vec { acc[0] += p[0]; acc[1] += p[1]; acc[2] += p[2]; } let a = [acc[0] / 4.0, acc[1] / 4.0, acc[2] / 4.0]; - let u = -0.1146 * a[0] - 0.3854 * a[1] + 0.5000 * a[2] + 0.5; - let v = 0.5000 * a[0] - 0.4542 * a[1] - 0.0458 * a[2] + 0.5; - out[y_size + cy * cw + cx] = to_byte(u); - out[y_size + uv_size + cy * cw + cx] = to_byte(v); + let uc = -0.1146 * a[0] - 0.3854 * a[1] + 0.5000 * a[2]; + let vc = 0.5000 * a[0] - 0.4542 * a[1] - 0.0458 * a[2]; + out[y_size + cy * cw + cx] = to_byte(co + cs * uc); + out[y_size + uv_size + cy * cw + cx] = to_byte(co + cs * vc); } } out } const SHADER: &str = r#" -// RGBA -> tight planar YUV420p (BT.709 full-range), packed 4 bytes/u32. +// RGBA -> tight planar YUV420p (BT.709), packed 4 bytes/u32. +// rng = (y_offset, y_scale, chroma_offset, chroma_scale): selects limited vs full range. @group(0) @binding(0) var input_rgba: texture_2d; @group(0) @binding(1) var out_buf: array; +@group(0) @binding(2) var rng: vec4; fn to_byte(v: f32) -> u32 { return u32(clamp(v, 0.0, 1.0) * 255.0 + 0.5); } @@ -201,7 +238,7 @@ fn y_main(@builtin(global_invocation_id) gid: vec3) { for (var i = 0u; i < 4u; i = i + 1u) { let c = textureLoad(input_rgba, vec2(x4 + i, y), 0).rgb; let yy = 0.2126 * c.r + 0.7152 * c.g + 0.0722 * c.b; - packed = packed | (to_byte(yy) << (8u * i)); + packed = packed | (to_byte(rng.x + rng.y * yy) << (8u * i)); } out_buf[(y * w + x4) / 4u] = packed; } @@ -230,10 +267,11 @@ fn uv_main(@builtin(global_invocation_id) gid: vec3) { let p01 = textureLoad(input_rgba, vec2(sx, sy + 1u), 0).rgb; let p11 = textureLoad(input_rgba, vec2(sx + 1u, sy + 1u), 0).rgb; let a = (p00 + p10 + p01 + p11) * 0.25; - let u = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b + 0.5; - let v = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b + 0.5; - up = up | (to_byte(u) << (8u * i)); - vp = vp | (to_byte(v) << (8u * i)); + // Centered chroma in [-0.5, 0.5], then map to range via (offset + scale*coef). + let uc = -0.1146 * a.r - 0.3854 * a.g + 0.5000 * a.b; + let vc = 0.5000 * a.r - 0.4542 * a.g - 0.0458 * a.b; + up = up | (to_byte(rng.z + rng.w * uc) << (8u * i)); + vp = vp | (to_byte(rng.z + rng.w * vc) << (8u * i)); } out_buf[(y_size + cy * cw + cx4) / 4u] = up; out_buf[(y_size + uv_size + cy * cw + cx4) / 4u] = vp; @@ -264,14 +302,14 @@ mod tests { fn reference_known_colors() { // 8x2 solid white → Y≈255, U≈V≈128. Solid black → Y=0, U=V≈128. let white = vec![255u8; 8 * 2 * 4]; - let out = cpu_reference(&white, 8, 2); + let out = cpu_reference(&white, 8, 2, true); let (cw, ch) = (4usize, 1usize); let y_size = 8 * 2; for &y in &out[..y_size] { assert!(y >= 254, "white Y={y}"); } for &u in &out[y_size..y_size + cw * ch] { assert!((u as i32 - 128).abs() <= 1, "white U={u}"); } let black = vec![0u8; 8 * 2 * 4]; - let out = cpu_reference(&black, 8, 2); + let out = cpu_reference(&black, 8, 2, true); for &y in &out[..y_size] { assert_eq!(y, 0); } for &v in &out[y_size + cw * ch..] { assert!((v as i32 - 128).abs() <= 1, "black V={v}"); } } @@ -280,7 +318,7 @@ mod tests { fn reference_red_bt709() { // Solid red (255,0,0): Y=0.2126*255≈54; V high, U low (full range). let red: Vec = (0..8 * 2).flat_map(|_| [255u8, 0, 0, 255]).collect(); - let out = cpu_reference(&red, 8, 2); + let out = cpu_reference(&red, 8, 2, true); assert!((out[0] as i32 - 54).abs() <= 1, "red Y={}", out[0]); let y_size = 8 * 2; let u = out[y_size]; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/hdr_frame.rs b/lightningbeam-ui/lightningbeam-editor/src/export/hdr_frame.rs new file mode 100644 index 0000000..4d78c50 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/export/hdr_frame.rs @@ -0,0 +1,308 @@ +//! 10-bit HDR frame production for video export (isolated from the SDR readback pipeline). +//! +//! Takes the compositor's Rgba16Float HDR accumulator and produces YUV420P10LE planes: +//! 1. GPU pass `linear_to_pq.wgsl` → PQ/HLG-encoded BT.2020 R'G'B' into an Rgba16Unorm texture +//! (the expensive per-pixel transfer + gamut work). +//! 2. Synchronous GPU→CPU readback of that texture. +//! 3. CPU BT.2020 R'G'B'→Y'CbCr (limited range), 4:2:0 average, 10-bit little-endian pack. +//! +//! Synchronous (no triple-buffering); HDR export favors correctness/simplicity over throughput. + +use lightningbeam_core::export::HdrExportMode; + +/// Round up to the wgpu copy row alignment (256 bytes). +fn align_256(n: u32) -> u32 { + (n + 255) & !255 +} + +pub struct HdrFramePipeline { + width: u32, + height: u32, + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + mode_buf: wgpu::Buffer, + /// PQ/HLG-encoded BT.2020 R'G'B' (Rgba16Unorm) render target. + enc_texture_view: wgpu::TextureView, + enc_texture: wgpu::Texture, + /// Staging buffer for readback; rows padded to 256-byte alignment. + staging: wgpu::Buffer, + padded_bytes_per_row: u32, +} + +impl HdrFramePipeline { + pub fn new(device: &wgpu::Device, width: u32, height: u32) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("linear_to_pq_shader"), + source: wgpu::ShaderSource::Wgsl(include_str!("shaders/linear_to_pq.wgsl").into()), + }); + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("linear_to_pq_bgl"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("linear_to_pq_pl"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("linear_to_pq_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + // Rgba16Float (not Unorm) so no TEXTURE_FORMAT_16BIT_NORM feature is needed; PQ/HLG + // values are in [0,1] where f16 has ~11 effective bits — ample for 10-bit output. + format: wgpu::TextureFormat::Rgba16Float, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("linear_to_pq_sampler"), + ..Default::default() + }); + + let mode_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("linear_to_pq_mode"), + size: 16, // vec4 + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + let enc_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("hdr_enc_texture"), + size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let enc_texture_view = enc_texture.create_view(&Default::default()); + + let padded_bytes_per_row = align_256(width * 8); // Rgba16Unorm = 8 bytes/texel + let staging = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("hdr_enc_staging"), + size: (padded_bytes_per_row * height) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + + Self { + width, + height, + pipeline, + bind_group_layout, + sampler, + mode_buf, + enc_texture_view, + enc_texture, + staging, + padded_bytes_per_row, + } + } + + /// Encode the composited HDR texture (`hdr_view`, Rgba16Float linear) to YUV420P10LE planes. + pub fn render_to_yuv10( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + hdr_view: &wgpu::TextureView, + mode: HdrExportMode, + ) -> (Vec, Vec, Vec) { + let mode_code: u32 = if matches!(mode, HdrExportMode::Hlg) { 1 } else { 0 }; + queue.write_buffer(&self.mode_buf, 0, bytemuck::cast_slice(&[mode_code, 0u32, 0, 0])); + + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("linear_to_pq_bg"), + layout: &self.bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(hdr_view) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&self.sampler) }, + wgpu::BindGroupEntry { binding: 2, resource: self.mode_buf.as_entire_binding() }, + ], + }); + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("hdr_frame_encoder"), + }); + { + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("linear_to_pq_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &self.enc_texture_view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + }); + rp.set_pipeline(&self.pipeline); + rp.set_bind_group(0, &bind_group, &[]); + rp.draw(0..3, 0..1); + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &self.enc_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &self.staging, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(self.padded_bytes_per_row), + rows_per_image: Some(self.height), + }, + }, + wgpu::Extent3d { width: self.width, height: self.height, depth_or_array_layers: 1 }, + ); + queue.submit(Some(encoder.finish())); + + // Synchronous map + wait. + let slice = self.staging.slice(..); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |r| { let _ = tx.send(r); }); + let _ = device.poll(wgpu::PollType::wait_indefinitely()); + let _ = rx.recv(); + + let w = self.width as usize; + let h = self.height as usize; + let mapped = slice.get_mapped_range(); + // Un-pad rows; decode f16 → f32 into a tight RGBA buffer. + let mut rgba = vec![0f32; w * h * 4]; + let row_bytes = w * 8; + for row in 0..h { + let src = row * self.padded_bytes_per_row as usize; + let dst = row * w * 4; + let bytes = &mapped[src..src + row_bytes]; + for px in 0..w * 4 { + let half = u16::from_le_bytes([bytes[px * 2], bytes[px * 2 + 1]]); + rgba[dst + px] = f16_to_f32(half); + } + } + drop(mapped); + self.staging.unmap(); + + rgba_to_yuv420p10le(&rgba, w, h) + } +} + +/// Decode an IEEE 754 half-float. Inputs are in [0,1] so the inf/NaN paths don't occur in practice. +fn f16_to_f32(h: u16) -> f32 { + let sign = (h >> 15) & 1; + let exp = (h >> 10) & 0x1f; + let mant = h & 0x3ff; + let v = if exp == 0 { + (mant as f32) * 2f32.powi(-24) // subnormal + } else if exp == 31 { + if mant == 0 { f32::INFINITY } else { f32::NAN } + } else { + (1.0 + mant as f32 / 1024.0) * 2f32.powi(exp as i32 - 15) + }; + if sign == 1 { -v } else { v } +} + +/// BT.2020 non-constant-luminance R'G'B'→Y'CbCr, limited range, 4:2:0, 10-bit little-endian. +/// Input R'G'B' is already gamma-encoded (PQ/HLG) in [0,1]. +fn rgba_to_yuv420p10le(rgba: &[f32], w: usize, h: usize) -> (Vec, Vec, Vec) { + const KR: f32 = 0.2627; + const KB: f32 = 0.0593; + let kg = 1.0 - KR - KB; + + let luma = |r: f32, g: f32, b: f32| KR * r + kg * g + KB * b; + // 10-bit limited: Y' [64,940] (scale 876), Cb/Cr center 512, excursion ±0.5 → scale 896. + let pack_y = |y: f32| ((y * 876.0 + 64.0).round().clamp(0.0, 1023.0)) as u16; + let pack_c = |c: f32| ((c * 896.0 + 512.0).round().clamp(0.0, 1023.0)) as u16; + + let mut y_plane = vec![0u8; w * h * 2]; + for j in 0..h { + for i in 0..w { + let p = (j * w + i) * 4; + let y10 = pack_y(luma(rgba[p], rgba[p + 1], rgba[p + 2])); + let o = (j * w + i) * 2; + y_plane[o] = (y10 & 0xff) as u8; + y_plane[o + 1] = (y10 >> 8) as u8; + } + } + + let (cw, ch) = (w / 2, h / 2); + let mut u_plane = vec![0u8; cw * ch * 2]; + let mut v_plane = vec![0u8; cw * ch * 2]; + for j in 0..ch { + for i in 0..cw { + let (mut cb, mut cr) = (0.0f32, 0.0f32); + for dy in 0..2 { + for dx in 0..2 { + let p = ((j * 2 + dy) * w + (i * 2 + dx)) * 4; + let (r, g, b) = (rgba[p], rgba[p + 1], rgba[p + 2]); + let yy = luma(r, g, b); + cb += (b - yy) / (2.0 * (1.0 - KB)); + cr += (r - yy) / (2.0 * (1.0 - KR)); + } + } + let cb10 = pack_c(cb / 4.0); + let cr10 = pack_c(cr / 4.0); + let o = (j * cw + i) * 2; + u_plane[o] = (cb10 & 0xff) as u8; + u_plane[o + 1] = (cb10 >> 8) as u8; + v_plane[o] = (cr10 & 0xff) as u8; + v_plane[o + 1] = (cr10 >> 8) as u8; + } + } + + (y_plane, u_plane, v_plane) +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs index 62381bc..d5b2851 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/mod.rs @@ -11,6 +11,7 @@ pub mod readback_pipeline; pub mod perf_metrics; pub mod cpu_yuv_converter; pub mod gpu_yuv; +pub mod hdr_frame; use lightningbeam_core::export::{AudioExportSettings, ImageExportSettings, VideoExportSettings, ExportProgress}; use lightningbeam_core::document::Document; @@ -52,6 +53,13 @@ pub struct VideoExportState { width: u32, /// Export height in pixels height: u32, + /// HDR output mode — HDR uses a synchronous 10-bit path instead of the async RGBA pipeline. + hdr: lightningbeam_core::export::HdrExportMode, + /// How the document is fit into the export frame (stretch/letterbox/crop). + fit: lightningbeam_core::export::ExportFitMode, + /// SDR color range: true = full (PC, 0–255), false = limited (TV, 16–235). The YUV + /// converters and the encoder color tag must agree on this. + full_range: bool, /// Channel to send rendered frames to encoder thread frame_tx: Option>, /// HDR GPU resources for compositing pipeline (effects, color conversion) @@ -79,6 +87,10 @@ struct ZeroCopyVideo { gpu_resources: video_exporter::ExportGpuResources, /// Reused RGBA target (RENDER_ATTACHMENT | TEXTURE_BINDING) on the encoder's device. rgba: wgpu::Texture, + /// True when running on the shared device → compositing can consume hardware-decoded GPU frames. + on_shared_device: bool, + /// How the document is fit into the export frame (stretch/letterbox/crop). + fit: lightningbeam_core::export::ExportFitMode, } /// State for a single-frame image export (runs on the GPU render thread, one frame per update). @@ -589,6 +601,7 @@ impl ExportOrchestrator { // ── First call: render the frame to the GPU output texture ──────── let w = state.width; let h = state.height; + let fit = state.settings.fit; if state.gpu_resources.is_none() { state.gpu_resources = Some(video_exporter::ExportGpuResources::new(device, w, h)); @@ -622,6 +635,8 @@ impl ExportOrchestrator { floating_selection, state.settings.allow_transparency, raster_store, + true, // image export composites on the shared device + fit, )?; queue.submit(Some(encoder.finish())); @@ -854,6 +869,9 @@ impl ExportOrchestrator { width: u32, height: u32, ) -> (std::thread::JoinHandle<()>, VideoExportState) { + let hdr = settings.hdr; + let fit = settings.fit; + let full_range = settings.color_range.is_full(); let handle = std::thread::spawn(move || { Self::run_video_encoder(settings, output_path, frame_rx, progress_tx, cancel_flag, total_frames); }); @@ -866,6 +884,9 @@ impl ExportOrchestrator { framerate, width, height, + hdr, + fit, + full_range, frame_tx: Some(frame_tx), gpu_resources: None, readback_pipeline: None, @@ -889,17 +910,35 @@ impl ExportOrchestrator { height: u32, framerate: f64, output_path: &std::path::Path, + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Option { - if !matches!(settings.codec, lightningbeam_core::export::VideoCodec::H264) { + // Zero-copy is 8-bit H.264 only; HDR needs the 10-bit HEVC software path. + if settings.hdr.is_hdr() + || !matches!(settings.codec, lightningbeam_core::export::VideoCodec::H264) + { return None; } - let encoder = match gpu_video_encoder::encoder::ZeroCopyEncoder::new( - width, - height, - framerate.round() as i32, - settings.quality.bitrate_kbps(), - output_path, - ) { + let bitrate = settings.quality.bitrate_kbps(); + let fr = framerate.round() as i32; + let full = settings.color_range.is_full(); + println!("🎬 [EXPORT] zero-copy H.264 color range: {} (full_range={})", + settings.color_range.name(), full); + let on_shared_device = shared_device.is_some(); + // Prefer the shared device → decode→composite→encode stay GPU-resident on one device. + // Without it, the encoder builds its own device (decode still downloads to CPU per Step 1's + // hardware_ok=false on this path). + let encoder_result = match shared_device { + Some((device, queue, adapter)) => { + println!("🎬 [EXPORT] zero-copy on shared device (GPU-resident decode)"); + gpu_video_encoder::encoder::ZeroCopyEncoder::new_on_device( + device, queue, adapter, width, height, fr, bitrate, output_path, full, + ) + } + None => gpu_video_encoder::encoder::ZeroCopyEncoder::new( + width, height, fr, bitrate, output_path, full, + ), + }; + let encoder = match encoder_result { Ok(e) => e, Err(e) => { println!("🎬 [EXPORT] zero-copy unavailable ({e}); software path"); @@ -935,7 +974,7 @@ impl ExportOrchestrator { view_formats: &[], }); println!("🎬 [EXPORT] zero-copy VAAPI H.264 enabled"); - Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba }) + Some(ZeroCopyVideo { encoder, renderer, gpu_resources, rgba, on_shared_device, fit: settings.fit }) } /// Start a video export in the background. @@ -953,6 +992,7 @@ impl ExportOrchestrator { /// # Returns /// Ok(()) on success, Err on failure #[allow(clippy::too_many_arguments)] + #[allow(clippy::too_many_arguments)] pub fn start_video_export( &mut self, settings: VideoExportSettings, @@ -961,6 +1001,8 @@ impl ExportOrchestrator { video_manager: Arc>, raster_store: lightningbeam_core::raster_store::RasterStore, container_path: Option, + // The shared VAAPI device, `Some` only when active → zero-copy encode runs on it. + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Result<(), String> { println!("🎬 [VIDEO EXPORT] Starting video export"); @@ -988,7 +1030,7 @@ impl ExportOrchestrator { // success it returns here; otherwise we fall through to the software encoder thread. #[cfg(target_os = "linux")] { - if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path) { + if let Some(zc) = Self::try_build_zero_copy(&settings, width, height, framerate, &output_path, shared_device) { drop(frame_rx); let document_snapshot = document.clone(); let mut image_cache = ImageCache::new(); @@ -1052,6 +1094,8 @@ impl ExportOrchestrator { video_manager: Arc>, raster_store: lightningbeam_core::raster_store::RasterStore, container_path: Option, + // The shared VAAPI device, `Some` only when active → zero-copy encode runs on it. + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, ) -> Result<(), String> { println!("🎬🎵 [PARALLEL EXPORT] Starting parallel video+audio export"); @@ -1118,7 +1162,7 @@ impl ExportOrchestrator { // by `render_next_video_frame` on the UI thread. #[cfg(target_os = "linux")] let (video_thread, video_state) = match Self::try_build_zero_copy( - &video_settings, video_width, video_height, video_framerate, &temp_video_path, + &video_settings, video_width, video_height, video_framerate, &temp_video_path, shared_device, ) { Some(zc) => { drop(frame_rx); // the zero-copy path renders internally, no frame channel @@ -1241,6 +1285,44 @@ impl ExportOrchestrator { let width = state.width; let height = state.height; + let fit = state.fit; + + // HDR path: synchronous 10-bit render (composite → PQ/HLG → readback → 10-bit YUV), one + // frame per call. Bypasses the SDR async RGBA pipeline (which is 8-bit only). + if state.hdr.is_hdr() { + if state.gpu_resources.is_none() { + println!("🎬 [VIDEO EXPORT] Initializing HDR GPU resources {}x{} ({})", width, height, state.hdr.name()); + state.gpu_resources = Some(video_exporter::ExportGpuResources::new(device, width, height)); + } + if state.current_frame < state.total_frames { + let timestamp = state.start_time + (state.current_frame as f64 / state.framerate); + let gpu_resources = state.gpu_resources.as_mut().unwrap(); + let (y, u, v) = video_exporter::render_frame_to_yuv10_hdr( + document, timestamp, width, height, + device, queue, renderer, image_cache, video_manager, + gpu_resources, state.hdr, fit, raster_store, + )?; + if let Some(tx) = &state.frame_tx { + tx.send(VideoFrameMessage::Frame { + frame_num: state.current_frame, + timestamp, + y_plane: y, + u_plane: u, + v_plane: v, + }).map_err(|_| "Failed to send HDR frame")?; + } + state.current_frame += 1; + } + if state.current_frame >= state.total_frames { + println!("🎬 [VIDEO EXPORT] HDR complete: {} frames", state.total_frames); + if let Some(tx) = state.frame_tx.take() { + tx.send(VideoFrameMessage::Done).ok(); + } + state.gpu_resources = None; + return Ok(false); + } + return Ok(true); + } // Initialize GPU resources and readback pipeline on first frame if state.gpu_resources.is_none() { @@ -1258,8 +1340,8 @@ impl ExportOrchestrator { if !gpu_yuv_tight { println!("🎬 [VIDEO EXPORT] YUV planes are padded at {width}x{height}; using CPU YUV path"); } - state.readback_pipeline = Some(readback_pipeline::ReadbackPipeline::new(device, queue, width, height, gpu_yuv_tight)); - state.cpu_yuv_converter = Some(cpu_yuv_converter::CpuYuvConverter::new(width, height)?); + state.readback_pipeline = Some(readback_pipeline::ReadbackPipeline::new(device, queue, width, height, gpu_yuv_tight, state.full_range)); + state.cpu_yuv_converter = Some(cpu_yuv_converter::CpuYuvConverter::new(width, height, state.full_range)?); println!("🚀 [ASYNC PIPELINE] Triple-buffered pipeline initialized"); println!("🚀 [CPU YUV] swscale converter initialized"); } @@ -1347,6 +1429,8 @@ impl ExportOrchestrator { None, // No floating selection during video export false, // Video export is never transparent raster_store, + true, // software export composites on the shared device → may use HW frames + fit, )?; let render_end = Instant::now(); @@ -1437,6 +1521,7 @@ impl ExportOrchestrator { let rgba_view = zc.rgba.create_view(&Default::default()); let t0 = std::time::Instant::now(); + let fit = zc.fit; let cmd = match video_exporter::render_frame_to_gpu_rgba( &mut document, timestamp, @@ -1452,6 +1537,8 @@ impl ExportOrchestrator { None, false, Some(&raster_store), + zc.on_shared_device, // GPU-resident decode only when on the shared device + fit, ) { Ok(cmd) => cmd, Err(e) => { @@ -1555,13 +1642,33 @@ impl ExportOrchestrator { // Initialize FFmpeg ffmpeg_next::init().map_err(|e| format!("Failed to initialize FFmpeg: {}", e))?; - // Convert codec enum to FFmpeg codec ID - let codec_id = match settings.codec { - VideoCodec::H264 => ffmpeg_next::codec::Id::H264, - VideoCodec::H265 => ffmpeg_next::codec::Id::HEVC, - VideoCodec::VP8 => ffmpeg_next::codec::Id::VP8, - VideoCodec::VP9 => ffmpeg_next::codec::Id::VP9, - VideoCodec::ProRes422 => ffmpeg_next::codec::Id::PRORES, + // Convert codec enum to FFmpeg codec ID. HDR requires 10-bit HEVC (Main10), so force HEVC + // regardless of the chosen codec when an HDR mode is selected. + let codec_id = if settings.hdr.is_hdr() { + // HEVC can only be muxed into MP4/MOV, not WebM — reject the incompatible combo up + // front with a clear message instead of letting the muxer fail cryptically. + if settings.codec.container_format() == "webm" { + return Err(format!( + "HDR export needs H.265/HEVC in an MP4 container, but {} uses WebM. \ + Pick H.265 (or H.264) for HDR.", + settings.codec.name() + )); + } + if !matches!(settings.codec, VideoCodec::H265) { + println!( + "⚠️ [ENCODER] HDR selected: overriding codec {} → H.265/HEVC (Main10)", + settings.codec.name() + ); + } + ffmpeg_next::codec::Id::HEVC + } else { + match settings.codec { + VideoCodec::H264 => ffmpeg_next::codec::Id::H264, + VideoCodec::H265 => ffmpeg_next::codec::Id::HEVC, + VideoCodec::VP8 => ffmpeg_next::codec::Id::VP8, + VideoCodec::VP9 => ffmpeg_next::codec::Id::VP9, + VideoCodec::ProRes422 => ffmpeg_next::codec::Id::PRORES, + } }; // Get bitrate from quality settings @@ -1604,8 +1711,17 @@ impl ExportOrchestrator { height, framerate, bitrate_kbps, + settings.hdr, + settings.color_range.is_full(), )?; + // Pixel format the encoder frames are built in (matches setup_video_encoder). + let pixel_format = if settings.hdr.is_hdr() { + ffmpeg_next::format::Pixel::YUV420P10LE + } else { + ffmpeg_next::format::Pixel::YUV420P + }; + // Create output file let mut output = ffmpeg_next::format::output(&output_path) .map_err(|e| format!("Failed to create output file: {}", e))?; @@ -1634,6 +1750,7 @@ impl ExportOrchestrator { width, height, timestamp, + pixel_format, )?; // Send progress update for first frame @@ -1661,6 +1778,7 @@ impl ExportOrchestrator { width, height, timestamp, + pixel_format, )?; frames_encoded += 1; @@ -1705,29 +1823,33 @@ impl ExportOrchestrator { width: u32, height: u32, timestamp: f64, + pixel_format: ffmpeg_next::format::Pixel, ) -> Result<(), String> { - // YUV planes already converted by GPU (no CPU conversion needed) + // YUV planes already converted (8-bit YUV420P, or 10-bit YUV420P10LE for HDR). - // Create FFmpeg video frame - let mut video_frame = ffmpeg_next::frame::Video::new( - ffmpeg_next::format::Pixel::YUV420P, - width, - height, - ); + // Create FFmpeg video frame in the encoder's pixel format. + let mut video_frame = ffmpeg_next::frame::Video::new(pixel_format, width, height); - // Copy YUV planes to frame - // Use safe slice copy - LLVM optimizes this to memcpy, same performance as copy_nonoverlapping - let y_dest = video_frame.data_mut(0); - let y_len = y_plane.len().min(y_dest.len()); - y_dest[..y_len].copy_from_slice(&y_plane[..y_len]); - - let u_dest = video_frame.data_mut(1); - let u_len = u_plane.len().min(u_dest.len()); - u_dest[..u_len].copy_from_slice(&u_plane[..u_len]); - - let v_dest = video_frame.data_mut(2); - let v_len = v_plane.len().min(v_dest.len()); - v_dest[..v_len].copy_from_slice(&v_plane[..v_len]); + // Copy each plane row-by-row honoring the frame's stride (10-bit / arbitrary widths can have + // row padding that a flat copy would misalign). `bytes_per_row` = samples × sample size. + let ten_bit = matches!(pixel_format, ffmpeg_next::format::Pixel::YUV420P10LE); + let sample_bytes = if ten_bit { 2usize } else { 1usize }; + let copy_plane = |frame: &mut ffmpeg_next::frame::Video, idx: usize, src: &[u8], w: usize, h: usize| { + let bytes_per_row = w * sample_bytes; + let stride = frame.stride(idx); + let dst = frame.data_mut(idx); + for row in 0..h { + let s = row * bytes_per_row; + let d = row * stride; + let n = bytes_per_row.min(src.len().saturating_sub(s)).min(dst.len().saturating_sub(d)); + if n == 0 { break; } + dst[d..d + n].copy_from_slice(&src[s..s + n]); + } + }; + let (w, h) = (width as usize, height as usize); + copy_plane(&mut video_frame, 0, y_plane, w, h); + copy_plane(&mut video_frame, 1, u_plane, w / 2, h / 2); + copy_plane(&mut video_frame, 2, v_plane, w / 2, h / 2); // Set PTS (presentation timestamp) in encoder's time base // Encoder time base is 1/(framerate * 1000), so PTS = timestamp * (framerate * 1000) diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs b/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs index 5997e5e..df552f3 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/readback_pipeline.rs @@ -91,12 +91,12 @@ impl ReadbackPipeline { /// `enable_gpu_yuv` should be `true` only when the caller has verified the encoder's /// `YUV420P` plane strides are tight (== width / width-2), so the packed GPU planes /// drop straight into the `AVFrame` without row misalignment. - pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, width: u32, height: u32, enable_gpu_yuv: bool) -> Self { + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue, width: u32, height: u32, enable_gpu_yuv: bool, full_range: bool) -> Self { let (readback_tx, readback_rx) = channel(); // GPU YUV conversion when enabled AND the dimensions fit the packed shader; else RGBA + CPU. let gpu_yuv = if enable_gpu_yuv && super::gpu_yuv::supports(width, height) { - Some(super::gpu_yuv::GpuYuv::new(device)) + Some(super::gpu_yuv::GpuYuv::new(device, full_range)) } else { None }; diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/shaders/linear_to_pq.wgsl b/lightningbeam-ui/lightningbeam-editor/src/export/shaders/linear_to_pq.wgsl new file mode 100644 index 0000000..6b71562 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/export/shaders/linear_to_pq.wgsl @@ -0,0 +1,78 @@ +// Linear-HDR → PQ/HLG BT.2020 encode (for 10-bit HDR video export). +// +// Input: the compositor's Rgba16Float HDR accumulator — PREMULTIPLIED scene-linear, BT.709 +// primaries, graphics white = 1.0, HDR highlights > 1.0. +// Output: gamma-encoded R'G'B' in BT.2020 primaries, PQ (mode 0) or HLG (mode 1), to an +// Rgba16Unorm target. A later CPU pass does only BT.2020 R'G'B'→Y'CbCr (no transfer) + 4:2:0 + 10-bit. +// +// This is the encode inverse of panes/shaders/nv12_blit.wgsl's decode (203-nit PQ white; HLG +// reference white at signal 0.75), so a decode→encode round-trip is the identity. + +@group(0) @binding(0) var input_tex: texture_2d; +@group(0) @binding(1) var input_sampler: sampler; +@group(0) @binding(2) var params: vec4; // .x = mode (0 = PQ, 1 = HLG) + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var out: VertexOutput; + let x = f32((vertex_index & 1u) << 1u); + let y = f32(vertex_index & 2u); + out.position = vec4(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0); + out.uv = vec2(x, y); + return out; +} + +// BT.709 → BT.2020 primaries, linear light (ITU-R BT.2087). +fn bt709_to_bt2020(c: vec3) -> vec3 { + let r = 0.627404 * c.r + 0.329283 * c.g + 0.043313 * c.b; + let g = 0.069097 * c.r + 0.919540 * c.g + 0.011362 * c.b; + let b = 0.016391 * c.r + 0.088013 * c.g + 0.895595 * c.b; + return vec3(r, g, b); +} + +// SMPTE ST 2084 (PQ) OETF: scene-linear (white = 1.0 = 203 nits) → PQ code [0,1]. +fn pq_oetf(lin: vec3) -> vec3 { + let nits = max(lin, vec3(0.0)) * 203.0; + let ln = min(nits / 10000.0, vec3(1.0)); + let m1 = 0.1593017578125; + let m2 = 78.84375; + let c1 = 0.8359375; + let c2 = 18.8515625; + let c3 = 18.6875; + let lm = pow(ln, vec3(m1)); + return pow((vec3(c1) + c2 * lm) / (vec3(1.0) + c3 * lm), vec3(m2)); +} + +// ARIB STD-B67 (HLG) OETF: scene-linear (white = 1.0) → HLG signal [0,1]. Reference white maps to +// signal 0.75 (matching the decode's /0.26496256 normalization). Display OOTF omitted (scene-referred). +fn hlg_oetf(lin: vec3) -> vec3 { + let a = 0.17883277; + let b = 0.28466892; + let c = 0.55991073; + let e = clamp(lin * 0.26496256, vec3(0.0), vec3(1.0)); + let lo = sqrt(3.0 * e); + let hi = a * log(12.0 * e - vec3(b)) + vec3(c); + return select(lo, hi, e > vec3(1.0 / 12.0)); +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + // Compositor stores PREMULTIPLIED linear; unpremultiply to straight (video is opaque, a≈1). + let texel = textureSample(input_tex, input_sampler, in.uv); + let a = texel.a; + let straight = select(texel.rgb / a, vec3(0.0), a <= 0.0); + + let bt2020 = max(bt709_to_bt2020(straight), vec3(0.0)); + var enc: vec3; + if params.x == 1u { + enc = hlg_oetf(bt2020); + } else { + enc = pq_oetf(bt2020); + } + return vec4(enc, 1.0); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 86bc8d3..b27694f 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -16,6 +16,30 @@ use lightningbeam_core::gpu::{ SrgbToLinearConverter, EffectProcessor, YuvConverter, HDR_FORMAT, }; +/// The document→export-pixels transform for a given fit mode. Stretch distorts to fill; Letterbox +/// scales uniformly to fit (centered, black bars); Crop scales uniformly to fill (centered, trims). +pub fn export_base_transform( + doc_w: f64, + doc_h: f64, + out_w: f64, + out_h: f64, + fit: lightningbeam_core::export::ExportFitMode, +) -> vello::kurbo::Affine { + use lightningbeam_core::export::ExportFitMode; + use vello::kurbo::Affine; + if doc_w <= 0.0 || doc_h <= 0.0 { + return Affine::IDENTITY; + } + let (sx, sy) = (out_w / doc_w, out_h / doc_h); + match fit { + ExportFitMode::Stretch => Affine::scale_non_uniform(sx, sy), + ExportFitMode::Letterbox | ExportFitMode::Crop => { + let s = if matches!(fit, ExportFitMode::Letterbox) { sx.min(sy) } else { sx.max(sy) }; + Affine::translate(((out_w - doc_w * s) / 2.0, (out_h - doc_h * s) / 2.0)) * Affine::scale(s) + } + } +} + /// Reusable frame buffers to avoid allocations struct FrameBuffers { /// RGBA buffer from GPU readback (width * height * 4 bytes) @@ -75,14 +99,25 @@ pub struct ExportGpuResources { pub staging_buffer: wgpu::Buffer, /// Linear to sRGB blit pipeline for final output pub linear_to_srgb_pipeline: wgpu::RenderPipeline, + /// Variant with highlight rolloff (document HDR output mode = Highlight rolloff). + pub linear_to_srgb_pipeline_rolloff: wgpu::RenderPipeline, /// Bind group layout for linear to sRGB blit pub linear_to_srgb_bind_group_layout: wgpu::BindGroupLayout, /// Sampler for linear to sRGB conversion pub linear_to_srgb_sampler: wgpu::Sampler, /// Canvas blit pipeline for raster/video/float layers (bypasses Vello). pub canvas_blit: crate::gpu_brush::CanvasBlitPipeline, + /// NV12→linear blit for hardware-decoded video frames (export on the shared device). + pub nv12_blit: crate::nv12_blit::Nv12BlitPipeline, /// Per-keyframe GPU texture cache for raster layers during export. pub raster_cache: std::collections::HashMap, + /// Cached HDR accumulator state after the (static) background is composited in. The document + /// background doesn't change across an export, so it's rendered once and restored with a cheap + /// texture copy each frame instead of a full Vello render + 2 passes/submits. `None` until the + /// first frame; invalidated on resize. + cached_bg_hdr: Option, + /// HDR encode pipeline (linear→PQ/HLG BT.2020 → 10-bit YUV). Lazily built on the first HDR frame. + hdr_pipeline: Option, } impl ExportGpuResources { @@ -108,7 +143,8 @@ impl ExportGpuResources { format: HDR_FORMAT, usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_SRC, + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::COPY_DST, // restore cached background each frame view_formats: &[], }); let hdr_texture_view = hdr_texture.create_view(&wgpu::TextureViewDescriptor::default()); @@ -230,6 +266,41 @@ impl ExportGpuResources { cache: None, }); + // Highlight-rolloff variant: identical but the `fs_main_rolloff` entry point. + let linear_to_srgb_pipeline_rolloff = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("linear_to_srgb_pipeline_rolloff"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main_rolloff"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + polygon_mode: wgpu::PolygonMode::Fill, + unclipped_depth: false, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + let linear_to_srgb_sampler = device.create_sampler(&wgpu::SamplerDescriptor { label: Some("linear_to_srgb_sampler"), address_mode_u: wgpu::AddressMode::ClampToEdge, @@ -242,6 +313,7 @@ impl ExportGpuResources { }); let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device); + let nv12_blit = crate::nv12_blit::Nv12BlitPipeline::new(device); Self { buffer_pool, @@ -257,10 +329,14 @@ impl ExportGpuResources { yuv_texture_view, staging_buffer, linear_to_srgb_pipeline, + linear_to_srgb_pipeline_rolloff, linear_to_srgb_bind_group_layout, linear_to_srgb_sampler, canvas_blit, + nv12_blit, raster_cache: std::collections::HashMap::new(), + cached_bg_hdr: None, + hdr_pipeline: None, } } @@ -279,10 +355,12 @@ impl ExportGpuResources { format: HDR_FORMAT, usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING - | wgpu::TextureUsages::COPY_SRC, + | wgpu::TextureUsages::COPY_SRC + | wgpu::TextureUsages::COPY_DST, view_formats: &[], }); self.hdr_texture_view = self.hdr_texture.create_view(&wgpu::TextureViewDescriptor::default()); + self.cached_bg_hdr = None; // dimensions changed — rebuild the background cache } } @@ -331,6 +409,32 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { return vec4(srgb, a); } + +// Highlight rolloff: identity below the knee, smooth C1 rolloff [knee,∞)→[knee,1) above (recovers +// super-white HDR detail). SDR below the knee is untouched. Mirrors panes/shaders/linear_to_srgb.wgsl. +fn highlight_rolloff_ch(x: f32) -> f32 { + let knee = 0.8; + if x <= knee { + return x; + } + let headroom = 1.0 - knee; + return knee + headroom * (1.0 - exp(-(x - knee) / headroom)); +} + +// Variant of fs_main with highlight rolloff (document HDR output mode = Highlight rolloff). +@fragment +fn fs_main_rolloff(in: VertexOutput) -> @location(0) vec4 { + let src = textureSample(source_tex, source_sampler, in.uv); + let a = src.a; + let straight = select(src.rgb / a, vec3(0.0), a <= 0.0); + let rolled = vec3( + highlight_rolloff_ch(straight.r), + highlight_rolloff_ch(straight.g), + highlight_rolloff_ch(straight.b), + ); + let srgb = linear_to_srgb(rolled); + return vec4(srgb, a); +} "#; /// Convert RGBA8 pixels to YUV420p format using BT.709 color space @@ -455,6 +559,8 @@ pub fn setup_video_encoder( height: u32, framerate: f64, bitrate_kbps: u32, + hdr: lightningbeam_core::export::HdrExportMode, + full_range: bool, ) -> Result<(ffmpeg::encoder::Video, ffmpeg::Codec), String> { // Try to find codec by ID first println!("🔍 Looking for codec: {:?}", codec_id); @@ -510,31 +616,46 @@ pub fn setup_video_encoder( // Configure encoder parameters BEFORE opening (critical!) encoder.set_width(aligned_width); encoder.set_height(aligned_height); - encoder.set_format(ffmpeg::format::Pixel::YUV420P); + // HDR encodes 10-bit BT.2020 (limited range); SDR keeps 8-bit full-range BT.709. + if hdr.is_hdr() { + encoder.set_format(ffmpeg::format::Pixel::YUV420P10LE); + } else { + encoder.set_format(ffmpeg::format::Pixel::YUV420P); + } encoder.set_time_base(ffmpeg::Rational(1, (framerate * 1000.0) as i32)); encoder.set_frame_rate(Some(ffmpeg::Rational(framerate as i32, 1))); encoder.set_bit_rate((bitrate_kbps * 1000) as usize); encoder.set_gop(framerate as u32); // 1 second GOP - // Tag the color metadata so players interpret the YUV correctly. Our - // RGB→YUV conversion uses the BT.709 matrix with FULL-range (0–255) luma - // and no transfer applied to the already-sRGB-encoded RGB. Tagging this - // as full-range BT.709 (matrix/primaries/transfer) prevents the level/ - // hue shift that occurs when a player assumes limited-range or BT.601. - // colorspace (matrix) and range have safe setters; primaries and trc are - // generic AVCodecContext options set via the open dictionary below. - encoder.set_colorspace(ffmpeg::color::Space::BT709); - encoder.set_color_range(ffmpeg::color::Range::JPEG); // full range - - println!("📐 Video dimensions: {}×{} (aligned to {}×{} for H.264)", - width, height, aligned_width, aligned_height); - - // Open encoder with codec (like working MP3 export). color_primaries and - // color_trc have no typed setter on the encoder, so pass them as generic - // AVCodecContext options (BT.709) through the open dictionary. + // Tag the color metadata so players interpret the YUV correctly. + // SDR: our RGB→YUV uses the BT.709 matrix with FULL-range (0–255) luma and no transfer applied + // to the already-sRGB-encoded RGB, so tag full-range BT.709 to avoid level/hue shifts. + // HDR: BT.2020 non-constant-luminance matrix, LIMITED range (standard for HDR10/HLG), with the + // PQ or HLG transfer; the 10-bit YUV is produced from PQ/HLG-encoded BT.2020 RGB. let mut color_opts = ffmpeg::Dictionary::new(); - color_opts.set("color_primaries", "bt709"); - color_opts.set("color_trc", "bt709"); + if hdr.is_hdr() { + encoder.set_colorspace(ffmpeg::color::Space::BT2020NCL); + encoder.set_color_range(ffmpeg::color::Range::MPEG); // limited + color_opts.set("color_primaries", "bt2020"); + color_opts.set("color_trc", hdr.transfer_name()); + // HEVC 10-bit profile (the only HDR-capable codec we wire up). + color_opts.set("profile", "main10"); + } else { + encoder.set_colorspace(ffmpeg::color::Space::BT709); + // Range must match what the YUV converters (gpu_yuv / cpu_yuv) actually produce. + encoder.set_color_range(if full_range { + ffmpeg::color::Range::JPEG // full (PC, 0–255) + } else { + ffmpeg::color::Range::MPEG // limited (TV, 16–235) + }); + color_opts.set("color_primaries", "bt709"); + color_opts.set("color_trc", "bt709"); + } + + println!("📐 Video dimensions: {}×{} (aligned to {}×{}){}", + width, height, aligned_width, aligned_height, + if hdr.is_hdr() { " [HDR 10-bit BT.2020]" } else { "" }); + let encoder = encoder .open_as_with(codec, color_opts) .map_err(|e| format!("Failed to open video encoder: {}", e))?; @@ -748,29 +869,74 @@ fn composite_document_to_hdr( antialiasing_method: vello::AaConfig::Area, }; - // --- Background --- - let bg_srgb = gpu_resources.buffer_pool.acquire(device, layer_spec); - let bg_hdr = gpu_resources.buffer_pool.acquire(device, hdr_spec); - if let (Some(bg_srgb_view), Some(bg_hdr_view)) = ( - gpu_resources.buffer_pool.get_view(bg_srgb), - gpu_resources.buffer_pool.get_view(bg_hdr), - ) { - renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &layer_render_params) - .map_err(|e| format!("Failed to render background: {e}"))?; - let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_srgb_to_linear") }); - gpu_resources.srgb_to_linear.convert(device, &mut enc, bg_srgb_view, bg_hdr_view); + let prof = render_profile_enabled(); + let t_c0 = std::time::Instant::now(); + + // --- Background (cached) --- + // The document background is static across an export, so render it through Vello exactly once + // (into the accumulator) and snapshot the result; every later frame restores it with a single + // GPU texture copy instead of a Vello render + sRGB-convert + composite (+2 submits). + let bg_cached = matches!( + &gpu_resources.cached_bg_hdr, + Some(t) if t.width() == width && t.height() == height + ); + let copy_size = wgpu::Extent3d { width, height, depth_or_array_layers: 1 }; + if bg_cached { + // Restore the cached background into the accumulator. + let cached = gpu_resources.cached_bg_hdr.as_ref().unwrap(); + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_restore") }); + enc.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { texture: cached, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyTextureInfo { texture: &gpu_resources.hdr_texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + copy_size, + ); queue.submit(Some(enc.finish())); - let bg_layer = CompositorLayer::normal(bg_hdr, 1.0); - let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_composite") }); - // When transparency is allowed, start from transparent black so the background's - // native alpha is preserved. Otherwise force an opaque black underlay. - let clear = if allow_transparency { [0.0, 0.0, 0.0, 0.0] } else { [0.0, 0.0, 0.0, 1.0] }; - gpu_resources.compositor.composite(device, queue, &mut enc, &[bg_layer], - &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, Some(clear)); + } else { + // First frame (or after a resize): full background render into the accumulator. + let bg_srgb = gpu_resources.buffer_pool.acquire(device, layer_spec); + let bg_hdr = gpu_resources.buffer_pool.acquire(device, hdr_spec); + if let (Some(bg_srgb_view), Some(bg_hdr_view)) = ( + gpu_resources.buffer_pool.get_view(bg_srgb), + gpu_resources.buffer_pool.get_view(bg_hdr), + ) { + renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &layer_render_params) + .map_err(|e| format!("Failed to render background: {e}"))?; + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_srgb_to_linear") }); + gpu_resources.srgb_to_linear.convert(device, &mut enc, bg_srgb_view, bg_hdr_view); + queue.submit(Some(enc.finish())); + let bg_layer = CompositorLayer::normal(bg_hdr, 1.0); + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_composite") }); + // When transparency is allowed, start from transparent black so the background's + // native alpha is preserved. Otherwise force an opaque black underlay. + let clear = if allow_transparency { [0.0, 0.0, 0.0, 0.0] } else { [0.0, 0.0, 0.0, 1.0] }; + gpu_resources.compositor.composite(device, queue, &mut enc, &[bg_layer], + &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, Some(clear)); + queue.submit(Some(enc.finish())); + } + gpu_resources.buffer_pool.release(bg_srgb); + gpu_resources.buffer_pool.release(bg_hdr); + + // Snapshot the composited background for reuse on subsequent frames. + let cached = device.create_texture(&wgpu::TextureDescriptor { + label: Some("export_cached_bg_hdr"), + size: copy_size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: HDR_FORMAT, + usage: wgpu::TextureUsages::COPY_SRC | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_bg_snapshot") }); + enc.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { texture: &gpu_resources.hdr_texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::TexelCopyTextureInfo { texture: &cached, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + copy_size, + ); queue.submit(Some(enc.finish())); + gpu_resources.cached_bg_hdr = Some(cached); } - gpu_resources.buffer_pool.release(bg_srgb); - gpu_resources.buffer_pool.release(bg_hdr); + let t_bg = std::time::Instant::now(); // --- Layers --- for rendered_layer in &composite_result.layers { @@ -827,16 +993,26 @@ fn composite_document_to_hdr( } RenderedLayerType::Video { instances } => { for inst in instances { - if inst.rgba_data.is_empty() { continue; } + if inst.gpu.is_none() && inst.rgba_data.is_empty() { continue; } let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { - // Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU - // decodes to linear on sample (no per-pixel CPU conversion). Blit with - // blit_straight so the shader doesn't unpremultiply. - let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex")); - let tex_view = tex.create_view(&Default::default()); let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height); - gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + if let Some(gpu) = &inst.gpu { + // Hardware-decoded NV12 plane textures → linear, no CPU upload. + let y_view = gpu.y.create_view(&Default::default()); + let uv_view = gpu.uv.create_view(&Default::default()); + gpu_resources.nv12_blit.blit( + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, + gpu.full_range, gpu.coeffs, gpu.transfer, gpu.primaries, + ); + } else { + // Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU + // decodes to linear on sample (no per-pixel CPU conversion). Blit with + // blit_straight so the shader doesn't unpremultiply. + let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex")); + let tex_view = tex.create_view(&Default::default()); + gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + } let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal); let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") }); gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None); @@ -906,10 +1082,33 @@ fn composite_document_to_hdr( } } + if prof { + record_composite_profile(t_bg.duration_since(t_c0), t_bg.elapsed()); + } + gpu_resources.buffer_pool.next_frame(); Ok(()) } +/// Split of `composite_document_to_hdr`: static-background re-render vs. the layer loop +/// (video upload + blits). Prints a running average every 200 frames under LB_RENDER_PROFILE. +fn record_composite_profile(background: std::time::Duration, layers: std::time::Duration) { + use std::sync::atomic::{AtomicU64, Ordering}; + static BG_US: AtomicU64 = AtomicU64::new(0); + static LAYERS_US: AtomicU64 = AtomicU64::new(0); + static N: AtomicU64 = AtomicU64::new(0); + BG_US.fetch_add(background.as_micros() as u64, Ordering::Relaxed); + LAYERS_US.fetch_add(layers.as_micros() as u64, Ordering::Relaxed); + let n = N.fetch_add(1, Ordering::Relaxed) + 1; + if n % 200 == 0 { + println!( + "📊 [COMPOSITE PROFILE] {n} frames avg: background-render {:.2}ms | layers(video upload+blit) {:.2}ms", + BG_US.load(Ordering::Relaxed) as f64 / n as f64 / 1000.0, + LAYERS_US.load(Ordering::Relaxed) as f64 / n as f64 / 1000.0, + ); + } +} + /// Upload `pixels` to a transient GPU texture (TEXTURE_BINDING | COPY_DST) in the /// given format. Use `Rgba8UnormSrgb` to upload raw sRGB bytes and let the GPU /// decode to linear on sample (no CPU conversion). @@ -940,221 +1139,9 @@ fn upload_transient_texture( tex } -/// Render a document frame using the HDR compositing pipeline with effects -/// -/// This function uses the same rendering pipeline as the stage preview, -/// ensuring effects are applied correctly during export. -/// -/// # Arguments -/// * `document` - Document to render (current_time will be modified) -/// * `timestamp` - Time in seconds to render at -/// * `width` - Frame width in pixels -/// * `height` - Frame height in pixels -/// * `device` - wgpu device -/// * `queue` - wgpu queue -/// * `renderer` - Vello renderer -/// * `image_cache` - Image cache for rendering -/// * `video_manager` - Video manager for video clips -/// * `gpu_resources` - HDR GPU resources for compositing -/// -/// # Returns -/// Ok((y_plane, u_plane, v_plane)) with YUV420p planes on success, Err with message on failure -pub fn render_frame_to_rgba_hdr( - document: &mut Document, - timestamp: f64, - width: u32, - height: u32, - device: &wgpu::Device, - queue: &wgpu::Queue, - renderer: &mut vello::Renderer, - image_cache: &mut ImageCache, - video_manager: &Arc>, - gpu_resources: &mut ExportGpuResources, -) -> Result<(Vec, Vec, Vec), String> { - use vello::kurbo::Affine; - - // Set document time to the frame timestamp - document.current_time = timestamp; - - // Scale the document to the export resolution. The core renderer bakes this - // base transform into every layer (vector scenes, raster and video layer - // transforms), so the whole stage scales up/down to fill the output. When the - // export size matches the document this is the identity. - let base_transform = if document.width > 0.0 && document.height > 0.0 { - Affine::scale_non_uniform( - width as f64 / document.width, - height as f64 / document.height, - ) - } else { - Affine::IDENTITY - }; - - // Render document for compositing (returns per-layer scenes) - let composite_result = render_document_for_compositing( - document, - base_transform, - image_cache, - video_manager, - None, // No webcam during export - None, // No floating selection during export - false, // No checkerboard in export - ); - - // Video export is never transparent. - composite_document_to_hdr(&composite_result, document, device, queue, renderer, gpu_resources, width, height, false)?; - - // Use persistent output texture (already created in ExportGpuResources) - let output_view = &gpu_resources.output_texture_view; - - // Convert HDR to sRGB for output - let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("export_linear_to_srgb_bind_group"), - layout: &gpu_resources.linear_to_srgb_bind_group_layout, - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&gpu_resources.hdr_texture_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler(&gpu_resources.linear_to_srgb_sampler), - }, - ], - }); - - let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("export_linear_to_srgb_encoder"), - }); - - { - let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("export_linear_to_srgb_pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &output_view, - resolve_target: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), - store: wgpu::StoreOp::Store, - }, - depth_slice: None, - })], - depth_stencil_attachment: None, - occlusion_query_set: None, - timestamp_writes: None, - }); - - render_pass.set_pipeline(&gpu_resources.linear_to_srgb_pipeline); - render_pass.set_bind_group(0, &bind_group, &[]); - render_pass.draw(0..4, 0..1); - } - - queue.submit(Some(encoder.finish())); - - // GPU YUV conversion: Convert RGBA output to YUV420p - let mut yuv_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("export_yuv_conversion_encoder"), - }); - - gpu_resources.yuv_converter.convert( - device, - &mut yuv_encoder, - output_view, - &gpu_resources.yuv_texture_view, - width, - height, - ); - - // Copy YUV texture to persistent staging buffer - let yuv_height = height + height / 2; // Y plane + U plane + V plane - yuv_encoder.copy_texture_to_buffer( - wgpu::TexelCopyTextureInfo { - texture: &gpu_resources.yuv_texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::TexelCopyBufferInfo { - buffer: &gpu_resources.staging_buffer, - layout: wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(width * 4), // Rgba8Unorm = 4 bytes per pixel - rows_per_image: Some(yuv_height), - }, - }, - wgpu::Extent3d { - width, - height: yuv_height, - depth_or_array_layers: 1, - }, - ); - - queue.submit(Some(yuv_encoder.finish())); - - // Map buffer and read YUV pixels (synchronous) - let buffer_slice = gpu_resources.staging_buffer.slice(..); - let (sender, receiver) = std::sync::mpsc::channel(); - buffer_slice.map_async(wgpu::MapMode::Read, move |result| { - sender.send(result).ok(); - }); - - let _ = device.poll(wgpu::PollType::wait_indefinitely()); - - receiver - .recv() - .map_err(|_| "Failed to receive buffer mapping result")? - .map_err(|e| format!("Failed to map buffer: {:?}", e))?; - - // Extract Y, U, V planes from packed YUV buffer - let data = buffer_slice.get_mapped_range(); - let width_usize = width as usize; - let height_usize = height as usize; - - // Y plane: rows 0 to height-1 (extract R channel from Rgba8Unorm) - let y_plane_size = width_usize * height_usize; - let mut y_plane = vec![0u8; y_plane_size]; - for y in 0..height_usize { - let src_row_offset = y * width_usize * 4; // 4 bytes per pixel (Rgba8Unorm) - let dst_row_offset = y * width_usize; - for x in 0..width_usize { - y_plane[dst_row_offset + x] = data[src_row_offset + x * 4]; // Extract R channel - } - } - - // U and V planes: rows height to height + height/2 - 1 (half resolution, side-by-side layout) - // U plane is in left half (columns 0 to width/2-1), V plane is in right half (columns width/2 to width-1) - let chroma_width = width_usize / 2; - let chroma_height = height_usize / 2; - let chroma_row_start = height_usize * width_usize * 4; // Start of chroma rows in bytes - - let mut u_plane = vec![0u8; chroma_width * chroma_height]; - let mut v_plane = vec![0u8; chroma_width * chroma_height]; - - for y in 0..chroma_height { - let row_offset = chroma_row_start + y * width_usize * 4; // Full width rows in chroma region - - // Extract U plane (left half: columns 0 to chroma_width-1) - let u_start = row_offset; - let dst_offset = y * chroma_width; - for x in 0..chroma_width { - u_plane[dst_offset + x] = data[u_start + x * 4]; // Extract R channel - } - - // Extract V plane (right half: columns width/2 to width/2+chroma_width-1) - let v_start = row_offset + chroma_width * 4; - for x in 0..chroma_width { - v_plane[dst_offset + x] = data[v_start + x * 4]; // Extract R channel - } - } - - drop(data); - gpu_resources.staging_buffer.unmap(); - - Ok((y_plane, u_plane, v_plane)) -} - /// Render frame to GPU RGBA texture (non-blocking, for async pipeline) /// -/// Similar to render_frame_to_rgba_hdr but renders to an external RGBA texture view +/// Renders to an external RGBA texture view /// (provided by ReadbackPipeline) and returns the command encoder WITHOUT blocking on readback. /// The caller (ReadbackPipeline) will submit the encoder and handle async readback. /// @@ -1206,6 +1193,51 @@ fn fault_in_raster_for_frame( } } +/// Render one frame as 10-bit HDR YUV420P10LE planes (BT.2020 + PQ/HLG). Synchronous: composites, +/// runs the linear→PQ/HLG GPU pass, reads it back, and CPU-converts to 10-bit YUV. Used by the +/// HDR export path instead of the async readback pipeline. +#[allow(clippy::too_many_arguments)] +pub fn render_frame_to_yuv10_hdr( + document: &mut Document, + timestamp: f64, + width: u32, + height: u32, + device: &wgpu::Device, + queue: &wgpu::Queue, + renderer: &mut vello::Renderer, + image_cache: &mut ImageCache, + video_manager: &Arc>, + gpu_resources: &mut ExportGpuResources, + hdr_mode: lightningbeam_core::export::HdrExportMode, + fit: lightningbeam_core::export::ExportFitMode, + raster_store: Option<&lightningbeam_core::raster_store::RasterStore>, +) -> Result<(Vec, Vec, Vec), String> { + document.current_time = timestamp; + fault_in_raster_for_frame(document, raster_store); + + let base_transform = export_base_transform(document.width, document.height, width as f64, height as f64, fit); + + // HDR export composites on the shared device, so it can consume hardware-decoded GPU frames. + if let Ok(mut vm) = video_manager.lock() { + vm.set_render_hardware_ok(true); + } + + let composite_result = render_document_for_compositing( + document, base_transform, image_cache, video_manager, None, None, false, + ); + composite_document_to_hdr(&composite_result, document, device, queue, renderer, gpu_resources, width, height, false)?; + + if gpu_resources.hdr_pipeline.is_none() { + gpu_resources.hdr_pipeline = Some(super::hdr_frame::HdrFramePipeline::new(device, width, height)); + } + let planes = gpu_resources + .hdr_pipeline + .as_ref() + .unwrap() + .render_to_yuv10(device, queue, &gpu_resources.hdr_texture_view, hdr_mode); + Ok(planes) +} + pub fn render_frame_to_gpu_rgba( document: &mut Document, timestamp: f64, @@ -1221,8 +1253,16 @@ pub fn render_frame_to_gpu_rgba( floating_selection: Option<&lightningbeam_core::selection::RasterFloatingSelection>, allow_transparency: bool, raster_store: Option<&lightningbeam_core::raster_store::RasterStore>, + // True when compositing on the shared device (software/image export) → may consume + // hardware-decoded GPU frames; false for the zero-copy path on its own device. + hardware_ok: bool, + fit: lightningbeam_core::export::ExportFitMode, ) -> Result { - use vello::kurbo::Affine; + // One-shot profiling of the render-bucket split (LB_RENDER_PROFILE=1): how much of the + // per-frame CPU "render" is document build (incl. video decode) vs. composite-command + // recording (incl. the frame texture upload) vs. the sRGB pass. Prints a running average. + let prof = render_profile_enabled(); + let t0 = std::time::Instant::now(); // Set document time to the frame timestamp document.current_time = timestamp; @@ -1237,14 +1277,13 @@ pub fn render_frame_to_gpu_rgba( // base transform into every layer (vector scenes, raster and video layer // transforms), so the whole stage scales up/down to fill the output. When the // export size matches the document this is the identity. - let base_transform = if document.width > 0.0 && document.height > 0.0 { - Affine::scale_non_uniform( - width as f64 / document.width, - height as f64 / document.height, - ) - } else { - Affine::IDENTITY - }; + let base_transform = export_base_transform(document.width, document.height, width as f64, height as f64, fit); + + // GPU frames are usable only on the shared device (software/image export); the zero-copy path + // runs on its own device and must download to CPU. + if let Ok(mut vm) = video_manager.lock() { + vm.set_render_hardware_ok(hardware_ok); + } // Render document for compositing (returns per-layer scenes) let composite_result = render_document_for_compositing( @@ -1256,8 +1295,10 @@ pub fn render_frame_to_gpu_rgba( floating_selection, false, // No checkerboard in export ); + let t_build = std::time::Instant::now(); composite_document_to_hdr(&composite_result, document, device, queue, renderer, gpu_resources, width, height, allow_transparency)?; + let t_composite = std::time::Instant::now(); // Convert HDR to sRGB (linear → sRGB), render directly to external RGBA texture let output_view = rgba_texture_view; @@ -1297,16 +1338,58 @@ pub fn render_frame_to_gpu_rgba( timestamp_writes: None, }); - render_pass.set_pipeline(&gpu_resources.linear_to_srgb_pipeline); + let final_pipeline = match document.hdr_output_mode { + lightningbeam_core::document::HdrOutputMode::HighlightRolloff => &gpu_resources.linear_to_srgb_pipeline_rolloff, + lightningbeam_core::document::HdrOutputMode::Clip => &gpu_resources.linear_to_srgb_pipeline, + }; + render_pass.set_pipeline(final_pipeline); render_pass.set_bind_group(0, &bind_group, &[]); render_pass.draw(0..4, 0..1); } + if prof { + record_render_profile( + t_build.duration_since(t0), + t_composite.duration_since(t_build), + t_composite.elapsed(), + ); + } + // Return encoder for caller to submit (ReadbackPipeline will handle submission and async readback) // Frame is already rendered to external RGBA texture, no GPU YUV conversion needed Ok(encoder) } +/// `LB_RENDER_PROFILE` gate, checked once. +fn render_profile_enabled() -> bool { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| std::env::var("LB_RENDER_PROFILE").is_ok()) +} + +/// Accumulate the per-frame render split and print a running average every 200 frames. +/// `build` = document build incl. video decode; `composite` = composite-command recording +/// incl. the frame texture upload; `srgb` = the linear→sRGB pass. +fn record_render_profile(build: std::time::Duration, composite: std::time::Duration, srgb: std::time::Duration) { + use std::sync::atomic::{AtomicU64, Ordering}; + static BUILD_US: AtomicU64 = AtomicU64::new(0); + static COMPOSITE_US: AtomicU64 = AtomicU64::new(0); + static SRGB_US: AtomicU64 = AtomicU64::new(0); + static N: AtomicU64 = AtomicU64::new(0); + BUILD_US.fetch_add(build.as_micros() as u64, Ordering::Relaxed); + COMPOSITE_US.fetch_add(composite.as_micros() as u64, Ordering::Relaxed); + SRGB_US.fetch_add(srgb.as_micros() as u64, Ordering::Relaxed); + let n = N.fetch_add(1, Ordering::Relaxed) + 1; + if n % 200 == 0 { + let (b, c, s) = (BUILD_US.load(Ordering::Relaxed), COMPOSITE_US.load(Ordering::Relaxed), SRGB_US.load(Ordering::Relaxed)); + println!( + "📊 [RENDER PROFILE] {n} frames avg: build(+decode) {:.2}ms | composite(+upload) {:.2}ms | srgb {:.2}ms", + b as f64 / n as f64 / 1000.0, + c as f64 / n as f64 / 1000.0, + s as f64 / n as f64 / 1000.0, + ); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs index 5cf8421..07ba0b1 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs @@ -1627,12 +1627,24 @@ impl GpuBrushEngine { self.proxy_layer_cache.get(kf_id) } - /// Remove the cached texture for a raster layer keyframe (e.g. when deleted). + /// Remove the cached texture for a raster layer keyframe (e.g. when deleted or edited). pub fn remove_layer_texture(&mut self, kf_id: &Uuid) { - if self.raster_layer_cache.remove(kf_id).is_some() { + let mut changed = self.raster_layer_cache.remove(kf_id).is_some(); + if changed { if let Some(pos) = self.raster_layer_lru.iter().position(|id| id == kf_id) { self.raster_layer_lru.remove(pos); } + } + // Also drop the low-res proxy: proxies are uploaded once and never refreshed, so a + // stale pre-edit proxy left here would be blitted (flashing old content) if the full-res + // texture is later evicted before the edited pixels page back in. + if self.proxy_layer_cache.remove(kf_id).is_some() { + if let Some(pos) = self.proxy_layer_lru.iter().position(|id| id == kf_id) { + self.proxy_layer_lru.remove(pos); + } + changed = true; + } + if changed { self.report_raster_cache_vram(); } } @@ -2238,6 +2250,8 @@ impl CanvasBlitPipeline { /// Blit a **straight-alpha** source (e.g. a video frame uploaded to an /// `Rgba8UnormSrgb` texture, hardware-decoded to linear on sample). Uses the /// `fs_main_straight` pipeline, which skips the unpremultiply that `blit` does. + /// Bilinear-sampled: video frames are scaled to the output size (document→export, or any + /// non-1:1 transform), and nearest sampling makes that look blocky. pub fn blit_straight( &self, device: &wgpu::Device, @@ -2247,7 +2261,7 @@ impl CanvasBlitPipeline { transform: &BlitTransform, mask_view: Option<&wgpu::TextureView>, ) { - self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline_straight); + self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler, &self.pipeline_straight); } #[allow(clippy::too_many_arguments)] diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs new file mode 100644 index 0000000..906599e --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -0,0 +1,180 @@ +//! Hardware video decode glue (Linux/VAAPI). The editor implements core's [`HwVideoImporter`]: +//! it maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane +//! textures on the **shared** device (the one eframe + the compositor run on, which has the +//! DMA-BUF-import extensions). [`install`] creates the VAAPI device and wires it into the +//! `VideoManager`. + +use ffmpeg_next::ffi as ff; +use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf}; +use lightningbeam_core::video::{ + ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, VideoPrimaries, + VideoTransfer, +}; +use std::sync::{Arc, Mutex}; + +/// Imports decoded VAAPI surfaces onto the shared wgpu device. Holds clones of the shared +/// device + adapter (Arc-backed, cheap). +struct SharedHwImporter { + device: wgpu::Device, + adapter: wgpu::Adapter, + /// Log the detected colour info once (under LB_VIDEO_DEBUG) rather than per frame. + logged: std::sync::atomic::AtomicBool, +} + +impl HwVideoImporter for SharedHwImporter { + unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option { + let frame = av_frame as *mut ff::AVFrame; + + // Map the VAAPI surface to a DRM-PRIME DMA-BUF (read-only). + let drm_f = ff::av_frame_alloc(); + (*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + let flags = ff::AV_HWFRAME_MAP_DIRECT as i32 | ff::AV_HWFRAME_MAP_READ as i32; + if ff::av_hwframe_map(drm_f, frame, flags) < 0 { + ff::av_frame_free(&mut (drm_f as *mut _)); + return None; + } + + let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor; + let obj = &(*desc).objects[0]; + let width = (*frame).width as u32; + let height = (*frame).height as u32; + + // 10/12/16-bit content decodes to P010-style surfaces (16-bit planes). Detect via the hw + // frames context's software format so the import builds R16/Rg16 textures. + let ten_bit = { + let hwfc = (*frame).hw_frames_ctx; + if hwfc.is_null() { + false + } else { + let ctx = (*hwfc).data as *const ff::AVHWFramesContext; + matches!( + (*ctx).sw_format, + ff::AVPixelFormat::AV_PIX_FMT_P010LE + | ff::AVPixelFormat::AV_PIX_FMT_P010BE + | ff::AVPixelFormat::AV_PIX_FMT_P012LE + | ff::AVPixelFormat::AV_PIX_FMT_P012BE + | ff::AVPixelFormat::AV_PIX_FMT_P016LE + | ff::AVPixelFormat::AV_PIX_FMT_P016BE + ) + } + }; + // NV12: Y then UV — two layers (one plane each) or one layer with two planes. + let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 { + (&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0]) + } else { + (&(*desc).layers[0].planes[0], &(*desc).layers[0].planes[1]) + }; + let buf = Nv12DmaBuf { + fd: obj.fd, + size: obj.size as u64, + modifier: obj.format_modifier, + width, + height, + y_offset: y_pl.offset as u64, + y_pitch: y_pl.pitch as u64, + uv_offset: uv_pl.offset as u64, + uv_pitch: uv_pl.pitch as u64, + ten_bit, + }; + let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG; + + // Luma weights (kr, kb) from the frame's matrix coefficients, so SD (BT.601) and HD/UHD + // (BT.709) clips each convert with the right matrix. Unspecified → guess by height, as + // players/swscale do. SMPTE240M and BT.2020 are handled too (the latter's transfer is still + // approximated as sRGB — fine for SDR; true HDR is out of scope). + let (kr, kb) = match (*frame).colorspace { + ff::AVColorSpace::AVCOL_SPC_BT709 => (0.2126, 0.0722), + ff::AVColorSpace::AVCOL_SPC_BT470BG | ff::AVColorSpace::AVCOL_SPC_SMPTE170M => { + (0.299, 0.114) + } + ff::AVColorSpace::AVCOL_SPC_SMPTE240M => (0.212, 0.087), + ff::AVColorSpace::AVCOL_SPC_BT2020_NCL | ff::AVColorSpace::AVCOL_SPC_BT2020_CL => { + (0.2627, 0.0593) + } + _ => { + if height <= 576 { + (0.299, 0.114) // SD → BT.601 + } else { + (0.2126, 0.0722) // HD/UHD → BT.709 + } + } + }; + let coeffs = ycbcr_coeffs(kr, kb); + + if std::env::var("LB_VIDEO_DEBUG").is_ok() + && !self.logged.swap(true, std::sync::atomic::Ordering::Relaxed) + { + eprintln!( + "[hw_video] {}x{} ten_bit={} full_range={} colorspace={:?} primaries={:?} trc={:?}", + width, height, ten_bit, full_range, + (*frame).colorspace, (*frame).color_primaries, (*frame).color_trc, + ); + } + + // Transfer characteristic → which EOTF the compositor applies to reach scene-linear. + let transfer = match (*frame).color_trc { + ff::AVColorTransferCharacteristic::AVCOL_TRC_SMPTE2084 => VideoTransfer::Pq, + ff::AVColorTransferCharacteristic::AVCOL_TRC_ARIB_STD_B67 => VideoTransfer::Hlg, + _ => VideoTransfer::Gamma, + }; + // Primaries → BT.2020 is gamut-mapped to BT.709; unspecified follows the matrix guess above. + let primaries = match (*frame).color_primaries { + ff::AVColorPrimaries::AVCOL_PRI_BT2020 => VideoPrimaries::Bt2020, + ff::AVColorPrimaries::AVCOL_PRI_UNSPECIFIED + if matches!( + (*frame).colorspace, + ff::AVColorSpace::AVCOL_SPC_BT2020_NCL | ff::AVColorSpace::AVCOL_SPC_BT2020_CL + ) => + { + VideoPrimaries::Bt2020 + } + _ => VideoPrimaries::Bt709, + }; + + let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); + ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan + let (y, uv) = match imported { + Ok(t) => t.into_planes(), + Err(e) => { + // Surface the failure: a silent None here makes core fall back to software (no gamut + // conversion → BT.2020 looks washed out). 10-bit P010 import is the likely culprit. + eprintln!("[hw_video] import_raw failed (ten_bit={ten_bit}): {e}"); + return None; + } + }; + Some(GpuVideoFrame { + y: Arc::new(y), + uv: Arc::new(uv), + width, + height, + full_range, + coeffs, + transfer, + primaries, + }) + } +} + +/// Create the VAAPI hardware device and install hardware decode into `vm`, importing onto the +/// shared `device`/`adapter`. Logs and no-ops if VAAPI is unavailable (→ software decode). +pub fn install(vm: &Arc>, device: &wgpu::Device, adapter: &wgpu::Adapter) { + match gpu_video_encoder::vaapi::create_device() { + Ok(hw_device) => { + let importer = Arc::new(SharedHwImporter { + device: device.clone(), + adapter: adapter.clone(), + logged: std::sync::atomic::AtomicBool::new(false), + }); + if let Ok(mut vm) = vm.lock() { + vm.set_hardware_decode( + HwDeviceHandle(hw_device as *mut std::ffi::c_void), + importer, + ); + } + println!("🎞 Hardware video decode enabled (VAAPI → shared device)"); + } + Err(e) => { + println!("🎞 Hardware video decode unavailable ({e}); using software decode"); + } + } +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 70e7789..435f7cf 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -51,12 +51,16 @@ mod custom_cursor; mod tablet; mod debug_overlay; mod gpu_timer; +#[cfg(target_os = "linux")] +mod hw_video; +mod nv12_blit; #[cfg(debug_assertions)] mod test_mode; mod sample_import; mod sample_import_dialog; +mod svg_import; mod curve_editor; @@ -79,6 +83,40 @@ struct Args { cpu_renderer: bool, } +/// The default eframe wgpu device setup (wgpu picks the adapter/device). Used on non-Linux, when +/// the shared VAAPI device is unavailable, or when disabled via `LB_NO_SHARED_DEVICE`. +fn lb_default_wgpu_setup() -> egui_wgpu::WgpuSetup { + egui_wgpu::WgpuSetup::CreateNew(egui_wgpu::WgpuSetupCreateNew { + device_descriptor: std::sync::Arc::new(|adapter| { + let features = adapter.features(); + // Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's + // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability). + // TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer + // (gpu_timer.rs); both are optional and no-op when unsupported. + let optional_features = wgpu::Features::SHADER_F16 + | wgpu::Features::TIMESTAMP_QUERY + | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS; + + let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { + wgpu::Limits::downlevel_webgl2_defaults() + } else { + wgpu::Limits::default() + }; + + wgpu::DeviceDescriptor { + label: Some("lightningbeam wgpu device"), + required_features: features & optional_features, + required_limits: wgpu::Limits { + max_texture_dimension_2d: 8192, + ..base_limits + }, + ..Default::default() + } + }), + ..Default::default() + }) +} + fn main() -> eframe::Result { println!("🚀 Starting Lightningbeam Editor..."); @@ -168,38 +206,43 @@ fn main() -> eframe::Result { viewport_builder = viewport_builder.with_icon(icon); } + // Prefer the shared VAAPI-capable wgpu device on Linux: eframe + the compositor + hardware + // video decode all run on one device, so decoded DMA-BUF frames are usable by the preview + // compositor (decode/encode can't share textures across devices). Falls back to wgpu's normal + // device (software video decode) when unavailable, on other platforms, or via LB_NO_SHARED_DEVICE. + // The DrmDevice's wgpu handles are cloned into eframe (Arc-backed, keep the device alive); the + // raw VkDevice persists with them. + #[allow(unused_mut)] + let mut wgpu_setup = lb_default_wgpu_setup(); + // Whether the shared VAAPI-capable device is in use — only then can decoded DMA-BUF frames be + // imported + composited, so hardware decode is injected into the VideoManager only if true. + #[allow(unused_assignments, unused_mut)] + let mut shared_device_active = false; + #[cfg(target_os = "linux")] + if std::env::var("LB_NO_SHARED_DEVICE").is_ok() { + println!("🖥 Shared device disabled via LB_NO_SHARED_DEVICE; default wgpu device (software video decode)"); + } else { + match gpu_video_encoder::vk_device::create_windowed() { + Ok(drm) => { + println!("🖥 Using shared VAAPI-capable wgpu device (hardware video decode enabled)"); + wgpu_setup = egui_wgpu::WgpuSetup::Existing(egui_wgpu::WgpuSetupExisting { + instance: drm.instance.clone(), + adapter: drm.adapter.clone(), + device: drm.device.clone(), + queue: drm.queue.clone(), + }); + shared_device_active = true; + } + Err(e) => { + println!("🖥 Shared device unavailable ({e}); default wgpu device (software video decode)"); + } + } + } + let options = eframe::NativeOptions { viewport: viewport_builder, wgpu_options: egui_wgpu::WgpuConfiguration { - wgpu_setup: egui_wgpu::WgpuSetup::CreateNew(egui_wgpu::WgpuSetupCreateNew { - device_descriptor: std::sync::Arc::new(|adapter| { - let features = adapter.features(); - // Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's - // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability). - // TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer - // (gpu_timer.rs); both are optional and no-op when unsupported. - let optional_features = wgpu::Features::SHADER_F16 - | wgpu::Features::TIMESTAMP_QUERY - | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS; - - let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { - wgpu::Limits::downlevel_webgl2_defaults() - } else { - wgpu::Limits::default() - }; - - wgpu::DeviceDescriptor { - label: Some("lightningbeam wgpu device"), - required_features: features & optional_features, - required_limits: wgpu::Limits { - max_texture_dimension_2d: 8192, - ..base_limits - }, - ..Default::default() - } - }), - ..Default::default() - }), + wgpu_setup, ..Default::default() }, ..Default::default() @@ -257,9 +300,20 @@ fn main() -> eframe::Result { options, Box::new(move |cc| { #[cfg(debug_assertions)] - let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); + #[allow(unused_mut)] + let mut app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); #[cfg(not(debug_assertions))] - let app = EditorApp::new(cc, layouts, theme); + #[allow(unused_mut)] + let mut app = EditorApp::new(cc, layouts, theme); + // Wire hardware video decode into the VideoManager now that the shared device exists, and + // stash the shared device handles so the zero-copy export encoder can run on it too. + #[cfg(target_os = "linux")] + if shared_device_active { + if let Some(rs) = cc.wgpu_render_state.as_ref() { + hw_video::install(&app.video_manager, &rs.device, &rs.adapter); + app.shared_device = Some((rs.device.clone(), rs.queue.clone(), rs.adapter.clone())); + } + } Ok(Box::new(app)) }), ) @@ -941,6 +995,9 @@ struct EditorApp { audio_channels: u32, // Video decoding and management video_manager: std::sync::Arc>, // Shared video manager + /// The shared VAAPI-capable wgpu device (device, queue, adapter), `Some` only when active. Lets + /// the zero-copy export encoder run on it (GPU-resident decode→composite→encode). + shared_device: Option<(wgpu::Device, wgpu::Queue, wgpu::Adapter)>, // Webcam capture state webcam: Option, /// Latest polled webcam frame (updated each frame for preview) @@ -1278,6 +1335,7 @@ impl EditorApp { video_manager: std::sync::Arc::new(std::sync::Mutex::new( lightningbeam_core::video::VideoManager::new() )), + shared_device: None, webcam: None, webcam_frame: None, webcam_record_command: None, @@ -2334,6 +2392,12 @@ impl EditorApp { kf.needs_fault_in = false; } } + // Track these resident pixels in the LRU so they count toward + // RASTER_RESIDENT_MAX and can be evicted later; without this, a frame + // faulted in for undo/redo that ends up clean would stay resident forever, + // letting resident RAM grow past the cap. + self.raster_resident_lru.retain(|id| *id != kf_id); + self.raster_resident_lru.push_back(kf_id); } } } @@ -3181,7 +3245,11 @@ impl EditorApp { .and_then(|e| e.to_str()) .unwrap_or(""); - let imported_asset = match get_file_type(extension) { + // SVG imports as a new vector layer (not a placeable asset). + let imported_asset = if extension.eq_ignore_ascii_case("svg") { + self.import_svg_file(&path); + None + } else { match get_file_type(extension) { Some(FileType::Image) => { self.last_import_filter = ImportFilter::Images; self.import_image(&path) @@ -3198,11 +3266,12 @@ impl EditorApp { self.last_import_filter = ImportFilter::Midi; self.import_midi(&path) } + Some(FileType::Vector) => None, // handled by the svg intercept above None => { println!("Unsupported file type: {}", extension); None } - }; + } }; eprintln!("[TIMING] import took {:.1}ms", _import_timer.elapsed().as_secs_f64() * 1000.0); // Auto-place if this is "Import" (not "Import to Library") @@ -3658,8 +3727,21 @@ impl EditorApp { // TODO: Implement delete layer } MenuAction::ToggleLayerVisibility => { - println!("Menu: Toggle Layer Visibility"); - // TODO: Implement toggle layer visibility + use lightningbeam_core::actions::{SetLayerPropertiesAction, set_layer_properties::LayerProperty}; + use lightningbeam_core::layer::LayerTrait; + if let Some(layer_id) = self.active_layer_id { + let cur = self.action_executor.document() + .get_layer(&layer_id) + .map(|l| l.visible()); + if let Some(cur) = cur { + let action = SetLayerPropertiesAction::new(layer_id, LayerProperty::Visible(!cur)); + if let Err(e) = self.action_executor.execute(Box::new(action)) { + eprintln!("Toggle layer visibility: {}", e); + } + } + } else { + println!("Toggle Layer Visibility: no active layer"); + } } MenuAction::ShowMasterTrack => { // Toggle show_master_track on all Timeline pane instances @@ -4455,6 +4537,56 @@ impl EditorApp { } /// Import an image file as an ImageAsset + /// Import an `.svg` file as a new vector layer (one static keyframe at the playhead). + fn import_svg_file(&mut self, path: &std::path::Path) { + let bytes = match std::fs::read(path) { + Ok(b) => b, + Err(e) => { + let msg = format!("Failed to read SVG: {}", e); + eprintln!("❌ {} ({})", msg, path.display()); + notifications::notify_error("SVG Import Failed", &msg); + return; + } + }; + + let graph = match svg_import::import_svg(&bytes) { + Ok(g) => g, + Err(e) => { + eprintln!("❌ {}", e); + notifications::notify_error("SVG Import Failed", &e); + return; + } + }; + + let name = path.file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("SVG") + .to_string(); + + // Build a vector layer holding the imported graph as a keyframe at the current time. + let mut layer = lightningbeam_core::layer::VectorLayer::new(name); + let mut keyframe = lightningbeam_core::layer::ShapeKeyframe::new(self.playback_time); + keyframe.graph = graph; + layer.keyframes.push(keyframe); + + let editing_clip_id = self.editing_context.current_clip_id(); + let action = lightningbeam_core::actions::AddLayerAction::new( + lightningbeam_core::layer::AnyLayer::Vector(layer), + ) + .with_target_clip(editing_clip_id); + if let Err(e) = self.action_executor.execute(Box::new(action)) { + eprintln!("❌ Failed to add imported SVG layer: {}", e); + return; + } + + // Select the newly created layer. + let context_layers = self.action_executor.document().context_layers(editing_clip_id.as_ref()); + if let Some(last_layer) = context_layers.last() { + self.active_layer_id = Some(last_layer.id()); + } + self.last_import_filter = ImportFilter::Images; + } + fn import_image(&mut self, path: &std::path::Path) -> Option { use lightningbeam_core::clip::ImageAsset; self.note_possible_large_media(path); @@ -4917,25 +5049,8 @@ impl EditorApp { if asset_info.clip_type == panes::DragClipType::Video { if let Some((video_width, video_height)) = asset_info.dimensions { let doc = self.action_executor.document(); - let doc_width = doc.width; - let doc_height = doc.height; - - // Calculate scale to fit (use minimum to preserve aspect ratio) - let scale_x = doc_width / video_width; - let scale_y = doc_height / video_height; - let uniform_scale = scale_x.min(scale_y); - - clip_instance.transform.scale_x = uniform_scale; - clip_instance.transform.scale_y = uniform_scale; - - // Center the video in the document - let scaled_width = video_width * uniform_scale; - let scaled_height = video_height * uniform_scale; - let center_x = (doc_width - scaled_width) / 2.0; - let center_y = (doc_height - scaled_height) / 2.0; - - clip_instance.transform.x = center_x; - clip_instance.transform.y = center_y; + // Fit uniformly + centered (preserve aspect). Shared with the timeline drag path. + clip_instance.transform.fit_centered(video_width, video_height, doc.width, doc.height); } } else { // Audio clips are centered in document @@ -6076,6 +6191,9 @@ impl eframe::App for EditorApp { self.export_orchestrator = Some(export::ExportOrchestrator::new()); } + // Clone before the &mut self.export_orchestrator borrow below. + let shared_device = self.shared_device.clone(); + let export_started = if let Some(orchestrator) = &mut self.export_orchestrator { match export_result { ExportResult::Image(settings, output_path) => { @@ -6089,6 +6207,19 @@ impl eframe::App for EditorApp { ); false // image export is silent (no progress dialog) } + ExportResult::Svg(time, output_path) => { + println!("🖋 [MAIN] Exporting SVG: {}", output_path.display()); + let svg = lightningbeam_core::svg_export::document_to_svg( + self.action_executor.document(), + time, + ); + if let Err(err) = std::fs::write(&output_path, svg) { + eprintln!("❌ Failed to write SVG: {}", err); + } else { + println!("✅ SVG written: {}", output_path.display()); + } + false // synchronous; no progress dialog + } ExportResult::AudioOnly(settings, output_path) => { println!("🎵 [MAIN] Starting audio-only export: {}", output_path.display()); @@ -6114,6 +6245,7 @@ impl eframe::App for EditorApp { Arc::clone(&self.video_manager), self.raster_store.clone(), self.current_file_path.clone(), + shared_device.clone(), ) { Ok(()) => true, Err(err) => { @@ -6135,6 +6267,7 @@ impl eframe::App for EditorApp { Arc::clone(&self.video_manager), self.raster_store.clone(), self.current_file_path.clone(), + shared_device.clone(), ) { Ok(()) => true, Err(err) => { diff --git a/lightningbeam-ui/lightningbeam-editor/src/notifications.rs b/lightningbeam-ui/lightningbeam-editor/src/notifications.rs index 2222d58..0489df2 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/notifications.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/notifications.rs @@ -32,6 +32,11 @@ pub fn notify_export_complete(output_path: &Path) { /// Show a desktop notification for an export error (fire-and-forget). pub fn notify_export_error(error_message: &str) { + notify_error("Export Failed", error_message); +} + +/// Show a desktop error notification with a custom title (fire-and-forget). +pub fn notify_error(title: &'static str, error_message: &str) { // Truncate very long error messages (on a char boundary). let truncated = if error_message.chars().count() > 100 { let prefix: String = error_message.chars().take(97).collect(); @@ -42,7 +47,7 @@ pub fn notify_export_error(error_message: &str) { std::thread::spawn(move || { if let Err(e) = Notification::new() - .summary("Export Failed") + .summary(title) .body(&truncated) .icon("dialog-error") // Standard error icon .timeout(10000) // 10 seconds for errors (longer to read) diff --git a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs new file mode 100644 index 0000000..c7949f7 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs @@ -0,0 +1,194 @@ +//! NV12 → linear-RGB blit: composites a hardware-decoded video frame (two wgpu plane textures, +//! Y = R8Unorm + CbCr = Rg8Unorm) directly into the Rgba16Float HDR layer, with no CPU upload. +//! The colour math mirrors the software path (BT.709 → sRGB-encoded → linear) so hardware- and +//! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`. + +use crate::gpu_brush::BlitTransform; +use lightningbeam_core::video::{VideoPrimaries, VideoTransfer}; + +/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]), the Y'CbCr→RGB +/// matrix coefficients, and a flags vec4. 80 bytes (48 matrix + 16 coeffs + 16 flags). +/// `flags`: `[full_range, transfer (0 gamma / 1 PQ / 2 HLG), primaries (0 BT.709 / 1 BT.2020), pad]`. +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Nv12Params { + transform: BlitTransform, + coeffs: [f32; 4], + flags: [u32; 4], +} + +pub struct Nv12BlitPipeline { + pipeline: wgpu::RenderPipeline, + bg_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, +} + +impl Nv12BlitPipeline { + pub fn new(device: &wgpu::Device) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("nv12_blit_shader"), + source: wgpu::ShaderSource::Wgsl( + include_str!("panes/shaders/nv12_blit.wgsl").into(), + ), + }); + + let tex_entry = |binding: u32| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + + let bg_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("nv12_blit_bgl"), + entries: &[ + tex_entry(0), // Y plane (R8Unorm) + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + tex_entry(3), // CbCr plane (Rg8Unorm) + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("nv12_blit_pl"), + bind_group_layouts: &[&bg_layout], + push_constant_ranges: &[], + }); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("nv12_blit_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba16Float, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + // Bilinear: the frame is scaled to the output size; nearest would look blocky. + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("nv12_blit_sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + Self { pipeline, bg_layout, sampler } + } + + /// Convert + blit the NV12 frame into `target_view` (Rgba16Float, cleared to transparent), + /// positioned by `transform` (built like the RGBA video path's `BlitTransform`). + #[allow(clippy::too_many_arguments)] + pub fn blit( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + y_view: &wgpu::TextureView, + uv_view: &wgpu::TextureView, + target_view: &wgpu::TextureView, + transform: &BlitTransform, + full_range: bool, + coeffs: [f32; 4], + transfer: VideoTransfer, + primaries: VideoPrimaries, + ) { + let transfer_code = match transfer { + VideoTransfer::Gamma => 0, + VideoTransfer::Pq => 1, + VideoTransfer::Hlg => 2, + }; + let primaries_code = match primaries { + VideoPrimaries::Bt709 => 0, + VideoPrimaries::Bt2020 => 1, + }; + let params = Nv12Params { + transform: *transform, + coeffs, + flags: [full_range as u32, transfer_code, primaries_code, 0], + }; + let param_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("nv12_blit_params"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + queue.write_buffer(¶m_buf, 0, bytemuck::bytes_of(¶ms)); + + let bg = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("nv12_blit_bg"), + layout: &self.bg_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(y_view) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&self.sampler) }, + wgpu::BindGroupEntry { binding: 2, resource: param_buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(uv_view) }, + ], + }); + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("nv12_blit_encoder"), + }); + { + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("nv12_blit_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: target_view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + }); + rp.set_pipeline(&self.pipeline); + rp.set_bind_group(0, &bg, &[]); + rp.draw(0..4, 0..1); + } + queue.submit(Some(encoder.finish())); + } +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs index 3e3cd41..0132c8f 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/asset_library.rs @@ -316,7 +316,10 @@ fn generate_video_thumbnail( let frame = { let mut video_mgr = video_manager.lock().ok()?; - video_mgr.get_frame(clip_id, timestamp)? + // Small CPU frame for the asset thumbnail (capped to native, aspect preserved). Force CPU: + // the render-pass hardware flag may be on, but the thumbnail needs RGBA bytes (a GPU frame + // has empty rgba_data → an all-black thumbnail). + video_mgr.get_frame_cpu(clip_id, timestamp, THUMBNAIL_SIZE, THUMBNAIL_SIZE)? }; let src_width = frame.width as usize; diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs index 5813e8e..0cb524a 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/infopanel.rs @@ -44,6 +44,8 @@ pub struct InfopanelPane { selected_tool_gradient_stop: Option, /// FPS value captured when a drag/focus-in starts (for single-undo-action on commit) fps_drag_start: Option, + /// Resize mode for the active raster layer's "to document size" action (scale vs canvas). + raster_resize_mode: lightningbeam_core::raster_layer::RasterResizeMode, } impl InfopanelPane { @@ -61,6 +63,7 @@ impl InfopanelPane { selected_shape_gradient_stop: None, selected_tool_gradient_stop: None, fps_drag_start: None, + raster_resize_mode: lightningbeam_core::raster_layer::RasterResizeMode::Scale, } } } @@ -980,10 +983,10 @@ impl InfopanelPane { // Extract all needed values up front, then drop the borrow before closures // that need mutable access to shared or self. - let (mut width, mut height, mut duration, mut framerate, layer_count, background_color) = { + let (mut width, mut height, mut duration, mut framerate, layer_count, background_color, mut hdr_mode) = { let document = shared.action_executor.document(); (document.width, document.height, document.duration, document.framerate, - document.root.children.len(), document.background_color) + document.root.children.len(), document.background_color, document.hdr_output_mode) }; // Canvas width @@ -1087,6 +1090,23 @@ impl InfopanelPane { } }); + // HDR output mode (how super-white video highlights map to SDR output) + ui.horizontal(|ui| { + use lightningbeam_core::document::HdrOutputMode; + ui.label("HDR output:"); + egui::ComboBox::from_id_salt(("hdr_output_mode", path)) + .selected_text(hdr_mode.name()) + .show_ui(ui, |ui| { + let mut changed = false; + changed |= ui.selectable_value(&mut hdr_mode, HdrOutputMode::Clip, HdrOutputMode::Clip.name()).changed(); + changed |= ui.selectable_value(&mut hdr_mode, HdrOutputMode::HighlightRolloff, HdrOutputMode::HighlightRolloff.name()).changed(); + if changed { + let action = SetDocumentPropertiesAction::set_hdr_output_mode(hdr_mode); + shared.pending_actions.push(Box::new(action)); + } + }); + }); + // Layer count (read-only) ui.horizontal(|ui| { ui.label("Layers:"); @@ -1180,6 +1200,53 @@ impl InfopanelPane { }); } + /// Render a raster-layer section: shows the active keyframe's canvas dimensions and, when they + /// differ from the document, a Scale/Expand-Crop mode toggle + a "Layer to document size" button. + /// Driven by the *active* layer (not selection focus), since painting doesn't focus the layer. + fn render_raster_layer_section(&mut self, ui: &mut Ui, path: &NodePath, shared: &mut SharedPaneState, layer_id: Uuid) { + use lightningbeam_core::raster_layer::RasterResizeMode; + + // Pull the values, then drop the document borrow before mutating `shared`. + let time = *shared.playback_time; + let dims = { + let document = shared.action_executor.document(); + match document.get_layer(&layer_id) { + Some(AnyLayer::Raster(rl)) => rl + .keyframe_at(time) + .map(|kf| (kf.width, kf.height, document.width as u32, document.height as u32)), + _ => None, + } + }; + let Some((kf_w, kf_h, doc_w, doc_h)) = dims else { return }; + + egui::CollapsingHeader::new("Raster Layer") + .id_salt(("raster_layer", path)) + .default_open(true) + .show(ui, |ui| { + ui.add_space(4.0); + ui.horizontal(|ui| { + ui.label("Size:"); + ui.label(format!("{} × {}", kf_w, kf_h)); + }); + + if kf_w != doc_w || kf_h != doc_h { + ui.horizontal(|ui| { + ui.label("Mode:"); + ui.selectable_value(&mut self.raster_resize_mode, RasterResizeMode::Scale, "Scale"); + ui.selectable_value(&mut self.raster_resize_mode, RasterResizeMode::Canvas, "Expand/Crop"); + }); + if ui.button(format!("Layer to document size ({} × {})", doc_w, doc_h)).clicked() { + let store = lightningbeam_core::raster_store::RasterStore::new(shared.container_path.clone()); + let action = lightningbeam_core::actions::ResizeRasterLayerAction::new( + layer_id, doc_w, doc_h, self.raster_resize_mode, store, + ); + shared.pending_actions.push(Box::new(action)); + } + } + ui.add_space(4.0); + }); + } + /// Render clip instance info section fn render_clip_instance_section(&self, ui: &mut Ui, path: &NodePath, shared: &SharedPaneState, clip_ids: &[Uuid]) { let document = shared.action_executor.document(); @@ -1530,6 +1597,12 @@ impl PaneRenderer for InfopanelPane { } } + // Active raster layer's size + "to document size" — shown whenever a raster layer is + // active (independent of selection focus, since painting doesn't focus the layer). + if let Some(active_id) = *shared.active_layer_id { + self.render_raster_layer_section(ui, path, shared, active_id); + } + // Onion-skinning view settings — always available, regardless of selection. ui.add_space(8.0); ui.separator(); diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl index 8537377..bc52aec 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/linear_to_srgb.wgsl @@ -50,3 +50,34 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { a ); } + +// Highlight rolloff: identity below the knee, then a smooth C1 rolloff that maps [knee, ∞) → [knee, 1) +// so super-white (HDR) detail is recovered instead of hard-clipped. SDR below the knee is untouched. +fn highlight_rolloff(x: f32) -> f32 { + let knee = 0.8; + if x <= knee { + return x; + } + let headroom = 1.0 - knee; + return knee + headroom * (1.0 - exp(-(x - knee) / headroom)); +} + +// Variant of fs_main with highlight rolloff (document HDR output mode = Highlight rolloff). +@fragment +fn fs_main_rolloff(in: VertexOutput) -> @location(0) vec4 { + let linear = textureSample(input_tex, input_sampler, in.uv); + let a = linear.a; + let straight = select(linear.rgb / a, vec3(0.0), a <= 0.0); + let rolled = vec3( + highlight_rolloff(straight.r), + highlight_rolloff(straight.g), + highlight_rolloff(straight.b), + ); + + return vec4( + linear_to_srgb_channel(rolled.r), + linear_to_srgb_channel(rolled.g), + linear_to_srgb_channel(rolled.b), + a + ); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl new file mode 100644 index 0000000..cf906ea --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -0,0 +1,136 @@ +// NV12 → linear-RGB blit shader. +// +// Samples a hardware-decoded video frame stored as two planes — Y (R8Unorm) and +// interleaved CbCr (Rg8Unorm, half-res) — converts BT.709 Y'CbCr → gamma-encoded +// R'G'B', then sRGB→linear, and writes straight-alpha linear into the Rgba16Float +// HDR layer. This mirrors the software path (swscale → sRGB RGBA8 → sampled as +// Rgba8UnormSrgb → linear) so hardware- and software-decoded video match. +// +// The affine transform (viewport UV → frame UV) is the same packing as +// canvas_blit.wgsl's BlitTransform; `full_range` selects full vs. studio-swing +// de-quantization. + +struct Nv12Params { + col0: vec4, + col1: vec4, + col2: vec4, + // Y'CbCr→R'G'B' matrix from the source colorspace: [Cr→R, Cb→G, Cr→G, Cb→B]. + coeffs: vec4, + // .x = full_range; .y = transfer (0 gamma, 1 PQ, 2 HLG); .z = primaries (0 BT.709, 1 BT.2020). + // A vec4 keeps each block 16-aligned and the struct 80 bytes (Rust `[f32;4] + u32 + [u32;3]`). + flags: vec4, +} + +@group(0) @binding(0) var y_tex: texture_2d; +@group(0) @binding(1) var samp: sampler; +@group(0) @binding(2) var params: Nv12Params; +@group(0) @binding(3) var uv_tex: texture_2d; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var out: VertexOutput; + let x = f32((vertex_index & 1u) << 1u); + let y = f32(vertex_index & 2u); + out.position = vec4(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0); + out.uv = vec2(x, y); + return out; +} + +fn srgb_to_linear(c: vec3) -> vec3 { + let lo = c / 12.92; + let hi = pow((c + vec3(0.055)) / 1.055, vec3(2.4)); + return select(lo, hi, c > vec3(0.04045)); +} + +// SMPTE ST 2084 (PQ) EOTF: encoded [0,1] → absolute luminance, then normalize so the 203-nit +// graphics white = 1.0 (HDR highlights exceed 1.0). Per-channel. +fn pq_to_linear(c: vec3) -> vec3 { + let m1 = 0.1593017578125; + let m2 = 78.84375; + let c1 = 0.8359375; + let c2 = 18.8515625; + let c3 = 18.6875; + let e = pow(max(c, vec3(0.0)), vec3(1.0 / m2)); + let num = max(e - vec3(c1), vec3(0.0)); + let den = vec3(c2) - c3 * e; + let nits = pow(num / den, vec3(1.0 / m1)) * 10000.0; // 0..10000 cd/m² + return nits / 203.0; +} + +// ARIB STD-B67 (HLG) inverse-OETF → scene light, normalized so reference white (signal 0.75) = 1.0. +// The display OOTF is omitted (scene-referred compositing); approximate but reasonable for SDR-out. +fn hlg_to_linear(c: vec3) -> vec3 { + let a = 0.17883277; + let b = 0.28466892; + let cc = 0.55991073; + let lo = (c * c) / 3.0; + let hi = (exp((c - vec3(cc)) / a) + vec3(b)) / 12.0; + let scene = select(lo, hi, c > vec3(0.5)); + return scene / 0.26496256; // hlg_inv_oetf(0.75): put reference white at 1.0 +} + +// BT.2020 → BT.709 primaries, linear light (ITU-R BT.2087). Out-of-709 colours go negative → clamp. +fn bt2020_to_bt709(c: vec3) -> vec3 { + let r = 1.660491 * c.r - 0.587641 * c.g - 0.072850 * c.b; + let g = -0.124550 * c.r + 1.132900 * c.g - 0.008349 * c.b; + let b = -0.018151 * c.r - 0.100579 * c.g + 1.118730 * c.b; + return max(vec3(r, g, b), vec3(0.0)); +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let m = mat3x3(params.col0.xyz, params.col1.xyz, params.col2.xyz); + let frame_uv = (m * vec3(in.uv.x, in.uv.y, 1.0)).xy; + + if frame_uv.x < 0.0 || frame_uv.x > 1.0 + || frame_uv.y < 0.0 || frame_uv.y > 1.0 { + return vec4(0.0, 0.0, 0.0, 0.0); + } + + let yv = textureSample(y_tex, samp, frame_uv).r; + let cbcr = textureSample(uv_tex, samp, frame_uv).rg; + + var Y: f32; + var Cb: f32; + var Cr: f32; + if params.flags.x != 0u { + // Full ("JPEG") range: [0,255] luma, chroma centered at 128. + Y = yv; + Cb = cbcr.r - 0.5; + Cr = cbcr.g - 0.5; + } else { + // Studio swing: Y'∈[16,235], Cb/Cr∈[16,240]. + Y = (yv * 255.0 - 16.0) / 219.0; + Cb = (cbcr.r * 255.0 - 128.0) / 224.0; + Cr = (cbcr.g * 255.0 - 128.0) / 224.0; + } + + // Y'CbCr → gamma-encoded R'G'B' using the source colorspace's matrix. + let r = Y + params.coeffs.x * Cr; + let g = Y + params.coeffs.y * Cb + params.coeffs.z * Cr; + let b = Y + params.coeffs.w * Cb; + // Valid encoded signal is [0,1]; clamp before the EOTF (HDR comes from the EOTF, not overshoot). + let rgb_enc = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); + + // Encoded R'G'B' → scene-linear (graphics white = 1.0; HDR may exceed 1.0). + var rgb_lin: vec3; + if params.flags.y == 1u { + rgb_lin = pq_to_linear(rgb_enc); + } else if params.flags.y == 2u { + rgb_lin = hlg_to_linear(rgb_enc); + } else { + rgb_lin = srgb_to_linear(rgb_enc); + } + + // Wide-gamut → BT.709 in linear light to match the compositor's primaries. + if params.flags.z == 1u { + rgb_lin = bt2020_to_bt709(rgb_lin); + } + + return vec4(rgb_lin, 1.0); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 5bd3181..92d85be 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -135,6 +135,8 @@ struct SharedVelloResources { blit_bind_group_layout: wgpu::BindGroupLayout, /// HDR to sRGB blit pipeline (linear→sRGB conversion for display) hdr_blit_pipeline: wgpu::RenderPipeline, + /// Variant of `hdr_blit_pipeline` with highlight rolloff (document HDR output mode). + hdr_blit_pipeline_rolloff: wgpu::RenderPipeline, sampler: wgpu::Sampler, /// Shared image cache for avoiding re-decoding images every frame image_cache: Mutex, @@ -152,6 +154,8 @@ struct SharedVelloResources { gpu_brush: Mutex, /// Canvas blit pipeline (renders GPU canvas to layer sRGB buffer) canvas_blit: crate::gpu_brush::CanvasBlitPipeline, + /// NV12→linear blit for hardware-decoded video frames (GPU plane textures → HDR layer). + nv12_blit: crate::nv12_blit::Nv12BlitPipeline, /// True when Vello is running its CPU software renderer (either forced or GPU fallback). /// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area. is_cpu_renderer: bool, @@ -344,6 +348,41 @@ impl SharedVelloResources { cache: None, }); + // Variant with highlight rolloff (document HDR output mode); identical but `fs_main_rolloff`. + let hdr_blit_pipeline_rolloff = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("hdr_blit_pipeline_rolloff"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &hdr_shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &hdr_shader, + entry_point: Some("fs_main_rolloff"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + unclipped_depth: false, + polygon_mode: wgpu::PolygonMode::Fill, + conservative: false, + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + // Create sampler let sampler = device.create_sampler(&wgpu::SamplerDescriptor { label: Some("vello_blit_sampler"), @@ -372,6 +411,7 @@ impl SharedVelloResources { // Initialize GPU raster brush engine let gpu_brush = crate::gpu_brush::GpuBrushEngine::new(device); let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device); + let nv12_blit = crate::nv12_blit::Nv12BlitPipeline::new(device); println!("✅ Vello shared resources initialized (renderer, shaders, HDR compositor, effect processor, color converter, and GPU brush engine)"); @@ -380,6 +420,7 @@ impl SharedVelloResources { blit_pipeline, blit_bind_group_layout, hdr_blit_pipeline, + hdr_blit_pipeline_rolloff, sampler, image_cache: Mutex::new(lightningbeam_core::renderer::ImageCache::new()), video_manager, @@ -389,6 +430,7 @@ impl SharedVelloResources { srgb_to_linear, gpu_brush: Mutex::new(gpu_brush), canvas_blit, + nv12_blit, is_cpu_renderer: use_cpu || is_cpu_renderer, gpu_timer: Mutex::new(None), video_frame_cache: Mutex::new(VideoFrameTexCache::new()), @@ -1062,6 +1104,12 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // Let the cache page image bytes from the project container on a decode miss. image_cache.set_container_path(self.ctx.container_path.clone()); + // Preview composites on the shared device, so it can consume hardware-decoded GPU + // frames — but only the GPU renderer; the CPU fallback needs software frames. + if let Ok(mut vm) = shared.video_manager.lock() { + vm.set_render_hardware_ok(!shared.is_cpu_renderer); + } + let composite_result = if shared.is_cpu_renderer { lightningbeam_core::renderer::render_document_for_compositing_cpu( &self.ctx.document, @@ -1678,25 +1726,34 @@ impl egui_wgpu::CallbackTrait for VelloCallback { RenderedLayerType::Video { instances } => { // Video layer — per-instance: (cached) frame texture → blit → composite. for inst in instances { - if inst.rgba_data.is_empty() { continue; } + if inst.gpu.is_none() && inst.rgba_data.is_empty() { continue; } let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); if let (Some(hdr_layer_view), Some(hdr_view)) = ( buffer_pool.get_view(hdr_layer_handle), &instance_resources.hdr_texture_view, ) { - // Reuse the GPU texture for this frame if it's unchanged (a - // static/paused video → no CPU conversion, alloc, or upload). - // Timed into `blit_ms` (incl the cache lookup + per-frame view). let _t = std::time::Instant::now(); - let tex_view = shared - .video_frame_cache - .lock() - .unwrap() - .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height); let bt = crate::gpu_brush::BlitTransform::new( inst.transform, inst.width, inst.height, width, height, ); - shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + if let Some(gpu) = &inst.gpu { + // Hardware-decoded NV12 plane textures → linear RGB, no CPU upload. + let y_view = gpu.y.create_view(&Default::default()); + let uv_view = gpu.uv.create_view(&Default::default()); + shared.nv12_blit.blit( + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, + gpu.full_range, gpu.coeffs, gpu.transfer, gpu.primaries, + ); + } else { + // Reuse the GPU texture for this frame if it's unchanged (a + // static/paused video → no CPU conversion, alloc, or upload). + let tex_view = shared + .video_frame_cache + .lock() + .unwrap() + .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height); + shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + } cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0; let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new( @@ -2017,6 +2074,41 @@ impl egui_wgpu::CallbackTrait for VelloCallback { scene }; + // Active raster-layer border: a black+yellow dashed outline of the layer's canvas bounds (in + // document space) so its size is visible — especially when it differs from the document (e.g. + // after a doc resize). Two strokes sharing one dash pattern, offset by a dash so the yellow + // fills the black's gaps → interlocking black/yellow marching-ants. + if let Some(active_id) = self.ctx.active_layer_id { + if let Some(lightningbeam_core::layer::AnyLayer::Raster(rl)) = self.ctx.document.get_layer(&active_id) { + // Use playback_time (clip-local when editing a movie clip) like every other + // keyframe lookup in prepare(), and overlay_transform so the outline tracks the + // layer's pixels through any clip-instance affine (it equals camera_transform + // outside clip-edit mode). + if let Some(kf) = rl.keyframe_at(self.ctx.playback_time) { + let rect = vello::kurbo::Rect::new(0.0, 0.0, kf.width as f64, kf.height as f64); + // Sizes are in document space; divide by zoom so they're ~constant on screen. + let inv_zoom = 1.0 / (self.ctx.zoom as f64).max(1e-3); + let stroke_w = 1.5 * inv_zoom; + let dash = 6.0 * inv_zoom; + let pattern = [dash, dash]; + scene.stroke( + &vello::kurbo::Stroke::new(stroke_w).with_dashes(0.0, pattern), + overlay_transform, + vello::peniko::Color::new([0.0, 0.0, 0.0, 1.0]), + None, + &rect, + ); + scene.stroke( + &vello::kurbo::Stroke::new(stroke_w).with_dashes(dash, pattern), + overlay_transform, + vello::peniko::Color::new([1.0, 0.85, 0.0, 1.0]), + None, + &rect, + ); + } + } + } + // Render drag preview objects with transparency if let (Some(delta), Some(active_layer_id)) = (self.ctx.drag_delta, self.ctx.active_layer_id) { if let Some(layer) = self.ctx.document.get_layer(&active_layer_id) { @@ -2928,7 +3020,11 @@ impl egui_wgpu::CallbackTrait for VelloCallback { occlusion_query_set: None, }); - render_pass.set_pipeline(&shared.hdr_blit_pipeline); + let hdr_pipeline = match self.ctx.document.hdr_output_mode { + lightningbeam_core::document::HdrOutputMode::HighlightRolloff => &shared.hdr_blit_pipeline_rolloff, + lightningbeam_core::document::HdrOutputMode::Clip => &shared.hdr_blit_pipeline, + }; + render_pass.set_pipeline(hdr_pipeline); render_pass.set_bind_group(0, hdr_bind_group, &[]); render_pass.draw(0..3, 0..1); // Full-screen triangle (3 vertices) } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs index 7762112..5cfdca3 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/timeline.rs @@ -6151,20 +6151,11 @@ impl PaneRenderer for TimelinePane { let mut clip_instance = ClipInstance::new(dragging.clip_id) .with_timeline_start(drop_time); - // For video clips, scale to fill document dimensions + // For video clips, fit uniformly + centered (preserve aspect). + // Shared with the direct-import path via Transform::fit_centered. if dragging.clip_type == DragClipType::Video { if let Some((video_width, video_height)) = dragging.dimensions { - // Calculate scale to fill document - let scale_x = doc.width / video_width; - let scale_y = doc.height / video_height; - - clip_instance.transform.scale_x = scale_x; - clip_instance.transform.scale_y = scale_y; - - // Position at (0, 0) to center the scaled video - // (scaled dimensions = document dimensions, so top-left at origin centers it) - clip_instance.transform.x = 0.0; - clip_instance.transform.y = 0.0; + clip_instance.transform.fit_centered(video_width, video_height, doc.width, doc.height); } else { // No dimensions available, use document center clip_instance.transform.x = center_x; diff --git a/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs b/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs new file mode 100644 index 0000000..be793eb --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/svg_import.rs @@ -0,0 +1,330 @@ +//! SVG import → `VectorGraph`. +//! +//! Parses an `.svg` with usvg (which resolves CSS, converts shapes/rects/circles to +//! paths, and computes absolute transforms), then bakes each path's absolute transform +//! into geometry and builds a single [`VectorGraph`] that becomes one new vector layer. +//! +//! Scope (matches the export pass): paths with solid/gradient fills and strokes. `` +//! and `` nodes are skipped, and nested groups are flattened (their transforms are +//! already baked into each path's `abs_transform`). +//! +//! Known limitation: imported edges are NOT intersection-split, so the paint-bucket tool +//! may need to re-process imported art. Display, transform, and round-trip are fine. + +use kurbo::{CubicBez, Point as KPoint}; +use lightningbeam_core::gradient::{GradientExtend, GradientStop, GradientType, ShapeGradient}; +use lightningbeam_core::shape::{Cap, FillRule, Join, ShapeColor, StrokeStyle}; +use lightningbeam_core::vector_graph::{Direction, EdgeId, VectorGraph, VertexId}; +use resvg::usvg; +use usvg::tiny_skia_path::{PathSegment, Point as SkPoint}; + +/// Parse SVG bytes into a single flattened [`VectorGraph`] in document (canvas) space. +pub fn import_svg(bytes: &[u8]) -> Result { + let tree = usvg::Tree::from_data(bytes, &usvg::Options::default()) + .map_err(|e| format!("Failed to parse SVG: {e}"))?; + let mut graph = VectorGraph::new(); + walk_group(tree.root(), &mut graph); + if graph.edges.is_empty() { + return Err("SVG contained no importable vector paths".to_string()); + } + Ok(graph) +} + +fn walk_group(group: &usvg::Group, graph: &mut VectorGraph) { + for node in group.children() { + match node { + usvg::Node::Group(g) => walk_group(g, graph), + usvg::Node::Path(p) => convert_path(p, graph), + usvg::Node::Image(_) | usvg::Node::Text(_) => {} // skipped this pass + } + } +} + +fn convert_path(path: &usvg::Path, graph: &mut VectorGraph) { + if !path.is_visible() { + return; + } + let ts = path.abs_transform(); + // Bake the absolute transform into the geometry so everything lives in canvas space. + let Some(data) = path.data().clone().transform(ts) else { + return; + }; + + // One stroke style/colour shared by every edge of this path. + let stroke = path.stroke().map(|s| stroke_to_style(s, ts)); + + // Walk the (transformed) segments, allocating vertices/edges and recording the + // boundary cycle. `EdgeId::NONE` separates subpaths (outer contour + holes). + let mut boundary: Vec<(EdgeId, Direction)> = Vec::new(); + let mut have_subpath = false; + let mut cur_v = VertexId(0); + let mut cur_p = SkPoint::from_xy(0.0, 0.0); + let mut start_v = VertexId(0); + let mut start_p = SkPoint::from_xy(0.0, 0.0); + + for seg in data.segments() { + match seg { + PathSegment::MoveTo(p) => { + if have_subpath { + boundary.push((EdgeId::NONE, Direction::Forward)); + } + let v = graph.alloc_vertex(kp(p)); + cur_v = v; + cur_p = p; + start_v = v; + start_p = p; + have_subpath = true; + } + PathSegment::LineTo(p) => { + let (c1, c2) = line_ctrls(cur_p, p); + cur_v = add_edge(graph, &mut boundary, cur_v, cur_p, c1, c2, p, &stroke); + cur_p = p; + } + PathSegment::QuadTo(c, p) => { + let (c1, c2) = quad_to_cubic(cur_p, c, p); + cur_v = add_edge(graph, &mut boundary, cur_v, cur_p, c1, c2, p, &stroke); + cur_p = p; + } + PathSegment::CubicTo(c1, c2, p) => { + cur_v = add_edge(graph, &mut boundary, cur_v, cur_p, c1, c2, p, &stroke); + cur_p = p; + } + PathSegment::Close => { + // Close back to the subpath start (reusing its vertex) unless already there. + if cur_p != start_p { + let (c1, c2) = line_ctrls(cur_p, start_p); + let curve = CubicBez::new(kp(cur_p), kp(c1), kp(c2), kp(start_p)); + let (style, color) = split_stroke(&stroke); + let e = graph.alloc_edge(curve, cur_v, start_v, style, color); + boundary.push((e, Direction::Forward)); + } + cur_v = start_v; + cur_p = start_p; + } + } + } + + // Fill (if any) references the whole boundary cycle. + if let Some(fill) = path.fill() { + if !boundary.is_empty() { + let rule = match fill.rule() { + usvg::FillRule::NonZero => FillRule::NonZero, + usvg::FillRule::EvenOdd => FillRule::EvenOdd, + }; + let fid = graph.alloc_fill(boundary, None, rule); + let slot = &mut graph.fills[fid.idx()]; + match fill.paint() { + usvg::Paint::Color(c) => { + slot.color = Some(ShapeColor::rgba(c.red, c.green, c.blue, opacity_u8(fill.opacity()))); + } + usvg::Paint::LinearGradient(g) => { + let mut grad = linear_gradient(g, ts); + apply_fill_opacity(&mut grad, fill.opacity()); + slot.gradient_fill = Some(grad); + } + usvg::Paint::RadialGradient(g) => { + let mut grad = radial_gradient(g, ts); + apply_fill_opacity(&mut grad, fill.opacity()); + slot.gradient_fill = Some(grad); + } + usvg::Paint::Pattern(_) => { + // Patterns aren't representable yet — neutral gray so the shape stays visible. + slot.color = Some(ShapeColor::rgba(128, 128, 128, opacity_u8(fill.opacity()))); + } + } + } + } +} + +/// Allocate the end vertex + a cubic edge from `av`/`ap` to `bp`, recording it on the boundary. +fn add_edge( + graph: &mut VectorGraph, + boundary: &mut Vec<(EdgeId, Direction)>, + av: VertexId, + ap: SkPoint, + c1: SkPoint, + c2: SkPoint, + bp: SkPoint, + stroke: &Option<(StrokeStyle, ShapeColor)>, +) -> VertexId { + let bv = graph.alloc_vertex(kp(bp)); + let curve = CubicBez::new(kp(ap), kp(c1), kp(c2), kp(bp)); + let (style, color) = split_stroke(stroke); + let e = graph.alloc_edge(curve, av, bv, style, color); + boundary.push((e, Direction::Forward)); + bv +} + +fn split_stroke(stroke: &Option<(StrokeStyle, ShapeColor)>) -> (Option, Option) { + match stroke { + Some((s, c)) => (Some(s.clone()), Some(*c)), + None => (None, None), + } +} + +fn stroke_to_style(s: &usvg::Stroke, ts: usvg::Transform) -> (StrokeStyle, ShapeColor) { + let scale = transform_scale(ts) as f64; + let style = StrokeStyle { + width: s.width().get() as f64 * scale, + cap: match s.linecap() { + usvg::LineCap::Butt => Cap::Butt, + usvg::LineCap::Round => Cap::Round, + usvg::LineCap::Square => Cap::Square, + }, + join: match s.linejoin() { + usvg::LineJoin::Miter | usvg::LineJoin::MiterClip => Join::Miter, + usvg::LineJoin::Round => Join::Round, + usvg::LineJoin::Bevel => Join::Bevel, + }, + miter_limit: s.miterlimit().get() as f64, + }; + let color = match s.paint() { + usvg::Paint::Color(c) => ShapeColor::rgba(c.red, c.green, c.blue, opacity_u8(s.opacity())), + // Gradient/pattern strokes aren't representable per-edge — fall back to opaque black. + _ => ShapeColor::rgba(0, 0, 0, opacity_u8(s.opacity())), + }; + (style, color) +} + +/// Geometric-mean scale of the transform's linear part (for stroke-width baking). +fn transform_scale(ts: usvg::Transform) -> f32 { + (ts.sx * ts.sy - ts.kx * ts.ky).abs().sqrt() +} + +fn linear_gradient(g: &usvg::LinearGradient, abs: usvg::Transform) -> ShapeGradient { + let ct = abs.pre_concat(g.transform()); + let start = map_pt(ct, g.x1(), g.y1()); + let end = map_pt(ct, g.x2(), g.y2()); + let angle = (end.1 - start.1).atan2(end.0 - start.0).to_degrees() as f32; + ShapeGradient { + kind: GradientType::Linear, + stops: gradient_stops(g), + angle, + extend: spread(g), + start_world: Some(start), + end_world: Some(end), + } +} + +fn radial_gradient(g: &usvg::RadialGradient, abs: usvg::Transform) -> ShapeGradient { + let ct = abs.pre_concat(g.transform()); + // Our model stores center as start_world and a rim point (defining the radius) as end_world. + let center = map_pt(ct, g.cx(), g.cy()); + let rim = map_pt(ct, g.cx() + g.r().get(), g.cy()); + ShapeGradient { + kind: GradientType::Radial, + stops: gradient_stops(g), + angle: 0.0, + extend: spread(g), + start_world: Some(center), + end_world: Some(rim), + } +} + +/// Fold the path's `fill-opacity` into a gradient's stop alphas (SVG multiplies them). +fn apply_fill_opacity(grad: &mut ShapeGradient, op: usvg::Opacity) { + let f = op.get(); + if f >= 1.0 { + return; + } + for s in &mut grad.stops { + s.color.a = (s.color.a as f32 * f).round().clamp(0.0, 255.0) as u8; + } +} + +fn gradient_stops(base: &usvg::BaseGradient) -> Vec { + base.stops() + .iter() + .map(|s| GradientStop { + position: s.offset().get(), + color: ShapeColor::rgba(s.color().red, s.color().green, s.color().blue, opacity_u8(s.opacity())), + }) + .collect() +} + +fn spread(base: &usvg::BaseGradient) -> GradientExtend { + match base.spread_method() { + usvg::SpreadMethod::Pad => GradientExtend::Pad, + usvg::SpreadMethod::Reflect => GradientExtend::Reflect, + usvg::SpreadMethod::Repeat => GradientExtend::Repeat, + } +} + +// ── small geometry helpers ────────────────────────────────────────────────── + +fn kp(p: SkPoint) -> KPoint { + KPoint::new(p.x as f64, p.y as f64) +} + +fn map_pt(ts: usvg::Transform, x: f32, y: f32) -> (f64, f64) { + let mut p = SkPoint::from_xy(x, y); + ts.map_point(&mut p); + (p.x as f64, p.y as f64) +} + +fn lerp(a: SkPoint, b: SkPoint, t: f32) -> SkPoint { + SkPoint::from_xy(a.x + (b.x - a.x) * t, a.y + (b.y - a.y) * t) +} + +/// Degenerate cubic control points for a straight segment (matches our edge model). +fn line_ctrls(a: SkPoint, b: SkPoint) -> (SkPoint, SkPoint) { + (lerp(a, b, 1.0 / 3.0), lerp(a, b, 2.0 / 3.0)) +} + +/// Elevate a quadratic Bézier to a cubic. +fn quad_to_cubic(a: SkPoint, c: SkPoint, b: SkPoint) -> (SkPoint, SkPoint) { + let c1 = SkPoint::from_xy(a.x + 2.0 / 3.0 * (c.x - a.x), a.y + 2.0 / 3.0 * (c.y - a.y)); + let c2 = SkPoint::from_xy(b.x + 2.0 / 3.0 * (c.x - b.x), b.y + 2.0 / 3.0 * (c.y - b.y)); + (c1, c2) +} + +fn opacity_u8(o: usvg::Opacity) -> u8 { + (o.get() * 255.0).round().clamp(0.0, 255.0) as u8 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn imports_solid_rect_fill() { + let svg = br##""##; + let g = import_svg(svg).expect("import"); + assert!(!g.edges.is_empty(), "expected edges from the rect"); + let fills: Vec<_> = g.fills.iter().filter(|f| !f.deleted).collect(); + assert_eq!(fills.len(), 1, "one fill expected"); + let c = fills[0].color.expect("solid color"); + assert_eq!((c.r, c.g, c.b), (255, 0, 0), "red fill"); + } + + #[test] + fn imports_stroke_only() { + let svg = br##""##; + let g = import_svg(svg).expect("import"); + let stroked = g.edges.iter().filter(|e| !e.deleted && e.stroke_color.is_some()).count(); + assert!(stroked >= 1, "expected at least one stroked edge"); + let c = g.edges.iter().find_map(|e| e.stroke_color).unwrap(); + assert_eq!((c.r, c.g, c.b), (0, 255, 0), "green stroke"); + } + + #[test] + fn imports_linear_gradient() { + let svg = br##" + + + + "##; + let g = import_svg(svg).expect("import"); + let fills: Vec<_> = g.fills.iter().filter(|f| !f.deleted).collect(); + assert_eq!(fills.len(), 1); + let grad = fills[0].gradient_fill.as_ref().expect("gradient"); + assert_eq!(grad.stops.len(), 2); + assert!(grad.start_world.is_some() && grad.end_world.is_some()); + } + + #[test] + fn empty_svg_errors() { + let svg = br#""#; + assert!(import_svg(svg).is_err()); + } +}