From 1848c920d9d03afa137adc54b32c92633d9c5f88 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:21:14 -0400 Subject: [PATCH] editor: wire hardware video decode + NV12 preview compositing (Stage 3c-preview) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the dormant core HW-decode engine live for the preview path: - hw_video.rs: editor's HwVideoImporter — maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane textures on the *shared* device (the only one with the import extensions). install() creates the VAAPI device and injects it + the importer into the VideoManager. - main.rs: track whether the shared device is actually in use; only then (Linux, not LB_NO_SHARED_DEVICE) install hardware decode, using the CreationContext's shared device + adapter. - nv12_blit.rs + nv12_blit.wgsl: NV12 plane textures → BT.709 → sRGB-encoded → linear, written straight into the Rgba16Float HDR layer (no CPU upload). Colour math mirrors the software path so HW/SW video match; honours full_range. - stage.rs: the preview Video arm branches on inst.gpu (NV12 blit) vs rgba_data (existing upload+blit_straight); sets render_hardware_ok = !cpu_renderer so the CPU fallback still gets software frames. - video_exporter.rs: sets render_hardware_ok(false) before both compositing passes — export composites on the encoder's separate device, so a hardware decoder downloads to CPU instead (export stays software, correct). - dmabuf.rs: imported plane textures now also carry SAMPLED/TEXTURE_BINDING so they can be sampled by the NV12 blit (they were render-target-only); into_planes hands the textures to the longer-lived GpuVideoFrame. - video.rs: cache-key the GPU/CPU representation on want_gpu (HW-configured AND render_hardware_ok) so software-only decode keeps a single cache entry. Preview only this pass; export GPU-residency is the 3c-export follow-up. Untested at runtime here (no GPU/display in container) — both crates compile. --- .../gpu-video-encoder/src/dmabuf.rs | 13 +- .../lightningbeam-core/src/video.rs | 10 +- .../src/export/video_exporter.rs | 12 ++ .../lightningbeam-editor/src/hw_video.rs | 89 +++++++++ .../lightningbeam-editor/src/main.rs | 15 ++ .../lightningbeam-editor/src/nv12_blit.rs | 180 ++++++++++++++++++ .../src/panes/shaders/nv12_blit.wgsl | 84 ++++++++ .../lightningbeam-editor/src/panes/stage.rs | 38 +++- 8 files changed, 425 insertions(+), 16 deletions(-) create mode 100644 lightningbeam-ui/lightningbeam-editor/src/hw_video.rs create mode 100644 lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs create mode 100644 lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl diff --git a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs index a6e1644..492e0b4 100644 --- a/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs +++ b/lightningbeam-ui/gpu-video-encoder/src/dmabuf.rs @@ -47,6 +47,10 @@ impl ImportedNv12 { pub fn uv(&self) -> &wgpu::Texture { &self.uv } + /// Consume into the `(Y, UV)` plane textures (for handing to a longer-lived owner). + pub fn into_planes(self) -> (wgpu::Texture, wgpu::Texture) { + (self.y, self.uv) + } } /// Convenience: map a freshly-allocated `MappedSurface` and import it onto `drm`. @@ -113,6 +117,7 @@ pub fn import_raw( .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) .usage( vk::ImageUsageFlags::COLOR_ATTACHMENT + | vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_SRC | vk::ImageUsageFlags::TRANSFER_DST, ) @@ -178,7 +183,9 @@ pub fn import_raw( sample_count: 1, dimension: wgpu::TextureDimension::D2, format, - usage: wgpu_types::TextureUses::COLOR_TARGET | wgpu_types::TextureUses::COPY_SRC, + usage: wgpu_types::TextureUses::COLOR_TARGET + | wgpu_types::TextureUses::RESOURCE + | wgpu_types::TextureUses::COPY_SRC, memory_flags: wgpu_hal::MemoryFlags::empty(), view_formats: vec![], }; @@ -192,7 +199,9 @@ pub fn import_raw( sample_count: 1, dimension: wgpu::TextureDimension::D2, format, - usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::COPY_SRC, view_formats: &[], }, ) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 0ddd083..958f197 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -875,11 +875,13 @@ impl VideoManager { /// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the /// preview, false for export, which composites on a different device). pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option> { - let hardware_ok = self.render_hardware_ok; - // The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export + // Whether this pass wants (and can produce) a GPU frame. Gated on HW being configured at all + // so that with software-only decode preview and export share one cache entry (no double-cache). + let want_gpu = self.render_hardware_ok && self.hw_device.is_some(); + // The cache key includes (target size, want_gpu): preview (GPU, preview res) and export // (CPU, export res) request the same clip/time and must not collide or cross representation. let timestamp_ms = (timestamp * 1000.0) as i64; - let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok); + let cache_key = (*clip_id, timestamp_ms, target_w, target_h, want_gpu); // Check frame cache first if let Some(cached_frame) = self.frame_cache.get(&cache_key) { @@ -892,7 +894,7 @@ impl VideoManager { let mut decoder = decoder_arc.lock().ok()?; // Decode the frame at the requested target (capped to native by the decoder). - let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?; + let decoded = decoder.get_frame(timestamp, target_w, target_h, want_gpu).ok()?; drop(decoder); // release the lock before touching `self` // Create VideoFrame and cache it. diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index efa7990..a19a227 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -1066,6 +1066,13 @@ pub fn render_frame_to_rgba_hdr( Affine::IDENTITY }; + // Export composites on the encoder's own device, not the shared one, so it cannot use + // hardware-decoded GPU frames (textures can't cross devices). Force software frames; a + // hardware decoder downloads its surface to CPU instead. + if let Ok(mut vm) = video_manager.lock() { + vm.set_render_hardware_ok(false); + } + // Render document for compositing (returns per-layer scenes) let composite_result = render_document_for_compositing( document, @@ -1329,6 +1336,11 @@ pub fn render_frame_to_gpu_rgba( Affine::IDENTITY }; + // Export composites on a separate device — force software frames (see above). + if let Ok(mut vm) = video_manager.lock() { + vm.set_render_hardware_ok(false); + } + // Render document for compositing (returns per-layer scenes) let composite_result = render_document_for_compositing( document, diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs new file mode 100644 index 0000000..7032ae8 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -0,0 +1,89 @@ +//! Hardware video decode glue (Linux/VAAPI). The editor implements core's [`HwVideoImporter`]: +//! it maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane +//! textures on the **shared** device (the one eframe + the compositor run on, which has the +//! DMA-BUF-import extensions). [`install`] creates the VAAPI device and wires it into the +//! `VideoManager`. + +use ffmpeg_next::ffi as ff; +use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf}; +use lightningbeam_core::video::{GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager}; +use std::sync::{Arc, Mutex}; + +/// Imports decoded VAAPI surfaces onto the shared wgpu device. Holds clones of the shared +/// device + adapter (Arc-backed, cheap). +struct SharedHwImporter { + device: wgpu::Device, + adapter: wgpu::Adapter, +} + +impl HwVideoImporter for SharedHwImporter { + unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option { + let frame = av_frame as *mut ff::AVFrame; + + // Map the VAAPI surface to a DRM-PRIME DMA-BUF (read-only). + let drm_f = ff::av_frame_alloc(); + (*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32; + let flags = ff::AV_HWFRAME_MAP_DIRECT as i32 | ff::AV_HWFRAME_MAP_READ as i32; + if ff::av_hwframe_map(drm_f, frame, flags) < 0 { + ff::av_frame_free(&mut (drm_f as *mut _)); + return None; + } + + let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor; + let obj = &(*desc).objects[0]; + let width = (*frame).width as u32; + let height = (*frame).height as u32; + // NV12: Y then UV — two layers (one plane each) or one layer with two planes. + let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 { + (&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0]) + } else { + (&(*desc).layers[0].planes[0], &(*desc).layers[0].planes[1]) + }; + let buf = Nv12DmaBuf { + fd: obj.fd, + size: obj.size as u64, + modifier: obj.format_modifier, + width, + height, + y_offset: y_pl.offset as u64, + y_pitch: y_pl.pitch as u64, + uv_offset: uv_pl.offset as u64, + uv_pitch: uv_pl.pitch as u64, + }; + let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG; + + let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); + ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan + let (y, uv) = imported.ok()?.into_planes(); + Some(GpuVideoFrame { + y: Arc::new(y), + uv: Arc::new(uv), + width, + height, + full_range, + }) + } +} + +/// Create the VAAPI hardware device and install hardware decode into `vm`, importing onto the +/// shared `device`/`adapter`. Logs and no-ops if VAAPI is unavailable (→ software decode). +pub fn install(vm: &Arc>, device: &wgpu::Device, adapter: &wgpu::Adapter) { + match gpu_video_encoder::vaapi::create_device() { + Ok(hw_device) => { + let importer = Arc::new(SharedHwImporter { + device: device.clone(), + adapter: adapter.clone(), + }); + if let Ok(mut vm) = vm.lock() { + vm.set_hardware_decode( + HwDeviceHandle(hw_device as *mut std::ffi::c_void), + importer, + ); + } + println!("🎞 Hardware video decode enabled (VAAPI → shared device)"); + } + Err(e) => { + println!("🎞 Hardware video decode unavailable ({e}); using software decode"); + } + } +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index 08a522a..d9152ad 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -51,6 +51,9 @@ mod custom_cursor; mod tablet; mod debug_overlay; mod gpu_timer; +#[cfg(target_os = "linux")] +mod hw_video; +mod nv12_blit; #[cfg(debug_assertions)] mod test_mode; @@ -210,6 +213,10 @@ fn main() -> eframe::Result { // raw VkDevice persists with them. #[allow(unused_mut)] let mut wgpu_setup = lb_default_wgpu_setup(); + // Whether the shared VAAPI-capable device is in use — only then can decoded DMA-BUF frames be + // imported + composited, so hardware decode is injected into the VideoManager only if true. + #[allow(unused_assignments, unused_mut)] + let mut shared_device_active = false; #[cfg(target_os = "linux")] if std::env::var("LB_NO_SHARED_DEVICE").is_ok() { println!("🖥 Shared device disabled via LB_NO_SHARED_DEVICE; default wgpu device (software video decode)"); @@ -223,6 +230,7 @@ fn main() -> eframe::Result { device: drm.device.clone(), queue: drm.queue.clone(), }); + shared_device_active = true; } Err(e) => { println!("🖥 Shared device unavailable ({e}); default wgpu device (software video decode)"); @@ -294,6 +302,13 @@ fn main() -> eframe::Result { let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); #[cfg(not(debug_assertions))] let app = EditorApp::new(cc, layouts, theme); + // Wire hardware video decode into the VideoManager now that the shared device exists. + #[cfg(target_os = "linux")] + if shared_device_active { + if let Some(rs) = cc.wgpu_render_state.as_ref() { + hw_video::install(&app.video_manager, &rs.device, &rs.adapter); + } + } Ok(Box::new(app)) }), ) diff --git a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs new file mode 100644 index 0000000..3f26640 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs @@ -0,0 +1,180 @@ +//! NV12 → linear-RGB blit: composites a hardware-decoded video frame (two wgpu plane textures, +//! Y = R8Unorm + CbCr = Rg8Unorm) directly into the Rgba16Float HDR layer, with no CPU upload. +//! The colour math mirrors the software path (BT.709 → sRGB-encoded → linear) so hardware- and +//! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`. + +use crate::gpu_brush::BlitTransform; + +/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]) plus the +/// full-range flag. 64 bytes (48 for the matrix + a u32 + 12 padding). +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct Nv12Params { + transform: BlitTransform, + full_range: u32, + _pad: [u32; 3], +} + +pub struct Nv12BlitPipeline { + pipeline: wgpu::RenderPipeline, + bg_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, +} + +impl Nv12BlitPipeline { + pub fn new(device: &wgpu::Device) -> Self { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("nv12_blit_shader"), + source: wgpu::ShaderSource::Wgsl( + include_str!("panes/shaders/nv12_blit.wgsl").into(), + ), + }); + + let tex_entry = |binding: u32| wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }; + + let bg_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("nv12_blit_bgl"), + entries: &[ + tex_entry(0), // Y plane (R8Unorm) + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, + tex_entry(3), // CbCr plane (Rg8Unorm) + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("nv12_blit_pl"), + bind_group_layouts: &[&bg_layout], + push_constant_ranges: &[], + }); + + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("nv12_blit_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba16Float, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + // Bilinear: the frame is scaled to the output size; nearest would look blocky. + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("nv12_blit_sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Linear, + ..Default::default() + }); + + Self { pipeline, bg_layout, sampler } + } + + /// Convert + blit the NV12 frame into `target_view` (Rgba16Float, cleared to transparent), + /// positioned by `transform` (built like the RGBA video path's `BlitTransform`). + #[allow(clippy::too_many_arguments)] + pub fn blit( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + y_view: &wgpu::TextureView, + uv_view: &wgpu::TextureView, + target_view: &wgpu::TextureView, + transform: &BlitTransform, + full_range: bool, + ) { + let params = Nv12Params { + transform: *transform, + full_range: full_range as u32, + _pad: [0; 3], + }; + let param_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("nv12_blit_params"), + size: std::mem::size_of::() as u64, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + queue.write_buffer(¶m_buf, 0, bytemuck::bytes_of(¶ms)); + + let bg = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("nv12_blit_bg"), + layout: &self.bg_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(y_view) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&self.sampler) }, + wgpu::BindGroupEntry { binding: 2, resource: param_buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(uv_view) }, + ], + }); + + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("nv12_blit_encoder"), + }); + { + let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("nv12_blit_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: target_view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + }); + rp.set_pipeline(&self.pipeline); + rp.set_bind_group(0, &bg, &[]); + rp.draw(0..4, 0..1); + } + queue.submit(Some(encoder.finish())); + } +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl new file mode 100644 index 0000000..46d5d79 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -0,0 +1,84 @@ +// NV12 → linear-RGB blit shader. +// +// Samples a hardware-decoded video frame stored as two planes — Y (R8Unorm) and +// interleaved CbCr (Rg8Unorm, half-res) — converts BT.709 Y'CbCr → gamma-encoded +// R'G'B', then sRGB→linear, and writes straight-alpha linear into the Rgba16Float +// HDR layer. This mirrors the software path (swscale → sRGB RGBA8 → sampled as +// Rgba8UnormSrgb → linear) so hardware- and software-decoded video match. +// +// The affine transform (viewport UV → frame UV) is the same packing as +// canvas_blit.wgsl's BlitTransform; `full_range` selects full vs. studio-swing +// de-quantization. + +struct Nv12Params { + col0: vec4, + col1: vec4, + col2: vec4, + full_range: u32, + _pad: vec3, +} + +@group(0) @binding(0) var y_tex: texture_2d; +@group(0) @binding(1) var samp: sampler; +@group(0) @binding(2) var params: Nv12Params; +@group(0) @binding(3) var uv_tex: texture_2d; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) uv: vec2, +} + +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { + var out: VertexOutput; + let x = f32((vertex_index & 1u) << 1u); + let y = f32(vertex_index & 2u); + out.position = vec4(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0); + out.uv = vec2(x, y); + return out; +} + +fn srgb_to_linear(c: vec3) -> vec3 { + let lo = c / 12.92; + let hi = pow((c + vec3(0.055)) / 1.055, vec3(2.4)); + return select(lo, hi, c > vec3(0.04045)); +} + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + let m = mat3x3(params.col0.xyz, params.col1.xyz, params.col2.xyz); + let frame_uv = (m * vec3(in.uv.x, in.uv.y, 1.0)).xy; + + if frame_uv.x < 0.0 || frame_uv.x > 1.0 + || frame_uv.y < 0.0 || frame_uv.y > 1.0 { + return vec4(0.0, 0.0, 0.0, 0.0); + } + + let yv = textureSample(y_tex, samp, frame_uv).r; + let cbcr = textureSample(uv_tex, samp, frame_uv).rg; + + var Y: f32; + var Cb: f32; + var Cr: f32; + if params.full_range != 0u { + // Full ("JPEG") range: [0,255] luma, chroma centered at 128. + Y = yv; + Cb = cbcr.r - 0.5; + Cr = cbcr.g - 0.5; + } else { + // Studio swing: Y'∈[16,235], Cb/Cr∈[16,240]. + Y = (yv * 255.0 - 16.0) / 219.0; + Cb = (cbcr.r * 255.0 - 128.0) / 224.0; + Cr = (cbcr.g * 255.0 - 128.0) / 224.0; + } + + // BT.709 Y'CbCr → gamma-encoded R'G'B'. + let r = Y + 1.5748 * Cr; + let g = Y - 0.1873 * Cb - 0.4681 * Cr; + let b = Y + 1.8556 * Cb; + let rgb_gamma = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); + + // R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer. + let rgb_lin = srgb_to_linear(rgb_gamma); + return vec4(rgb_lin, 1.0); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 5bd3181..de42f0c 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -152,6 +152,8 @@ struct SharedVelloResources { gpu_brush: Mutex, /// Canvas blit pipeline (renders GPU canvas to layer sRGB buffer) canvas_blit: crate::gpu_brush::CanvasBlitPipeline, + /// NV12→linear blit for hardware-decoded video frames (GPU plane textures → HDR layer). + nv12_blit: crate::nv12_blit::Nv12BlitPipeline, /// True when Vello is running its CPU software renderer (either forced or GPU fallback). /// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area. is_cpu_renderer: bool, @@ -372,6 +374,7 @@ impl SharedVelloResources { // Initialize GPU raster brush engine let gpu_brush = crate::gpu_brush::GpuBrushEngine::new(device); let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device); + let nv12_blit = crate::nv12_blit::Nv12BlitPipeline::new(device); println!("✅ Vello shared resources initialized (renderer, shaders, HDR compositor, effect processor, color converter, and GPU brush engine)"); @@ -389,6 +392,7 @@ impl SharedVelloResources { srgb_to_linear, gpu_brush: Mutex::new(gpu_brush), canvas_blit, + nv12_blit, is_cpu_renderer: use_cpu || is_cpu_renderer, gpu_timer: Mutex::new(None), video_frame_cache: Mutex::new(VideoFrameTexCache::new()), @@ -1062,6 +1066,12 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // Let the cache page image bytes from the project container on a decode miss. image_cache.set_container_path(self.ctx.container_path.clone()); + // Preview composites on the shared device, so it can consume hardware-decoded GPU + // frames — but only the GPU renderer; the CPU fallback needs software frames. + if let Ok(mut vm) = shared.video_manager.lock() { + vm.set_render_hardware_ok(!shared.is_cpu_renderer); + } + let composite_result = if shared.is_cpu_renderer { lightningbeam_core::renderer::render_document_for_compositing_cpu( &self.ctx.document, @@ -1678,25 +1688,33 @@ impl egui_wgpu::CallbackTrait for VelloCallback { RenderedLayerType::Video { instances } => { // Video layer — per-instance: (cached) frame texture → blit → composite. for inst in instances { - if inst.rgba_data.is_empty() { continue; } + if inst.gpu.is_none() && inst.rgba_data.is_empty() { continue; } let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); if let (Some(hdr_layer_view), Some(hdr_view)) = ( buffer_pool.get_view(hdr_layer_handle), &instance_resources.hdr_texture_view, ) { - // Reuse the GPU texture for this frame if it's unchanged (a - // static/paused video → no CPU conversion, alloc, or upload). - // Timed into `blit_ms` (incl the cache lookup + per-frame view). let _t = std::time::Instant::now(); - let tex_view = shared - .video_frame_cache - .lock() - .unwrap() - .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height); let bt = crate::gpu_brush::BlitTransform::new( inst.transform, inst.width, inst.height, width, height, ); - shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + if let Some(gpu) = &inst.gpu { + // Hardware-decoded NV12 plane textures → linear RGB, no CPU upload. + let y_view = gpu.y.create_view(&Default::default()); + let uv_view = gpu.uv.create_view(&Default::default()); + shared.nv12_blit.blit( + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, + ); + } else { + // Reuse the GPU texture for this frame if it's unchanged (a + // static/paused video → no CPU conversion, alloc, or upload). + let tex_view = shared + .video_frame_cache + .lock() + .unwrap() + .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height); + shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + } cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0; let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(