editor: wire hardware video decode + NV12 preview compositing (Stage 3c-preview)

Make the dormant core HW-decode engine live for the preview path:

- hw_video.rs: editor's HwVideoImporter — maps a decoded VAAPI surface to a
  DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane textures on the *shared*
  device (the only one with the import extensions). install() creates the VAAPI
  device and injects it + the importer into the VideoManager.
- main.rs: track whether the shared device is actually in use; only then (Linux,
  not LB_NO_SHARED_DEVICE) install hardware decode, using the CreationContext's
  shared device + adapter.
- nv12_blit.rs + nv12_blit.wgsl: NV12 plane textures → BT.709 → sRGB-encoded →
  linear, written straight into the Rgba16Float HDR layer (no CPU upload). Colour
  math mirrors the software path so HW/SW video match; honours full_range.
- stage.rs: the preview Video arm branches on inst.gpu (NV12 blit) vs rgba_data
  (existing upload+blit_straight); sets render_hardware_ok = !cpu_renderer so the
  CPU fallback still gets software frames.
- video_exporter.rs: sets render_hardware_ok(false) before both compositing
  passes — export composites on the encoder's separate device, so a hardware
  decoder downloads to CPU instead (export stays software, correct).
- dmabuf.rs: imported plane textures now also carry SAMPLED/TEXTURE_BINDING so
  they can be sampled by the NV12 blit (they were render-target-only); into_planes
  hands the textures to the longer-lived GpuVideoFrame.
- video.rs: cache-key the GPU/CPU representation on want_gpu (HW-configured AND
  render_hardware_ok) so software-only decode keeps a single cache entry.

Preview only this pass; export GPU-residency is the 3c-export follow-up. Untested
at runtime here (no GPU/display in container) — both crates compile.
This commit is contained in:
Skyler Lehmkuhl 2026-06-26 02:21:14 -04:00
parent 863edc80fc
commit 1848c920d9
8 changed files with 425 additions and 16 deletions

View File

@ -47,6 +47,10 @@ impl ImportedNv12 {
pub fn uv(&self) -> &wgpu::Texture { pub fn uv(&self) -> &wgpu::Texture {
&self.uv &self.uv
} }
/// Consume into the `(Y, UV)` plane textures (for handing to a longer-lived owner).
pub fn into_planes(self) -> (wgpu::Texture, wgpu::Texture) {
(self.y, self.uv)
}
} }
/// Convenience: map a freshly-allocated `MappedSurface` and import it onto `drm`. /// Convenience: map a freshly-allocated `MappedSurface` and import it onto `drm`.
@ -113,6 +117,7 @@ pub fn import_raw(
.tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT) .tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT)
.usage( .usage(
vk::ImageUsageFlags::COLOR_ATTACHMENT vk::ImageUsageFlags::COLOR_ATTACHMENT
| vk::ImageUsageFlags::SAMPLED
| vk::ImageUsageFlags::TRANSFER_SRC | vk::ImageUsageFlags::TRANSFER_SRC
| vk::ImageUsageFlags::TRANSFER_DST, | vk::ImageUsageFlags::TRANSFER_DST,
) )
@ -178,7 +183,9 @@ pub fn import_raw(
sample_count: 1, sample_count: 1,
dimension: wgpu::TextureDimension::D2, dimension: wgpu::TextureDimension::D2,
format, format,
usage: wgpu_types::TextureUses::COLOR_TARGET | wgpu_types::TextureUses::COPY_SRC, usage: wgpu_types::TextureUses::COLOR_TARGET
| wgpu_types::TextureUses::RESOURCE
| wgpu_types::TextureUses::COPY_SRC,
memory_flags: wgpu_hal::MemoryFlags::empty(), memory_flags: wgpu_hal::MemoryFlags::empty(),
view_formats: vec![], view_formats: vec![],
}; };
@ -192,7 +199,9 @@ pub fn import_raw(
sample_count: 1, sample_count: 1,
dimension: wgpu::TextureDimension::D2, dimension: wgpu::TextureDimension::D2,
format, format,
usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, usage: wgpu::TextureUsages::RENDER_ATTACHMENT
| wgpu::TextureUsages::TEXTURE_BINDING
| wgpu::TextureUsages::COPY_SRC,
view_formats: &[], view_formats: &[],
}, },
) )

View File

@ -875,11 +875,13 @@ impl VideoManager {
/// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the /// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the
/// preview, false for export, which composites on a different device). /// preview, false for export, which composites on a different device).
pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> { pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> {
let hardware_ok = self.render_hardware_ok; // Whether this pass wants (and can produce) a GPU frame. Gated on HW being configured at all
// The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export // so that with software-only decode preview and export share one cache entry (no double-cache).
let want_gpu = self.render_hardware_ok && self.hw_device.is_some();
// The cache key includes (target size, want_gpu): preview (GPU, preview res) and export
// (CPU, export res) request the same clip/time and must not collide or cross representation. // (CPU, export res) request the same clip/time and must not collide or cross representation.
let timestamp_ms = (timestamp * 1000.0) as i64; let timestamp_ms = (timestamp * 1000.0) as i64;
let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok); let cache_key = (*clip_id, timestamp_ms, target_w, target_h, want_gpu);
// Check frame cache first // Check frame cache first
if let Some(cached_frame) = self.frame_cache.get(&cache_key) { if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
@ -892,7 +894,7 @@ impl VideoManager {
let mut decoder = decoder_arc.lock().ok()?; let mut decoder = decoder_arc.lock().ok()?;
// Decode the frame at the requested target (capped to native by the decoder). // Decode the frame at the requested target (capped to native by the decoder).
let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?; let decoded = decoder.get_frame(timestamp, target_w, target_h, want_gpu).ok()?;
drop(decoder); // release the lock before touching `self` drop(decoder); // release the lock before touching `self`
// Create VideoFrame and cache it. // Create VideoFrame and cache it.

View File

@ -1066,6 +1066,13 @@ pub fn render_frame_to_rgba_hdr(
Affine::IDENTITY Affine::IDENTITY
}; };
// Export composites on the encoder's own device, not the shared one, so it cannot use
// hardware-decoded GPU frames (textures can't cross devices). Force software frames; a
// hardware decoder downloads its surface to CPU instead.
if let Ok(mut vm) = video_manager.lock() {
vm.set_render_hardware_ok(false);
}
// Render document for compositing (returns per-layer scenes) // Render document for compositing (returns per-layer scenes)
let composite_result = render_document_for_compositing( let composite_result = render_document_for_compositing(
document, document,
@ -1329,6 +1336,11 @@ pub fn render_frame_to_gpu_rgba(
Affine::IDENTITY Affine::IDENTITY
}; };
// Export composites on a separate device — force software frames (see above).
if let Ok(mut vm) = video_manager.lock() {
vm.set_render_hardware_ok(false);
}
// Render document for compositing (returns per-layer scenes) // Render document for compositing (returns per-layer scenes)
let composite_result = render_document_for_compositing( let composite_result = render_document_for_compositing(
document, document,

View File

@ -0,0 +1,89 @@
//! Hardware video decode glue (Linux/VAAPI). The editor implements core's [`HwVideoImporter`]:
//! it maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane
//! textures on the **shared** device (the one eframe + the compositor run on, which has the
//! DMA-BUF-import extensions). [`install`] creates the VAAPI device and wires it into the
//! `VideoManager`.
use ffmpeg_next::ffi as ff;
use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf};
use lightningbeam_core::video::{GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager};
use std::sync::{Arc, Mutex};
/// Imports decoded VAAPI surfaces onto the shared wgpu device. Holds clones of the shared
/// device + adapter (Arc-backed, cheap).
struct SharedHwImporter {
device: wgpu::Device,
adapter: wgpu::Adapter,
}
impl HwVideoImporter for SharedHwImporter {
unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option<GpuVideoFrame> {
let frame = av_frame as *mut ff::AVFrame;
// Map the VAAPI surface to a DRM-PRIME DMA-BUF (read-only).
let drm_f = ff::av_frame_alloc();
(*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
let flags = ff::AV_HWFRAME_MAP_DIRECT as i32 | ff::AV_HWFRAME_MAP_READ as i32;
if ff::av_hwframe_map(drm_f, frame, flags) < 0 {
ff::av_frame_free(&mut (drm_f as *mut _));
return None;
}
let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor;
let obj = &(*desc).objects[0];
let width = (*frame).width as u32;
let height = (*frame).height as u32;
// NV12: Y then UV — two layers (one plane each) or one layer with two planes.
let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 {
(&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0])
} else {
(&(*desc).layers[0].planes[0], &(*desc).layers[0].planes[1])
};
let buf = Nv12DmaBuf {
fd: obj.fd,
size: obj.size as u64,
modifier: obj.format_modifier,
width,
height,
y_offset: y_pl.offset as u64,
y_pitch: y_pl.pitch as u64,
uv_offset: uv_pl.offset as u64,
uv_pitch: uv_pl.pitch as u64,
};
let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG;
let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf);
ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan
let (y, uv) = imported.ok()?.into_planes();
Some(GpuVideoFrame {
y: Arc::new(y),
uv: Arc::new(uv),
width,
height,
full_range,
})
}
}
/// Create the VAAPI hardware device and install hardware decode into `vm`, importing onto the
/// shared `device`/`adapter`. Logs and no-ops if VAAPI is unavailable (→ software decode).
pub fn install(vm: &Arc<Mutex<VideoManager>>, device: &wgpu::Device, adapter: &wgpu::Adapter) {
match gpu_video_encoder::vaapi::create_device() {
Ok(hw_device) => {
let importer = Arc::new(SharedHwImporter {
device: device.clone(),
adapter: adapter.clone(),
});
if let Ok(mut vm) = vm.lock() {
vm.set_hardware_decode(
HwDeviceHandle(hw_device as *mut std::ffi::c_void),
importer,
);
}
println!("🎞 Hardware video decode enabled (VAAPI → shared device)");
}
Err(e) => {
println!("🎞 Hardware video decode unavailable ({e}); using software decode");
}
}
}

View File

@ -51,6 +51,9 @@ mod custom_cursor;
mod tablet; mod tablet;
mod debug_overlay; mod debug_overlay;
mod gpu_timer; mod gpu_timer;
#[cfg(target_os = "linux")]
mod hw_video;
mod nv12_blit;
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
mod test_mode; mod test_mode;
@ -210,6 +213,10 @@ fn main() -> eframe::Result {
// raw VkDevice persists with them. // raw VkDevice persists with them.
#[allow(unused_mut)] #[allow(unused_mut)]
let mut wgpu_setup = lb_default_wgpu_setup(); let mut wgpu_setup = lb_default_wgpu_setup();
// Whether the shared VAAPI-capable device is in use — only then can decoded DMA-BUF frames be
// imported + composited, so hardware decode is injected into the VideoManager only if true.
#[allow(unused_assignments, unused_mut)]
let mut shared_device_active = false;
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
if std::env::var("LB_NO_SHARED_DEVICE").is_ok() { if std::env::var("LB_NO_SHARED_DEVICE").is_ok() {
println!("🖥 Shared device disabled via LB_NO_SHARED_DEVICE; default wgpu device (software video decode)"); println!("🖥 Shared device disabled via LB_NO_SHARED_DEVICE; default wgpu device (software video decode)");
@ -223,6 +230,7 @@ fn main() -> eframe::Result {
device: drm.device.clone(), device: drm.device.clone(),
queue: drm.queue.clone(), queue: drm.queue.clone(),
}); });
shared_device_active = true;
} }
Err(e) => { Err(e) => {
println!("🖥 Shared device unavailable ({e}); default wgpu device (software video decode)"); println!("🖥 Shared device unavailable ({e}); default wgpu device (software video decode)");
@ -294,6 +302,13 @@ fn main() -> eframe::Result {
let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app); let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app);
#[cfg(not(debug_assertions))] #[cfg(not(debug_assertions))]
let app = EditorApp::new(cc, layouts, theme); let app = EditorApp::new(cc, layouts, theme);
// Wire hardware video decode into the VideoManager now that the shared device exists.
#[cfg(target_os = "linux")]
if shared_device_active {
if let Some(rs) = cc.wgpu_render_state.as_ref() {
hw_video::install(&app.video_manager, &rs.device, &rs.adapter);
}
}
Ok(Box::new(app)) Ok(Box::new(app))
}), }),
) )

View File

@ -0,0 +1,180 @@
//! NV12 → linear-RGB blit: composites a hardware-decoded video frame (two wgpu plane textures,
//! Y = R8Unorm + CbCr = Rg8Unorm) directly into the Rgba16Float HDR layer, with no CPU upload.
//! The colour math mirrors the software path (BT.709 → sRGB-encoded → linear) so hardware- and
//! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`.
use crate::gpu_brush::BlitTransform;
/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]) plus the
/// full-range flag. 64 bytes (48 for the matrix + a u32 + 12 padding).
#[repr(C)]
#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)]
struct Nv12Params {
transform: BlitTransform,
full_range: u32,
_pad: [u32; 3],
}
pub struct Nv12BlitPipeline {
pipeline: wgpu::RenderPipeline,
bg_layout: wgpu::BindGroupLayout,
sampler: wgpu::Sampler,
}
impl Nv12BlitPipeline {
pub fn new(device: &wgpu::Device) -> Self {
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("nv12_blit_shader"),
source: wgpu::ShaderSource::Wgsl(
include_str!("panes/shaders/nv12_blit.wgsl").into(),
),
});
let tex_entry = |binding: u32| wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
};
let bg_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("nv12_blit_bgl"),
entries: &[
tex_entry(0), // Y plane (R8Unorm)
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
tex_entry(3), // CbCr plane (Rg8Unorm)
],
});
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("nv12_blit_pl"),
bind_group_layouts: &[&bg_layout],
push_constant_ranges: &[],
});
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("nv12_blit_pipeline"),
layout: Some(&pipeline_layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: Some("vs_main"),
buffers: &[],
compilation_options: Default::default(),
},
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: Some("fs_main"),
targets: &[Some(wgpu::ColorTargetState {
format: wgpu::TextureFormat::Rgba16Float,
blend: None,
write_mask: wgpu::ColorWrites::ALL,
})],
compilation_options: Default::default(),
}),
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleStrip,
..Default::default()
},
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview: None,
cache: None,
});
// Bilinear: the frame is scaled to the output size; nearest would look blocky.
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
label: Some("nv12_blit_sampler"),
address_mode_u: wgpu::AddressMode::ClampToEdge,
address_mode_v: wgpu::AddressMode::ClampToEdge,
address_mode_w: wgpu::AddressMode::ClampToEdge,
mag_filter: wgpu::FilterMode::Linear,
min_filter: wgpu::FilterMode::Linear,
mipmap_filter: wgpu::FilterMode::Linear,
..Default::default()
});
Self { pipeline, bg_layout, sampler }
}
/// Convert + blit the NV12 frame into `target_view` (Rgba16Float, cleared to transparent),
/// positioned by `transform` (built like the RGBA video path's `BlitTransform`).
#[allow(clippy::too_many_arguments)]
pub fn blit(
&self,
device: &wgpu::Device,
queue: &wgpu::Queue,
y_view: &wgpu::TextureView,
uv_view: &wgpu::TextureView,
target_view: &wgpu::TextureView,
transform: &BlitTransform,
full_range: bool,
) {
let params = Nv12Params {
transform: *transform,
full_range: full_range as u32,
_pad: [0; 3],
};
let param_buf = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("nv12_blit_params"),
size: std::mem::size_of::<Nv12Params>() as u64,
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
queue.write_buffer(&param_buf, 0, bytemuck::bytes_of(&params));
let bg = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("nv12_blit_bg"),
layout: &self.bg_layout,
entries: &[
wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(y_view) },
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&self.sampler) },
wgpu::BindGroupEntry { binding: 2, resource: param_buf.as_entire_binding() },
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(uv_view) },
],
});
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("nv12_blit_encoder"),
});
{
let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("nv12_blit_pass"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view: target_view,
resolve_target: None,
depth_slice: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
store: wgpu::StoreOp::Store,
},
})],
depth_stencil_attachment: None,
occlusion_query_set: None,
timestamp_writes: None,
});
rp.set_pipeline(&self.pipeline);
rp.set_bind_group(0, &bg, &[]);
rp.draw(0..4, 0..1);
}
queue.submit(Some(encoder.finish()));
}
}

View File

@ -0,0 +1,84 @@
// NV12 linear-RGB blit shader.
//
// Samples a hardware-decoded video frame stored as two planes Y (R8Unorm) and
// interleaved CbCr (Rg8Unorm, half-res) converts BT.709 Y'CbCr gamma-encoded
// R'G'B', then sRGBlinear, and writes straight-alpha linear into the Rgba16Float
// HDR layer. This mirrors the software path (swscale sRGB RGBA8 sampled as
// Rgba8UnormSrgb linear) so hardware- and software-decoded video match.
//
// The affine transform (viewport UV frame UV) is the same packing as
// canvas_blit.wgsl's BlitTransform; `full_range` selects full vs. studio-swing
// de-quantization.
struct Nv12Params {
col0: vec4<f32>,
col1: vec4<f32>,
col2: vec4<f32>,
full_range: u32,
_pad: vec3<u32>,
}
@group(0) @binding(0) var y_tex: texture_2d<f32>;
@group(0) @binding(1) var samp: sampler;
@group(0) @binding(2) var<uniform> params: Nv12Params;
@group(0) @binding(3) var uv_tex: texture_2d<f32>;
struct VertexOutput {
@builtin(position) position: vec4<f32>,
@location(0) uv: vec2<f32>,
}
@vertex
fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput {
var out: VertexOutput;
let x = f32((vertex_index & 1u) << 1u);
let y = f32(vertex_index & 2u);
out.position = vec4<f32>(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0);
out.uv = vec2<f32>(x, y);
return out;
}
fn srgb_to_linear(c: vec3<f32>) -> vec3<f32> {
let lo = c / 12.92;
let hi = pow((c + vec3<f32>(0.055)) / 1.055, vec3<f32>(2.4));
return select(lo, hi, c > vec3<f32>(0.04045));
}
@fragment
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
let m = mat3x3<f32>(params.col0.xyz, params.col1.xyz, params.col2.xyz);
let frame_uv = (m * vec3<f32>(in.uv.x, in.uv.y, 1.0)).xy;
if frame_uv.x < 0.0 || frame_uv.x > 1.0
|| frame_uv.y < 0.0 || frame_uv.y > 1.0 {
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
}
let yv = textureSample(y_tex, samp, frame_uv).r;
let cbcr = textureSample(uv_tex, samp, frame_uv).rg;
var Y: f32;
var Cb: f32;
var Cr: f32;
if params.full_range != 0u {
// Full ("JPEG") range: [0,255] luma, chroma centered at 128.
Y = yv;
Cb = cbcr.r - 0.5;
Cr = cbcr.g - 0.5;
} else {
// Studio swing: Y'[16,235], Cb/Cr[16,240].
Y = (yv * 255.0 - 16.0) / 219.0;
Cb = (cbcr.r * 255.0 - 128.0) / 224.0;
Cr = (cbcr.g * 255.0 - 128.0) / 224.0;
}
// BT.709 Y'CbCr gamma-encoded R'G'B'.
let r = Y + 1.5748 * Cr;
let g = Y - 0.1873 * Cb - 0.4681 * Cr;
let b = Y + 1.8556 * Cb;
let rgb_gamma = clamp(vec3<f32>(r, g, b), vec3<f32>(0.0), vec3<f32>(1.0));
// R'G'B' is gamma-encoded; the HDR target is linear undo the transfer.
let rgb_lin = srgb_to_linear(rgb_gamma);
return vec4<f32>(rgb_lin, 1.0);
}

View File

@ -152,6 +152,8 @@ struct SharedVelloResources {
gpu_brush: Mutex<crate::gpu_brush::GpuBrushEngine>, gpu_brush: Mutex<crate::gpu_brush::GpuBrushEngine>,
/// Canvas blit pipeline (renders GPU canvas to layer sRGB buffer) /// Canvas blit pipeline (renders GPU canvas to layer sRGB buffer)
canvas_blit: crate::gpu_brush::CanvasBlitPipeline, canvas_blit: crate::gpu_brush::CanvasBlitPipeline,
/// NV12→linear blit for hardware-decoded video frames (GPU plane textures → HDR layer).
nv12_blit: crate::nv12_blit::Nv12BlitPipeline,
/// True when Vello is running its CPU software renderer (either forced or GPU fallback). /// True when Vello is running its CPU software renderer (either forced or GPU fallback).
/// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area. /// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area.
is_cpu_renderer: bool, is_cpu_renderer: bool,
@ -372,6 +374,7 @@ impl SharedVelloResources {
// Initialize GPU raster brush engine // Initialize GPU raster brush engine
let gpu_brush = crate::gpu_brush::GpuBrushEngine::new(device); let gpu_brush = crate::gpu_brush::GpuBrushEngine::new(device);
let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device); let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device);
let nv12_blit = crate::nv12_blit::Nv12BlitPipeline::new(device);
println!("✅ Vello shared resources initialized (renderer, shaders, HDR compositor, effect processor, color converter, and GPU brush engine)"); println!("✅ Vello shared resources initialized (renderer, shaders, HDR compositor, effect processor, color converter, and GPU brush engine)");
@ -389,6 +392,7 @@ impl SharedVelloResources {
srgb_to_linear, srgb_to_linear,
gpu_brush: Mutex::new(gpu_brush), gpu_brush: Mutex::new(gpu_brush),
canvas_blit, canvas_blit,
nv12_blit,
is_cpu_renderer: use_cpu || is_cpu_renderer, is_cpu_renderer: use_cpu || is_cpu_renderer,
gpu_timer: Mutex::new(None), gpu_timer: Mutex::new(None),
video_frame_cache: Mutex::new(VideoFrameTexCache::new()), video_frame_cache: Mutex::new(VideoFrameTexCache::new()),
@ -1062,6 +1066,12 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
// Let the cache page image bytes from the project container on a decode miss. // Let the cache page image bytes from the project container on a decode miss.
image_cache.set_container_path(self.ctx.container_path.clone()); image_cache.set_container_path(self.ctx.container_path.clone());
// Preview composites on the shared device, so it can consume hardware-decoded GPU
// frames — but only the GPU renderer; the CPU fallback needs software frames.
if let Ok(mut vm) = shared.video_manager.lock() {
vm.set_render_hardware_ok(!shared.is_cpu_renderer);
}
let composite_result = if shared.is_cpu_renderer { let composite_result = if shared.is_cpu_renderer {
lightningbeam_core::renderer::render_document_for_compositing_cpu( lightningbeam_core::renderer::render_document_for_compositing_cpu(
&self.ctx.document, &self.ctx.document,
@ -1678,25 +1688,33 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
RenderedLayerType::Video { instances } => { RenderedLayerType::Video { instances } => {
// Video layer — per-instance: (cached) frame texture → blit → composite. // Video layer — per-instance: (cached) frame texture → blit → composite.
for inst in instances { for inst in instances {
if inst.rgba_data.is_empty() { continue; } if inst.gpu.is_none() && inst.rgba_data.is_empty() { continue; }
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
if let (Some(hdr_layer_view), Some(hdr_view)) = ( if let (Some(hdr_layer_view), Some(hdr_view)) = (
buffer_pool.get_view(hdr_layer_handle), buffer_pool.get_view(hdr_layer_handle),
&instance_resources.hdr_texture_view, &instance_resources.hdr_texture_view,
) { ) {
// Reuse the GPU texture for this frame if it's unchanged (a
// static/paused video → no CPU conversion, alloc, or upload).
// Timed into `blit_ms` (incl the cache lookup + per-frame view).
let _t = std::time::Instant::now(); let _t = std::time::Instant::now();
let tex_view = shared
.video_frame_cache
.lock()
.unwrap()
.texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
let bt = crate::gpu_brush::BlitTransform::new( let bt = crate::gpu_brush::BlitTransform::new(
inst.transform, inst.width, inst.height, width, height, inst.transform, inst.width, inst.height, width, height,
); );
shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); if let Some(gpu) = &inst.gpu {
// Hardware-decoded NV12 plane textures → linear RGB, no CPU upload.
let y_view = gpu.y.create_view(&Default::default());
let uv_view = gpu.uv.create_view(&Default::default());
shared.nv12_blit.blit(
device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range,
);
} else {
// Reuse the GPU texture for this frame if it's unchanged (a
// static/paused video → no CPU conversion, alloc, or upload).
let tex_view = shared
.video_frame_cache
.lock()
.unwrap()
.texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
}
cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0; cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0;
let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new( let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(