editor: wire hardware video decode + NV12 preview compositing (Stage 3c-preview)
Make the dormant core HW-decode engine live for the preview path: - hw_video.rs: editor's HwVideoImporter — maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane textures on the *shared* device (the only one with the import extensions). install() creates the VAAPI device and injects it + the importer into the VideoManager. - main.rs: track whether the shared device is actually in use; only then (Linux, not LB_NO_SHARED_DEVICE) install hardware decode, using the CreationContext's shared device + adapter. - nv12_blit.rs + nv12_blit.wgsl: NV12 plane textures → BT.709 → sRGB-encoded → linear, written straight into the Rgba16Float HDR layer (no CPU upload). Colour math mirrors the software path so HW/SW video match; honours full_range. - stage.rs: the preview Video arm branches on inst.gpu (NV12 blit) vs rgba_data (existing upload+blit_straight); sets render_hardware_ok = !cpu_renderer so the CPU fallback still gets software frames. - video_exporter.rs: sets render_hardware_ok(false) before both compositing passes — export composites on the encoder's separate device, so a hardware decoder downloads to CPU instead (export stays software, correct). - dmabuf.rs: imported plane textures now also carry SAMPLED/TEXTURE_BINDING so they can be sampled by the NV12 blit (they were render-target-only); into_planes hands the textures to the longer-lived GpuVideoFrame. - video.rs: cache-key the GPU/CPU representation on want_gpu (HW-configured AND render_hardware_ok) so software-only decode keeps a single cache entry. Preview only this pass; export GPU-residency is the 3c-export follow-up. Untested at runtime here (no GPU/display in container) — both crates compile.
This commit is contained in:
Skyler Lehmkuhl
parent
863edc80fc
commit
1848c920d9
|
|
@ -47,6 +47,10 @@ impl ImportedNv12 {
|
|||
pub fn uv(&self) -> &wgpu::Texture {
|
||||
&self.uv
|
||||
}
|
||||
/// Consume into the `(Y, UV)` plane textures (for handing to a longer-lived owner).
|
||||
pub fn into_planes(self) -> (wgpu::Texture, wgpu::Texture) {
|
||||
(self.y, self.uv)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience: map a freshly-allocated `MappedSurface` and import it onto `drm`.
|
||||
|
|
@ -113,6 +117,7 @@ pub fn import_raw(
|
|||
.tiling(vk::ImageTiling::DRM_FORMAT_MODIFIER_EXT)
|
||||
.usage(
|
||||
vk::ImageUsageFlags::COLOR_ATTACHMENT
|
||||
| vk::ImageUsageFlags::SAMPLED
|
||||
| vk::ImageUsageFlags::TRANSFER_SRC
|
||||
| vk::ImageUsageFlags::TRANSFER_DST,
|
||||
)
|
||||
|
|
@ -178,7 +183,9 @@ pub fn import_raw(
|
|||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format,
|
||||
usage: wgpu_types::TextureUses::COLOR_TARGET | wgpu_types::TextureUses::COPY_SRC,
|
||||
usage: wgpu_types::TextureUses::COLOR_TARGET
|
||||
| wgpu_types::TextureUses::RESOURCE
|
||||
| wgpu_types::TextureUses::COPY_SRC,
|
||||
memory_flags: wgpu_hal::MemoryFlags::empty(),
|
||||
view_formats: vec![],
|
||||
};
|
||||
|
|
@ -192,7 +199,9 @@ pub fn import_raw(
|
|||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format,
|
||||
usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC,
|
||||
usage: wgpu::TextureUsages::RENDER_ATTACHMENT
|
||||
| wgpu::TextureUsages::TEXTURE_BINDING
|
||||
| wgpu::TextureUsages::COPY_SRC,
|
||||
view_formats: &[],
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -875,11 +875,13 @@ impl VideoManager {
|
|||
/// [`set_render_hardware_ok`](Self::set_render_hardware_ok), set per render pass (true for the
|
||||
/// preview, false for export, which composites on a different device).
|
||||
pub fn get_frame(&mut self, clip_id: &Uuid, timestamp: f64, target_w: u32, target_h: u32) -> Option<Arc<VideoFrame>> {
|
||||
let hardware_ok = self.render_hardware_ok;
|
||||
// The cache key includes (target size, hardware_ok): preview (GPU, preview res) and export
|
||||
// Whether this pass wants (and can produce) a GPU frame. Gated on HW being configured at all
|
||||
// so that with software-only decode preview and export share one cache entry (no double-cache).
|
||||
let want_gpu = self.render_hardware_ok && self.hw_device.is_some();
|
||||
// The cache key includes (target size, want_gpu): preview (GPU, preview res) and export
|
||||
// (CPU, export res) request the same clip/time and must not collide or cross representation.
|
||||
let timestamp_ms = (timestamp * 1000.0) as i64;
|
||||
let cache_key = (*clip_id, timestamp_ms, target_w, target_h, hardware_ok);
|
||||
let cache_key = (*clip_id, timestamp_ms, target_w, target_h, want_gpu);
|
||||
|
||||
// Check frame cache first
|
||||
if let Some(cached_frame) = self.frame_cache.get(&cache_key) {
|
||||
|
|
@ -892,7 +894,7 @@ impl VideoManager {
|
|||
let mut decoder = decoder_arc.lock().ok()?;
|
||||
|
||||
// Decode the frame at the requested target (capped to native by the decoder).
|
||||
let decoded = decoder.get_frame(timestamp, target_w, target_h, hardware_ok).ok()?;
|
||||
let decoded = decoder.get_frame(timestamp, target_w, target_h, want_gpu).ok()?;
|
||||
drop(decoder); // release the lock before touching `self`
|
||||
|
||||
// Create VideoFrame and cache it.
|
||||
|
|
|
|||
|
|
@ -1066,6 +1066,13 @@ pub fn render_frame_to_rgba_hdr(
|
|||
Affine::IDENTITY
|
||||
};
|
||||
|
||||
// Export composites on the encoder's own device, not the shared one, so it cannot use
|
||||
// hardware-decoded GPU frames (textures can't cross devices). Force software frames; a
|
||||
// hardware decoder downloads its surface to CPU instead.
|
||||
if let Ok(mut vm) = video_manager.lock() {
|
||||
vm.set_render_hardware_ok(false);
|
||||
}
|
||||
|
||||
// Render document for compositing (returns per-layer scenes)
|
||||
let composite_result = render_document_for_compositing(
|
||||
document,
|
||||
|
|
@ -1329,6 +1336,11 @@ pub fn render_frame_to_gpu_rgba(
|
|||
Affine::IDENTITY
|
||||
};
|
||||
|
||||
// Export composites on a separate device — force software frames (see above).
|
||||
if let Ok(mut vm) = video_manager.lock() {
|
||||
vm.set_render_hardware_ok(false);
|
||||
}
|
||||
|
||||
// Render document for compositing (returns per-layer scenes)
|
||||
let composite_result = render_document_for_compositing(
|
||||
document,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,89 @@
|
|||
//! Hardware video decode glue (Linux/VAAPI). The editor implements core's [`HwVideoImporter`]:
|
||||
//! it maps a decoded VAAPI surface to a DRM-PRIME DMA-BUF and imports it as wgpu NV12 plane
|
||||
//! textures on the **shared** device (the one eframe + the compositor run on, which has the
|
||||
//! DMA-BUF-import extensions). [`install`] creates the VAAPI device and wires it into the
|
||||
//! `VideoManager`.
|
||||
|
||||
use ffmpeg_next::ffi as ff;
|
||||
use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf};
|
||||
use lightningbeam_core::video::{GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// Imports decoded VAAPI surfaces onto the shared wgpu device. Holds clones of the shared
|
||||
/// device + adapter (Arc-backed, cheap).
|
||||
struct SharedHwImporter {
|
||||
device: wgpu::Device,
|
||||
adapter: wgpu::Adapter,
|
||||
}
|
||||
|
||||
impl HwVideoImporter for SharedHwImporter {
|
||||
unsafe fn import(&self, av_frame: *mut std::ffi::c_void) -> Option<GpuVideoFrame> {
|
||||
let frame = av_frame as *mut ff::AVFrame;
|
||||
|
||||
// Map the VAAPI surface to a DRM-PRIME DMA-BUF (read-only).
|
||||
let drm_f = ff::av_frame_alloc();
|
||||
(*drm_f).format = ff::AVPixelFormat::AV_PIX_FMT_DRM_PRIME as i32;
|
||||
let flags = ff::AV_HWFRAME_MAP_DIRECT as i32 | ff::AV_HWFRAME_MAP_READ as i32;
|
||||
if ff::av_hwframe_map(drm_f, frame, flags) < 0 {
|
||||
ff::av_frame_free(&mut (drm_f as *mut _));
|
||||
return None;
|
||||
}
|
||||
|
||||
let desc = (*drm_f).data[0] as *const ff::AVDRMFrameDescriptor;
|
||||
let obj = &(*desc).objects[0];
|
||||
let width = (*frame).width as u32;
|
||||
let height = (*frame).height as u32;
|
||||
// NV12: Y then UV — two layers (one plane each) or one layer with two planes.
|
||||
let (y_pl, uv_pl) = if (*desc).nb_layers >= 2 {
|
||||
(&(*desc).layers[0].planes[0], &(*desc).layers[1].planes[0])
|
||||
} else {
|
||||
(&(*desc).layers[0].planes[0], &(*desc).layers[0].planes[1])
|
||||
};
|
||||
let buf = Nv12DmaBuf {
|
||||
fd: obj.fd,
|
||||
size: obj.size as u64,
|
||||
modifier: obj.format_modifier,
|
||||
width,
|
||||
height,
|
||||
y_offset: y_pl.offset as u64,
|
||||
y_pitch: y_pl.pitch as u64,
|
||||
uv_offset: uv_pl.offset as u64,
|
||||
uv_pitch: uv_pl.pitch as u64,
|
||||
};
|
||||
let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG;
|
||||
|
||||
let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf);
|
||||
ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan
|
||||
let (y, uv) = imported.ok()?.into_planes();
|
||||
Some(GpuVideoFrame {
|
||||
y: Arc::new(y),
|
||||
uv: Arc::new(uv),
|
||||
width,
|
||||
height,
|
||||
full_range,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Create the VAAPI hardware device and install hardware decode into `vm`, importing onto the
|
||||
/// shared `device`/`adapter`. Logs and no-ops if VAAPI is unavailable (→ software decode).
|
||||
pub fn install(vm: &Arc<Mutex<VideoManager>>, device: &wgpu::Device, adapter: &wgpu::Adapter) {
|
||||
match gpu_video_encoder::vaapi::create_device() {
|
||||
Ok(hw_device) => {
|
||||
let importer = Arc::new(SharedHwImporter {
|
||||
device: device.clone(),
|
||||
adapter: adapter.clone(),
|
||||
});
|
||||
if let Ok(mut vm) = vm.lock() {
|
||||
vm.set_hardware_decode(
|
||||
HwDeviceHandle(hw_device as *mut std::ffi::c_void),
|
||||
importer,
|
||||
);
|
||||
}
|
||||
println!("🎞 Hardware video decode enabled (VAAPI → shared device)");
|
||||
}
|
||||
Err(e) => {
|
||||
println!("🎞 Hardware video decode unavailable ({e}); using software decode");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -51,6 +51,9 @@ mod custom_cursor;
|
|||
mod tablet;
|
||||
mod debug_overlay;
|
||||
mod gpu_timer;
|
||||
#[cfg(target_os = "linux")]
|
||||
mod hw_video;
|
||||
mod nv12_blit;
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
mod test_mode;
|
||||
|
|
@ -210,6 +213,10 @@ fn main() -> eframe::Result {
|
|||
// raw VkDevice persists with them.
|
||||
#[allow(unused_mut)]
|
||||
let mut wgpu_setup = lb_default_wgpu_setup();
|
||||
// Whether the shared VAAPI-capable device is in use — only then can decoded DMA-BUF frames be
|
||||
// imported + composited, so hardware decode is injected into the VideoManager only if true.
|
||||
#[allow(unused_assignments, unused_mut)]
|
||||
let mut shared_device_active = false;
|
||||
#[cfg(target_os = "linux")]
|
||||
if std::env::var("LB_NO_SHARED_DEVICE").is_ok() {
|
||||
println!("🖥 Shared device disabled via LB_NO_SHARED_DEVICE; default wgpu device (software video decode)");
|
||||
|
|
@ -223,6 +230,7 @@ fn main() -> eframe::Result {
|
|||
device: drm.device.clone(),
|
||||
queue: drm.queue.clone(),
|
||||
});
|
||||
shared_device_active = true;
|
||||
}
|
||||
Err(e) => {
|
||||
println!("🖥 Shared device unavailable ({e}); default wgpu device (software video decode)");
|
||||
|
|
@ -294,6 +302,13 @@ fn main() -> eframe::Result {
|
|||
let app = EditorApp::new(cc, layouts, theme, test_mode_panic_snapshot_for_app, test_mode_pending_event_for_app, test_mode_is_replaying_for_app, test_mode_pending_geometry_for_app);
|
||||
#[cfg(not(debug_assertions))]
|
||||
let app = EditorApp::new(cc, layouts, theme);
|
||||
// Wire hardware video decode into the VideoManager now that the shared device exists.
|
||||
#[cfg(target_os = "linux")]
|
||||
if shared_device_active {
|
||||
if let Some(rs) = cc.wgpu_render_state.as_ref() {
|
||||
hw_video::install(&app.video_manager, &rs.device, &rs.adapter);
|
||||
}
|
||||
}
|
||||
Ok(Box::new(app))
|
||||
}),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
//! NV12 → linear-RGB blit: composites a hardware-decoded video frame (two wgpu plane textures,
|
||||
//! Y = R8Unorm + CbCr = Rg8Unorm) directly into the Rgba16Float HDR layer, with no CPU upload.
|
||||
//! The colour math mirrors the software path (BT.709 → sRGB-encoded → linear) so hardware- and
|
||||
//! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`.
|
||||
|
||||
use crate::gpu_brush::BlitTransform;
|
||||
|
||||
/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]) plus the
|
||||
/// full-range flag. 64 bytes (48 for the matrix + a u32 + 12 padding).
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)]
|
||||
struct Nv12Params {
|
||||
transform: BlitTransform,
|
||||
full_range: u32,
|
||||
_pad: [u32; 3],
|
||||
}
|
||||
|
||||
pub struct Nv12BlitPipeline {
|
||||
pipeline: wgpu::RenderPipeline,
|
||||
bg_layout: wgpu::BindGroupLayout,
|
||||
sampler: wgpu::Sampler,
|
||||
}
|
||||
|
||||
impl Nv12BlitPipeline {
|
||||
pub fn new(device: &wgpu::Device) -> Self {
|
||||
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
|
||||
label: Some("nv12_blit_shader"),
|
||||
source: wgpu::ShaderSource::Wgsl(
|
||||
include_str!("panes/shaders/nv12_blit.wgsl").into(),
|
||||
),
|
||||
});
|
||||
|
||||
let tex_entry = |binding: u32| wgpu::BindGroupLayoutEntry {
|
||||
binding,
|
||||
visibility: wgpu::ShaderStages::FRAGMENT,
|
||||
ty: wgpu::BindingType::Texture {
|
||||
sample_type: wgpu::TextureSampleType::Float { filterable: true },
|
||||
view_dimension: wgpu::TextureViewDimension::D2,
|
||||
multisampled: false,
|
||||
},
|
||||
count: None,
|
||||
};
|
||||
|
||||
let bg_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
|
||||
label: Some("nv12_blit_bgl"),
|
||||
entries: &[
|
||||
tex_entry(0), // Y plane (R8Unorm)
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 1,
|
||||
visibility: wgpu::ShaderStages::FRAGMENT,
|
||||
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
|
||||
count: None,
|
||||
},
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 2,
|
||||
visibility: wgpu::ShaderStages::FRAGMENT,
|
||||
ty: wgpu::BindingType::Buffer {
|
||||
ty: wgpu::BufferBindingType::Uniform,
|
||||
has_dynamic_offset: false,
|
||||
min_binding_size: None,
|
||||
},
|
||||
count: None,
|
||||
},
|
||||
tex_entry(3), // CbCr plane (Rg8Unorm)
|
||||
],
|
||||
});
|
||||
|
||||
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("nv12_blit_pl"),
|
||||
bind_group_layouts: &[&bg_layout],
|
||||
push_constant_ranges: &[],
|
||||
});
|
||||
|
||||
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
|
||||
label: Some("nv12_blit_pipeline"),
|
||||
layout: Some(&pipeline_layout),
|
||||
vertex: wgpu::VertexState {
|
||||
module: &shader,
|
||||
entry_point: Some("vs_main"),
|
||||
buffers: &[],
|
||||
compilation_options: Default::default(),
|
||||
},
|
||||
fragment: Some(wgpu::FragmentState {
|
||||
module: &shader,
|
||||
entry_point: Some("fs_main"),
|
||||
targets: &[Some(wgpu::ColorTargetState {
|
||||
format: wgpu::TextureFormat::Rgba16Float,
|
||||
blend: None,
|
||||
write_mask: wgpu::ColorWrites::ALL,
|
||||
})],
|
||||
compilation_options: Default::default(),
|
||||
}),
|
||||
primitive: wgpu::PrimitiveState {
|
||||
topology: wgpu::PrimitiveTopology::TriangleStrip,
|
||||
..Default::default()
|
||||
},
|
||||
depth_stencil: None,
|
||||
multisample: wgpu::MultisampleState::default(),
|
||||
multiview: None,
|
||||
cache: None,
|
||||
});
|
||||
|
||||
// Bilinear: the frame is scaled to the output size; nearest would look blocky.
|
||||
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
|
||||
label: Some("nv12_blit_sampler"),
|
||||
address_mode_u: wgpu::AddressMode::ClampToEdge,
|
||||
address_mode_v: wgpu::AddressMode::ClampToEdge,
|
||||
address_mode_w: wgpu::AddressMode::ClampToEdge,
|
||||
mag_filter: wgpu::FilterMode::Linear,
|
||||
min_filter: wgpu::FilterMode::Linear,
|
||||
mipmap_filter: wgpu::FilterMode::Linear,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
Self { pipeline, bg_layout, sampler }
|
||||
}
|
||||
|
||||
/// Convert + blit the NV12 frame into `target_view` (Rgba16Float, cleared to transparent),
|
||||
/// positioned by `transform` (built like the RGBA video path's `BlitTransform`).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn blit(
|
||||
&self,
|
||||
device: &wgpu::Device,
|
||||
queue: &wgpu::Queue,
|
||||
y_view: &wgpu::TextureView,
|
||||
uv_view: &wgpu::TextureView,
|
||||
target_view: &wgpu::TextureView,
|
||||
transform: &BlitTransform,
|
||||
full_range: bool,
|
||||
) {
|
||||
let params = Nv12Params {
|
||||
transform: *transform,
|
||||
full_range: full_range as u32,
|
||||
_pad: [0; 3],
|
||||
};
|
||||
let param_buf = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("nv12_blit_params"),
|
||||
size: std::mem::size_of::<Nv12Params>() as u64,
|
||||
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
queue.write_buffer(¶m_buf, 0, bytemuck::bytes_of(¶ms));
|
||||
|
||||
let bg = device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("nv12_blit_bg"),
|
||||
layout: &self.bg_layout,
|
||||
entries: &[
|
||||
wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(y_view) },
|
||||
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&self.sampler) },
|
||||
wgpu::BindGroupEntry { binding: 2, resource: param_buf.as_entire_binding() },
|
||||
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(uv_view) },
|
||||
],
|
||||
});
|
||||
|
||||
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||
label: Some("nv12_blit_encoder"),
|
||||
});
|
||||
{
|
||||
let mut rp = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("nv12_blit_pass"),
|
||||
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
|
||||
view: target_view,
|
||||
resolve_target: None,
|
||||
depth_slice: None,
|
||||
ops: wgpu::Operations {
|
||||
load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
|
||||
store: wgpu::StoreOp::Store,
|
||||
},
|
||||
})],
|
||||
depth_stencil_attachment: None,
|
||||
occlusion_query_set: None,
|
||||
timestamp_writes: None,
|
||||
});
|
||||
rp.set_pipeline(&self.pipeline);
|
||||
rp.set_bind_group(0, &bg, &[]);
|
||||
rp.draw(0..4, 0..1);
|
||||
}
|
||||
queue.submit(Some(encoder.finish()));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
// NV12 → linear-RGB blit shader.
|
||||
//
|
||||
// Samples a hardware-decoded video frame stored as two planes — Y (R8Unorm) and
|
||||
// interleaved CbCr (Rg8Unorm, half-res) — converts BT.709 Y'CbCr → gamma-encoded
|
||||
// R'G'B', then sRGB→linear, and writes straight-alpha linear into the Rgba16Float
|
||||
// HDR layer. This mirrors the software path (swscale → sRGB RGBA8 → sampled as
|
||||
// Rgba8UnormSrgb → linear) so hardware- and software-decoded video match.
|
||||
//
|
||||
// The affine transform (viewport UV → frame UV) is the same packing as
|
||||
// canvas_blit.wgsl's BlitTransform; `full_range` selects full vs. studio-swing
|
||||
// de-quantization.
|
||||
|
||||
struct Nv12Params {
|
||||
col0: vec4<f32>,
|
||||
col1: vec4<f32>,
|
||||
col2: vec4<f32>,
|
||||
full_range: u32,
|
||||
_pad: vec3<u32>,
|
||||
}
|
||||
|
||||
@group(0) @binding(0) var y_tex: texture_2d<f32>;
|
||||
@group(0) @binding(1) var samp: sampler;
|
||||
@group(0) @binding(2) var<uniform> params: Nv12Params;
|
||||
@group(0) @binding(3) var uv_tex: texture_2d<f32>;
|
||||
|
||||
struct VertexOutput {
|
||||
@builtin(position) position: vec4<f32>,
|
||||
@location(0) uv: vec2<f32>,
|
||||
}
|
||||
|
||||
@vertex
|
||||
fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput {
|
||||
var out: VertexOutput;
|
||||
let x = f32((vertex_index & 1u) << 1u);
|
||||
let y = f32(vertex_index & 2u);
|
||||
out.position = vec4<f32>(x * 2.0 - 1.0, 1.0 - y * 2.0, 0.0, 1.0);
|
||||
out.uv = vec2<f32>(x, y);
|
||||
return out;
|
||||
}
|
||||
|
||||
fn srgb_to_linear(c: vec3<f32>) -> vec3<f32> {
|
||||
let lo = c / 12.92;
|
||||
let hi = pow((c + vec3<f32>(0.055)) / 1.055, vec3<f32>(2.4));
|
||||
return select(lo, hi, c > vec3<f32>(0.04045));
|
||||
}
|
||||
|
||||
@fragment
|
||||
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
|
||||
let m = mat3x3<f32>(params.col0.xyz, params.col1.xyz, params.col2.xyz);
|
||||
let frame_uv = (m * vec3<f32>(in.uv.x, in.uv.y, 1.0)).xy;
|
||||
|
||||
if frame_uv.x < 0.0 || frame_uv.x > 1.0
|
||||
|| frame_uv.y < 0.0 || frame_uv.y > 1.0 {
|
||||
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
|
||||
}
|
||||
|
||||
let yv = textureSample(y_tex, samp, frame_uv).r;
|
||||
let cbcr = textureSample(uv_tex, samp, frame_uv).rg;
|
||||
|
||||
var Y: f32;
|
||||
var Cb: f32;
|
||||
var Cr: f32;
|
||||
if params.full_range != 0u {
|
||||
// Full ("JPEG") range: [0,255] luma, chroma centered at 128.
|
||||
Y = yv;
|
||||
Cb = cbcr.r - 0.5;
|
||||
Cr = cbcr.g - 0.5;
|
||||
} else {
|
||||
// Studio swing: Y'∈[16,235], Cb/Cr∈[16,240].
|
||||
Y = (yv * 255.0 - 16.0) / 219.0;
|
||||
Cb = (cbcr.r * 255.0 - 128.0) / 224.0;
|
||||
Cr = (cbcr.g * 255.0 - 128.0) / 224.0;
|
||||
}
|
||||
|
||||
// BT.709 Y'CbCr → gamma-encoded R'G'B'.
|
||||
let r = Y + 1.5748 * Cr;
|
||||
let g = Y - 0.1873 * Cb - 0.4681 * Cr;
|
||||
let b = Y + 1.8556 * Cb;
|
||||
let rgb_gamma = clamp(vec3<f32>(r, g, b), vec3<f32>(0.0), vec3<f32>(1.0));
|
||||
|
||||
// R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer.
|
||||
let rgb_lin = srgb_to_linear(rgb_gamma);
|
||||
return vec4<f32>(rgb_lin, 1.0);
|
||||
}
|
||||
|
|
@ -152,6 +152,8 @@ struct SharedVelloResources {
|
|||
gpu_brush: Mutex<crate::gpu_brush::GpuBrushEngine>,
|
||||
/// Canvas blit pipeline (renders GPU canvas to layer sRGB buffer)
|
||||
canvas_blit: crate::gpu_brush::CanvasBlitPipeline,
|
||||
/// NV12→linear blit for hardware-decoded video frames (GPU plane textures → HDR layer).
|
||||
nv12_blit: crate::nv12_blit::Nv12BlitPipeline,
|
||||
/// True when Vello is running its CPU software renderer (either forced or GPU fallback).
|
||||
/// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area.
|
||||
is_cpu_renderer: bool,
|
||||
|
|
@ -372,6 +374,7 @@ impl SharedVelloResources {
|
|||
// Initialize GPU raster brush engine
|
||||
let gpu_brush = crate::gpu_brush::GpuBrushEngine::new(device);
|
||||
let canvas_blit = crate::gpu_brush::CanvasBlitPipeline::new(device);
|
||||
let nv12_blit = crate::nv12_blit::Nv12BlitPipeline::new(device);
|
||||
|
||||
println!("✅ Vello shared resources initialized (renderer, shaders, HDR compositor, effect processor, color converter, and GPU brush engine)");
|
||||
|
||||
|
|
@ -389,6 +392,7 @@ impl SharedVelloResources {
|
|||
srgb_to_linear,
|
||||
gpu_brush: Mutex::new(gpu_brush),
|
||||
canvas_blit,
|
||||
nv12_blit,
|
||||
is_cpu_renderer: use_cpu || is_cpu_renderer,
|
||||
gpu_timer: Mutex::new(None),
|
||||
video_frame_cache: Mutex::new(VideoFrameTexCache::new()),
|
||||
|
|
@ -1062,6 +1066,12 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
// Let the cache page image bytes from the project container on a decode miss.
|
||||
image_cache.set_container_path(self.ctx.container_path.clone());
|
||||
|
||||
// Preview composites on the shared device, so it can consume hardware-decoded GPU
|
||||
// frames — but only the GPU renderer; the CPU fallback needs software frames.
|
||||
if let Ok(mut vm) = shared.video_manager.lock() {
|
||||
vm.set_render_hardware_ok(!shared.is_cpu_renderer);
|
||||
}
|
||||
|
||||
let composite_result = if shared.is_cpu_renderer {
|
||||
lightningbeam_core::renderer::render_document_for_compositing_cpu(
|
||||
&self.ctx.document,
|
||||
|
|
@ -1678,25 +1688,33 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
RenderedLayerType::Video { instances } => {
|
||||
// Video layer — per-instance: (cached) frame texture → blit → composite.
|
||||
for inst in instances {
|
||||
if inst.rgba_data.is_empty() { continue; }
|
||||
if inst.gpu.is_none() && inst.rgba_data.is_empty() { continue; }
|
||||
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
|
||||
if let (Some(hdr_layer_view), Some(hdr_view)) = (
|
||||
buffer_pool.get_view(hdr_layer_handle),
|
||||
&instance_resources.hdr_texture_view,
|
||||
) {
|
||||
// Reuse the GPU texture for this frame if it's unchanged (a
|
||||
// static/paused video → no CPU conversion, alloc, or upload).
|
||||
// Timed into `blit_ms` (incl the cache lookup + per-frame view).
|
||||
let _t = std::time::Instant::now();
|
||||
let tex_view = shared
|
||||
.video_frame_cache
|
||||
.lock()
|
||||
.unwrap()
|
||||
.texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
|
||||
let bt = crate::gpu_brush::BlitTransform::new(
|
||||
inst.transform, inst.width, inst.height, width, height,
|
||||
);
|
||||
shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||
if let Some(gpu) = &inst.gpu {
|
||||
// Hardware-decoded NV12 plane textures → linear RGB, no CPU upload.
|
||||
let y_view = gpu.y.create_view(&Default::default());
|
||||
let uv_view = gpu.uv.create_view(&Default::default());
|
||||
shared.nv12_blit.blit(
|
||||
device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range,
|
||||
);
|
||||
} else {
|
||||
// Reuse the GPU texture for this frame if it's unchanged (a
|
||||
// static/paused video → no CPU conversion, alloc, or upload).
|
||||
let tex_view = shared
|
||||
.video_frame_cache
|
||||
.lock()
|
||||
.unwrap()
|
||||
.texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
|
||||
shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||
}
|
||||
cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(
|
||||
|
|
|
|||
Loading…
Reference in New Issue