From 6348e57de0af71089ba8604998fd24392f8cc691 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:32:05 -0400 Subject: [PATCH] nv12: convert with the source colorspace matrix (not hardcoded BT.709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NV12→RGB pass hardcoded BT.709, so SD (BT.601) clips had slightly wrong hues. Read each frame's AVColorSpace in the importer and derive the Y'CbCr→R'G'B' matrix (BT.709/601/240M/2020; Unspecified guessed by height like swscale/players), carry the four coefficients on GpuVideoFrame, and apply them in the shader. - core: GpuVideoFrame.coeffs + ycbcr_coeffs(kr, kb) helper. - hw_video.rs: map AVColorSpace → (kr, kb) → coeffs. - nv12_blit{.rs,.wgsl}: uniform grows to 80 bytes (adds a coeffs vec4); the matrix multiply uses params.coeffs instead of literals. BT.2020's transfer is still approximated as sRGB. The DRM-modifier-without-SAMPLED case stays a graceful software fallback. Co-Authored-By: Claude Opus 4.8 --- .../lightningbeam-core/src/video.rs | 16 +++++++++++ .../lightningbeam-editor/src/hw_video.rs | 28 ++++++++++++++++++- .../lightningbeam-editor/src/nv12_blit.rs | 7 +++-- .../src/panes/shaders/nv12_blit.wgsl | 14 ++++++---- .../lightningbeam-editor/src/panes/stage.rs | 2 +- 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 958f197..83cc257 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -715,6 +715,22 @@ pub struct GpuVideoFrame { /// Source YUV range: true = full/PC (0–255), false = limited/TV (16–235). Drives the NV12→RGB /// offset/scale in the compositor. pub full_range: bool, + /// Y'CbCr→R'G'B' matrix coefficients derived from the frame's colorspace (BT.709/601/2020), + /// so SD (BT.601) and HD/UHD clips each convert correctly: `[Cr→R, Cb→G, Cr→G, Cb→B]`. + /// R = Y + c[0]·Cr, G = Y + c[1]·Cb + c[2]·Cr, B = Y + c[3]·Cb + pub coeffs: [f32; 4], +} + +/// Y'CbCr→R'G'B' matrix coefficients (`[Cr→R, Cb→G, Cr→G, Cb→B]`) from the luma weights `kr`/`kb` +/// (`kg = 1−kr−kb`). BT.709 → `[1.5748, −0.1873, −0.4681, 1.8556]`. +pub fn ycbcr_coeffs(kr: f32, kb: f32) -> [f32; 4] { + let kg = 1.0 - kr - kb; + [ + 2.0 * (1.0 - kr), + -2.0 * kb * (1.0 - kb) / kg, + -2.0 * kr * (1.0 - kr) / kg, + 2.0 * (1.0 - kb), + ] } /// Imports a decoded VAAPI surface (a `*mut AVFrame`, passed as an opaque pointer so core needn't diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs index 7032ae8..b32d963 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -6,7 +6,9 @@ use ffmpeg_next::ffi as ff; use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf}; -use lightningbeam_core::video::{GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager}; +use lightningbeam_core::video::{ + ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, +}; use std::sync::{Arc, Mutex}; /// Imports decoded VAAPI surfaces onto the shared wgpu device. Holds clones of the shared @@ -52,6 +54,29 @@ impl HwVideoImporter for SharedHwImporter { }; let full_range = (*frame).color_range == ff::AVColorRange::AVCOL_RANGE_JPEG; + // Luma weights (kr, kb) from the frame's matrix coefficients, so SD (BT.601) and HD/UHD + // (BT.709) clips each convert with the right matrix. Unspecified → guess by height, as + // players/swscale do. SMPTE240M and BT.2020 are handled too (the latter's transfer is still + // approximated as sRGB — fine for SDR; true HDR is out of scope). + let (kr, kb) = match (*frame).colorspace { + ff::AVColorSpace::AVCOL_SPC_BT709 => (0.2126, 0.0722), + ff::AVColorSpace::AVCOL_SPC_BT470BG | ff::AVColorSpace::AVCOL_SPC_SMPTE170M => { + (0.299, 0.114) + } + ff::AVColorSpace::AVCOL_SPC_SMPTE240M => (0.212, 0.087), + ff::AVColorSpace::AVCOL_SPC_BT2020_NCL | ff::AVColorSpace::AVCOL_SPC_BT2020_CL => { + (0.2627, 0.0593) + } + _ => { + if height <= 576 { + (0.299, 0.114) // SD → BT.601 + } else { + (0.2126, 0.0722) // HD/UHD → BT.709 + } + } + }; + let coeffs = ycbcr_coeffs(kr, kb); + let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan let (y, uv) = imported.ok()?.into_planes(); @@ -61,6 +86,7 @@ impl HwVideoImporter for SharedHwImporter { width, height, full_range, + coeffs, }) } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs index 3f26640..17d8900 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs @@ -5,12 +5,13 @@ use crate::gpu_brush::BlitTransform; -/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]) plus the -/// full-range flag. 64 bytes (48 for the matrix + a u32 + 12 padding). +/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]), the Y'CbCr→RGB +/// matrix coefficients, and the full-range flag. 80 bytes (48 matrix + 16 coeffs + u32 + 12 pad). #[repr(C)] #[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] struct Nv12Params { transform: BlitTransform, + coeffs: [f32; 4], full_range: u32, _pad: [u32; 3], } @@ -127,9 +128,11 @@ impl Nv12BlitPipeline { target_view: &wgpu::TextureView, transform: &BlitTransform, full_range: bool, + coeffs: [f32; 4], ) { let params = Nv12Params { transform: *transform, + coeffs, full_range: full_range as u32, _pad: [0; 3], }; diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl index 5a7e3f7..33dc8d4 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -14,8 +14,10 @@ struct Nv12Params { col0: vec4, col1: vec4, col2: vec4, - // .x = full_range flag; .yzw padding. A vec4 keeps the struct 64 bytes (matching the Rust - // `u32 + [u32; 3]`); a bare u32 + vec3 would round up to 80 and mismatch the bound buffer. + // Y'CbCr→R'G'B' matrix from the source colorspace: [Cr→R, Cb→G, Cr→G, Cb→B]. + coeffs: vec4, + // .x = full_range flag; .yzw padding. A vec4 keeps each block 16-aligned and the struct size + // matching the Rust `[f32;4] + u32 + [u32;3]` (80 bytes). flags: vec4, } @@ -73,10 +75,10 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { Cr = (cbcr.g * 255.0 - 128.0) / 224.0; } - // BT.709 Y'CbCr → gamma-encoded R'G'B'. - let r = Y + 1.5748 * Cr; - let g = Y - 0.1873 * Cb - 0.4681 * Cr; - let b = Y + 1.8556 * Cb; + // Y'CbCr → gamma-encoded R'G'B' using the source colorspace's matrix. + let r = Y + params.coeffs.x * Cr; + let g = Y + params.coeffs.y * Cb + params.coeffs.z * Cr; + let b = Y + params.coeffs.w * Cb; let rgb_gamma = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); // R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer. diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index de42f0c..4a52a03 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -1703,7 +1703,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback { let y_view = gpu.y.create_view(&Default::default()); let uv_view = gpu.uv.create_view(&Default::default()); shared.nv12_blit.blit( - device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, gpu.coeffs, ); } else { // Reuse the GPU texture for this frame if it's unchanged (a