From ff490ab9ae1c2c3f47f1f6e4d3e80193978d5ec6 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Fri, 26 Jun 2026 02:47:05 -0400 Subject: [PATCH] =?UTF-8?q?nv12:=20HDR-correct=20input=20=E2=80=94=20PQ/HL?= =?UTF-8?q?G=20EOTF=20+=20BT.2020=E2=86=92709=20gamut=20(Stage=20A=20pt=20?= =?UTF-8?q?1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode HDR video into the linear compositor correctly instead of approximating everything as sRGB/BT.709: - Read the frame's color_trc and color_primaries in the importer → VideoTransfer {Gamma,Pq,Hlg} + VideoPrimaries {Bt709,Bt2020} on GpuVideoFrame. - nv12_blit.wgsl: branch the EOTF — sRGB gamma (SDR), SMPTE2084 PQ (normalized so 203-nit graphics white = 1.0; highlights exceed 1.0), or HLG inverse-OETF (reference white ≈ 1.0). Then BT.2020→BT.709 primaries in linear light when wide-gamut, clamping out-of-709 colours. Establishes the white=1.0 scene-linear convention: SDR content is unchanged (stays in [0,1]); HDR video carries super-white highlights through compositing. SDR-output mapping (clip default vs highlight rolloff) is Part 2. HLG's display OOTF is omitted (scene-referred) — approximate but reasonable for SDR-out. --- .../lightningbeam-core/src/video.rs | 25 ++++++++ .../lightningbeam-editor/src/hw_video.rs | 25 +++++++- .../lightningbeam-editor/src/nv12_blit.rs | 21 +++++-- .../src/panes/shaders/nv12_blit.wgsl | 59 +++++++++++++++++-- .../lightningbeam-editor/src/panes/stage.rs | 3 +- 5 files changed, 121 insertions(+), 12 deletions(-) diff --git a/lightningbeam-ui/lightningbeam-core/src/video.rs b/lightningbeam-ui/lightningbeam-core/src/video.rs index 83cc257..31c31ce 100644 --- a/lightningbeam-ui/lightningbeam-core/src/video.rs +++ b/lightningbeam-ui/lightningbeam-core/src/video.rs @@ -719,6 +719,31 @@ pub struct GpuVideoFrame { /// so SD (BT.601) and HD/UHD clips each convert correctly: `[Cr→R, Cb→G, Cr→G, Cb→B]`. /// R = Y + c[0]·Cr, G = Y + c[1]·Cb + c[2]·Cr, B = Y + c[3]·Cb pub coeffs: [f32; 4], + /// Opto-electronic transfer of the encoded R'G'B' — the compositor applies the matching EOTF to + /// reach scene-linear (graphics white = 1.0). HDR (PQ/HLG) values exceed 1.0. + pub transfer: VideoTransfer, + /// Colour primaries; BT.2020 is gamut-mapped to the compositor's BT.709 space in linear light. + pub primaries: VideoPrimaries, +} + +/// Transfer characteristic of a decoded video frame (selects the EOTF in the NV12→linear pass). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VideoTransfer { + /// SDR gamma (BT.709/sRGB/601/gamma22) — approximated by the sRGB EOTF. + Gamma, + /// SMPTE ST 2084 (PQ) — absolute, normalized so 203 nits (graphics white) = 1.0. + Pq, + /// ARIB STD-B67 (HLG) — scene-referred, normalized so reference white ≈ 1.0. + Hlg, +} + +/// Colour primaries of a decoded video frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VideoPrimaries { + /// BT.709 / sRGB (also used for BT.601, whose primaries differ only slightly). + Bt709, + /// BT.2020 (wide gamut) — converted to BT.709 in linear light by the compositor. + Bt2020, } /// Y'CbCr→R'G'B' matrix coefficients (`[Cr→R, Cb→G, Cr→G, Cb→B]`) from the luma weights `kr`/`kb` diff --git a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs index b32d963..d3f915d 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/hw_video.rs @@ -7,7 +7,8 @@ use ffmpeg_next::ffi as ff; use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf}; use lightningbeam_core::video::{ - ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, + ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, VideoPrimaries, + VideoTransfer, }; use std::sync::{Arc, Mutex}; @@ -77,6 +78,26 @@ impl HwVideoImporter for SharedHwImporter { }; let coeffs = ycbcr_coeffs(kr, kb); + // Transfer characteristic → which EOTF the compositor applies to reach scene-linear. + let transfer = match (*frame).color_trc { + ff::AVColorTransferCharacteristic::AVCOL_TRC_SMPTE2084 => VideoTransfer::Pq, + ff::AVColorTransferCharacteristic::AVCOL_TRC_ARIB_STD_B67 => VideoTransfer::Hlg, + _ => VideoTransfer::Gamma, + }; + // Primaries → BT.2020 is gamut-mapped to BT.709; unspecified follows the matrix guess above. + let primaries = match (*frame).color_primaries { + ff::AVColorPrimaries::AVCOL_PRI_BT2020 => VideoPrimaries::Bt2020, + ff::AVColorPrimaries::AVCOL_PRI_UNSPECIFIED + if matches!( + (*frame).colorspace, + ff::AVColorSpace::AVCOL_SPC_BT2020_NCL | ff::AVColorSpace::AVCOL_SPC_BT2020_CL + ) => + { + VideoPrimaries::Bt2020 + } + _ => VideoPrimaries::Bt709, + }; + let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf); ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan let (y, uv) = imported.ok()?.into_planes(); @@ -87,6 +108,8 @@ impl HwVideoImporter for SharedHwImporter { height, full_range, coeffs, + transfer, + primaries, }) } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs index 17d8900..c7949f7 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/nv12_blit.rs @@ -4,16 +4,17 @@ //! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`. use crate::gpu_brush::BlitTransform; +use lightningbeam_core::video::{VideoPrimaries, VideoTransfer}; /// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]), the Y'CbCr→RGB -/// matrix coefficients, and the full-range flag. 80 bytes (48 matrix + 16 coeffs + u32 + 12 pad). +/// matrix coefficients, and a flags vec4. 80 bytes (48 matrix + 16 coeffs + 16 flags). +/// `flags`: `[full_range, transfer (0 gamma / 1 PQ / 2 HLG), primaries (0 BT.709 / 1 BT.2020), pad]`. #[repr(C)] #[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] struct Nv12Params { transform: BlitTransform, coeffs: [f32; 4], - full_range: u32, - _pad: [u32; 3], + flags: [u32; 4], } pub struct Nv12BlitPipeline { @@ -129,12 +130,22 @@ impl Nv12BlitPipeline { transform: &BlitTransform, full_range: bool, coeffs: [f32; 4], + transfer: VideoTransfer, + primaries: VideoPrimaries, ) { + let transfer_code = match transfer { + VideoTransfer::Gamma => 0, + VideoTransfer::Pq => 1, + VideoTransfer::Hlg => 2, + }; + let primaries_code = match primaries { + VideoPrimaries::Bt709 => 0, + VideoPrimaries::Bt2020 => 1, + }; let params = Nv12Params { transform: *transform, coeffs, - full_range: full_range as u32, - _pad: [0; 3], + flags: [full_range as u32, transfer_code, primaries_code, 0], }; let param_buf = device.create_buffer(&wgpu::BufferDescriptor { label: Some("nv12_blit_params"), diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl index 33dc8d4..cf906ea 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/nv12_blit.wgsl @@ -16,8 +16,8 @@ struct Nv12Params { col2: vec4, // Y'CbCr→R'G'B' matrix from the source colorspace: [Cr→R, Cb→G, Cr→G, Cb→B]. coeffs: vec4, - // .x = full_range flag; .yzw padding. A vec4 keeps each block 16-aligned and the struct size - // matching the Rust `[f32;4] + u32 + [u32;3]` (80 bytes). + // .x = full_range; .y = transfer (0 gamma, 1 PQ, 2 HLG); .z = primaries (0 BT.709, 1 BT.2020). + // A vec4 keeps each block 16-aligned and the struct 80 bytes (Rust `[f32;4] + u32 + [u32;3]`). flags: vec4, } @@ -47,6 +47,41 @@ fn srgb_to_linear(c: vec3) -> vec3 { return select(lo, hi, c > vec3(0.04045)); } +// SMPTE ST 2084 (PQ) EOTF: encoded [0,1] → absolute luminance, then normalize so the 203-nit +// graphics white = 1.0 (HDR highlights exceed 1.0). Per-channel. +fn pq_to_linear(c: vec3) -> vec3 { + let m1 = 0.1593017578125; + let m2 = 78.84375; + let c1 = 0.8359375; + let c2 = 18.8515625; + let c3 = 18.6875; + let e = pow(max(c, vec3(0.0)), vec3(1.0 / m2)); + let num = max(e - vec3(c1), vec3(0.0)); + let den = vec3(c2) - c3 * e; + let nits = pow(num / den, vec3(1.0 / m1)) * 10000.0; // 0..10000 cd/m² + return nits / 203.0; +} + +// ARIB STD-B67 (HLG) inverse-OETF → scene light, normalized so reference white (signal 0.75) = 1.0. +// The display OOTF is omitted (scene-referred compositing); approximate but reasonable for SDR-out. +fn hlg_to_linear(c: vec3) -> vec3 { + let a = 0.17883277; + let b = 0.28466892; + let cc = 0.55991073; + let lo = (c * c) / 3.0; + let hi = (exp((c - vec3(cc)) / a) + vec3(b)) / 12.0; + let scene = select(lo, hi, c > vec3(0.5)); + return scene / 0.26496256; // hlg_inv_oetf(0.75): put reference white at 1.0 +} + +// BT.2020 → BT.709 primaries, linear light (ITU-R BT.2087). Out-of-709 colours go negative → clamp. +fn bt2020_to_bt709(c: vec3) -> vec3 { + let r = 1.660491 * c.r - 0.587641 * c.g - 0.072850 * c.b; + let g = -0.124550 * c.r + 1.132900 * c.g - 0.008349 * c.b; + let b = -0.018151 * c.r - 0.100579 * c.g + 1.118730 * c.b; + return max(vec3(r, g, b), vec3(0.0)); +} + @fragment fn fs_main(in: VertexOutput) -> @location(0) vec4 { let m = mat3x3(params.col0.xyz, params.col1.xyz, params.col2.xyz); @@ -79,9 +114,23 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { let r = Y + params.coeffs.x * Cr; let g = Y + params.coeffs.y * Cb + params.coeffs.z * Cr; let b = Y + params.coeffs.w * Cb; - let rgb_gamma = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); + // Valid encoded signal is [0,1]; clamp before the EOTF (HDR comes from the EOTF, not overshoot). + let rgb_enc = clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); + + // Encoded R'G'B' → scene-linear (graphics white = 1.0; HDR may exceed 1.0). + var rgb_lin: vec3; + if params.flags.y == 1u { + rgb_lin = pq_to_linear(rgb_enc); + } else if params.flags.y == 2u { + rgb_lin = hlg_to_linear(rgb_enc); + } else { + rgb_lin = srgb_to_linear(rgb_enc); + } + + // Wide-gamut → BT.709 in linear light to match the compositor's primaries. + if params.flags.z == 1u { + rgb_lin = bt2020_to_bt709(rgb_lin); + } - // R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer. - let rgb_lin = srgb_to_linear(rgb_gamma); return vec4(rgb_lin, 1.0); } diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index 4a52a03..7582049 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -1703,7 +1703,8 @@ impl egui_wgpu::CallbackTrait for VelloCallback { let y_view = gpu.y.create_view(&Default::default()); let uv_view = gpu.uv.create_view(&Default::default()); shared.nv12_blit.blit( - device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, gpu.coeffs, + device, queue, &y_view, &uv_view, hdr_layer_view, &bt, + gpu.full_range, gpu.coeffs, gpu.transfer, gpu.primaries, ); } else { // Reuse the GPU texture for this frame if it's unchanged (a