nv12: HDR-correct input — PQ/HLG EOTF + BT.2020→709 gamut (Stage A pt 1)
Decode HDR video into the linear compositor correctly instead of approximating
everything as sRGB/BT.709:
- Read the frame's color_trc and color_primaries in the importer → VideoTransfer
{Gamma,Pq,Hlg} + VideoPrimaries {Bt709,Bt2020} on GpuVideoFrame.
- nv12_blit.wgsl: branch the EOTF — sRGB gamma (SDR), SMPTE2084 PQ (normalized so
203-nit graphics white = 1.0; highlights exceed 1.0), or HLG inverse-OETF
(reference white ≈ 1.0). Then BT.2020→BT.709 primaries in linear light when
wide-gamut, clamping out-of-709 colours.
Establishes the white=1.0 scene-linear convention: SDR content is unchanged
(stays in [0,1]); HDR video carries super-white highlights through compositing.
SDR-output mapping (clip default vs highlight rolloff) is Part 2. HLG's display
OOTF is omitted (scene-referred) — approximate but reasonable for SDR-out.
This commit is contained in:
Skyler Lehmkuhl
parent
6348e57de0
commit
ff490ab9ae
|
|
@ -719,6 +719,31 @@ pub struct GpuVideoFrame {
|
|||
/// so SD (BT.601) and HD/UHD clips each convert correctly: `[Cr→R, Cb→G, Cr→G, Cb→B]`.
|
||||
/// R = Y + c[0]·Cr, G = Y + c[1]·Cb + c[2]·Cr, B = Y + c[3]·Cb
|
||||
pub coeffs: [f32; 4],
|
||||
/// Opto-electronic transfer of the encoded R'G'B' — the compositor applies the matching EOTF to
|
||||
/// reach scene-linear (graphics white = 1.0). HDR (PQ/HLG) values exceed 1.0.
|
||||
pub transfer: VideoTransfer,
|
||||
/// Colour primaries; BT.2020 is gamut-mapped to the compositor's BT.709 space in linear light.
|
||||
pub primaries: VideoPrimaries,
|
||||
}
|
||||
|
||||
/// Transfer characteristic of a decoded video frame (selects the EOTF in the NV12→linear pass).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum VideoTransfer {
|
||||
/// SDR gamma (BT.709/sRGB/601/gamma22) — approximated by the sRGB EOTF.
|
||||
Gamma,
|
||||
/// SMPTE ST 2084 (PQ) — absolute, normalized so 203 nits (graphics white) = 1.0.
|
||||
Pq,
|
||||
/// ARIB STD-B67 (HLG) — scene-referred, normalized so reference white ≈ 1.0.
|
||||
Hlg,
|
||||
}
|
||||
|
||||
/// Colour primaries of a decoded video frame.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum VideoPrimaries {
|
||||
/// BT.709 / sRGB (also used for BT.601, whose primaries differ only slightly).
|
||||
Bt709,
|
||||
/// BT.2020 (wide gamut) — converted to BT.709 in linear light by the compositor.
|
||||
Bt2020,
|
||||
}
|
||||
|
||||
/// Y'CbCr→R'G'B' matrix coefficients (`[Cr→R, Cb→G, Cr→G, Cb→B]`) from the luma weights `kr`/`kb`
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
use ffmpeg_next::ffi as ff;
|
||||
use gpu_video_encoder::dmabuf::{self, Nv12DmaBuf};
|
||||
use lightningbeam_core::video::{
|
||||
ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager,
|
||||
ycbcr_coeffs, GpuVideoFrame, HwDeviceHandle, HwVideoImporter, VideoManager, VideoPrimaries,
|
||||
VideoTransfer,
|
||||
};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
|
|
@ -77,6 +78,26 @@ impl HwVideoImporter for SharedHwImporter {
|
|||
};
|
||||
let coeffs = ycbcr_coeffs(kr, kb);
|
||||
|
||||
// Transfer characteristic → which EOTF the compositor applies to reach scene-linear.
|
||||
let transfer = match (*frame).color_trc {
|
||||
ff::AVColorTransferCharacteristic::AVCOL_TRC_SMPTE2084 => VideoTransfer::Pq,
|
||||
ff::AVColorTransferCharacteristic::AVCOL_TRC_ARIB_STD_B67 => VideoTransfer::Hlg,
|
||||
_ => VideoTransfer::Gamma,
|
||||
};
|
||||
// Primaries → BT.2020 is gamut-mapped to BT.709; unspecified follows the matrix guess above.
|
||||
let primaries = match (*frame).color_primaries {
|
||||
ff::AVColorPrimaries::AVCOL_PRI_BT2020 => VideoPrimaries::Bt2020,
|
||||
ff::AVColorPrimaries::AVCOL_PRI_UNSPECIFIED
|
||||
if matches!(
|
||||
(*frame).colorspace,
|
||||
ff::AVColorSpace::AVCOL_SPC_BT2020_NCL | ff::AVColorSpace::AVCOL_SPC_BT2020_CL
|
||||
) =>
|
||||
{
|
||||
VideoPrimaries::Bt2020
|
||||
}
|
||||
_ => VideoPrimaries::Bt709,
|
||||
};
|
||||
|
||||
let imported = dmabuf::import_raw(&self.device, &self.adapter, &buf);
|
||||
ff::av_frame_free(&mut (drm_f as *mut _)); // the fd was dup'd into Vulkan
|
||||
let (y, uv) = imported.ok()?.into_planes();
|
||||
|
|
@ -87,6 +108,8 @@ impl HwVideoImporter for SharedHwImporter {
|
|||
height,
|
||||
full_range,
|
||||
coeffs,
|
||||
transfer,
|
||||
primaries,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,16 +4,17 @@
|
|||
//! software-decoded video look identical. See `panes/shaders/nv12_blit.wgsl`.
|
||||
|
||||
use crate::gpu_brush::BlitTransform;
|
||||
use lightningbeam_core::video::{VideoPrimaries, VideoTransfer};
|
||||
|
||||
/// Uniform: the `viewport_uv → frame_uv` affine (same packing as [`BlitTransform`]), the Y'CbCr→RGB
|
||||
/// matrix coefficients, and the full-range flag. 80 bytes (48 matrix + 16 coeffs + u32 + 12 pad).
|
||||
/// matrix coefficients, and a flags vec4. 80 bytes (48 matrix + 16 coeffs + 16 flags).
|
||||
/// `flags`: `[full_range, transfer (0 gamma / 1 PQ / 2 HLG), primaries (0 BT.709 / 1 BT.2020), pad]`.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)]
|
||||
struct Nv12Params {
|
||||
transform: BlitTransform,
|
||||
coeffs: [f32; 4],
|
||||
full_range: u32,
|
||||
_pad: [u32; 3],
|
||||
flags: [u32; 4],
|
||||
}
|
||||
|
||||
pub struct Nv12BlitPipeline {
|
||||
|
|
@ -129,12 +130,22 @@ impl Nv12BlitPipeline {
|
|||
transform: &BlitTransform,
|
||||
full_range: bool,
|
||||
coeffs: [f32; 4],
|
||||
transfer: VideoTransfer,
|
||||
primaries: VideoPrimaries,
|
||||
) {
|
||||
let transfer_code = match transfer {
|
||||
VideoTransfer::Gamma => 0,
|
||||
VideoTransfer::Pq => 1,
|
||||
VideoTransfer::Hlg => 2,
|
||||
};
|
||||
let primaries_code = match primaries {
|
||||
VideoPrimaries::Bt709 => 0,
|
||||
VideoPrimaries::Bt2020 => 1,
|
||||
};
|
||||
let params = Nv12Params {
|
||||
transform: *transform,
|
||||
coeffs,
|
||||
full_range: full_range as u32,
|
||||
_pad: [0; 3],
|
||||
flags: [full_range as u32, transfer_code, primaries_code, 0],
|
||||
};
|
||||
let param_buf = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("nv12_blit_params"),
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ struct Nv12Params {
|
|||
col2: vec4<f32>,
|
||||
// Y'CbCr→R'G'B' matrix from the source colorspace: [Cr→R, Cb→G, Cr→G, Cb→B].
|
||||
coeffs: vec4<f32>,
|
||||
// .x = full_range flag; .yzw padding. A vec4 keeps each block 16-aligned and the struct size
|
||||
// matching the Rust `[f32;4] + u32 + [u32;3]` (80 bytes).
|
||||
// .x = full_range; .y = transfer (0 gamma, 1 PQ, 2 HLG); .z = primaries (0 BT.709, 1 BT.2020).
|
||||
// A vec4 keeps each block 16-aligned and the struct 80 bytes (Rust `[f32;4] + u32 + [u32;3]`).
|
||||
flags: vec4<u32>,
|
||||
}
|
||||
|
||||
|
|
@ -47,6 +47,41 @@ fn srgb_to_linear(c: vec3<f32>) -> vec3<f32> {
|
|||
return select(lo, hi, c > vec3<f32>(0.04045));
|
||||
}
|
||||
|
||||
// SMPTE ST 2084 (PQ) EOTF: encoded [0,1] → absolute luminance, then normalize so the 203-nit
|
||||
// graphics white = 1.0 (HDR highlights exceed 1.0). Per-channel.
|
||||
fn pq_to_linear(c: vec3<f32>) -> vec3<f32> {
|
||||
let m1 = 0.1593017578125;
|
||||
let m2 = 78.84375;
|
||||
let c1 = 0.8359375;
|
||||
let c2 = 18.8515625;
|
||||
let c3 = 18.6875;
|
||||
let e = pow(max(c, vec3<f32>(0.0)), vec3<f32>(1.0 / m2));
|
||||
let num = max(e - vec3<f32>(c1), vec3<f32>(0.0));
|
||||
let den = vec3<f32>(c2) - c3 * e;
|
||||
let nits = pow(num / den, vec3<f32>(1.0 / m1)) * 10000.0; // 0..10000 cd/m²
|
||||
return nits / 203.0;
|
||||
}
|
||||
|
||||
// ARIB STD-B67 (HLG) inverse-OETF → scene light, normalized so reference white (signal 0.75) = 1.0.
|
||||
// The display OOTF is omitted (scene-referred compositing); approximate but reasonable for SDR-out.
|
||||
fn hlg_to_linear(c: vec3<f32>) -> vec3<f32> {
|
||||
let a = 0.17883277;
|
||||
let b = 0.28466892;
|
||||
let cc = 0.55991073;
|
||||
let lo = (c * c) / 3.0;
|
||||
let hi = (exp((c - vec3<f32>(cc)) / a) + vec3<f32>(b)) / 12.0;
|
||||
let scene = select(lo, hi, c > vec3<f32>(0.5));
|
||||
return scene / 0.26496256; // hlg_inv_oetf(0.75): put reference white at 1.0
|
||||
}
|
||||
|
||||
// BT.2020 → BT.709 primaries, linear light (ITU-R BT.2087). Out-of-709 colours go negative → clamp.
|
||||
fn bt2020_to_bt709(c: vec3<f32>) -> vec3<f32> {
|
||||
let r = 1.660491 * c.r - 0.587641 * c.g - 0.072850 * c.b;
|
||||
let g = -0.124550 * c.r + 1.132900 * c.g - 0.008349 * c.b;
|
||||
let b = -0.018151 * c.r - 0.100579 * c.g + 1.118730 * c.b;
|
||||
return max(vec3<f32>(r, g, b), vec3<f32>(0.0));
|
||||
}
|
||||
|
||||
@fragment
|
||||
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
|
||||
let m = mat3x3<f32>(params.col0.xyz, params.col1.xyz, params.col2.xyz);
|
||||
|
|
@ -79,9 +114,23 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
|
|||
let r = Y + params.coeffs.x * Cr;
|
||||
let g = Y + params.coeffs.y * Cb + params.coeffs.z * Cr;
|
||||
let b = Y + params.coeffs.w * Cb;
|
||||
let rgb_gamma = clamp(vec3<f32>(r, g, b), vec3<f32>(0.0), vec3<f32>(1.0));
|
||||
// Valid encoded signal is [0,1]; clamp before the EOTF (HDR comes from the EOTF, not overshoot).
|
||||
let rgb_enc = clamp(vec3<f32>(r, g, b), vec3<f32>(0.0), vec3<f32>(1.0));
|
||||
|
||||
// Encoded R'G'B' → scene-linear (graphics white = 1.0; HDR may exceed 1.0).
|
||||
var rgb_lin: vec3<f32>;
|
||||
if params.flags.y == 1u {
|
||||
rgb_lin = pq_to_linear(rgb_enc);
|
||||
} else if params.flags.y == 2u {
|
||||
rgb_lin = hlg_to_linear(rgb_enc);
|
||||
} else {
|
||||
rgb_lin = srgb_to_linear(rgb_enc);
|
||||
}
|
||||
|
||||
// Wide-gamut → BT.709 in linear light to match the compositor's primaries.
|
||||
if params.flags.z == 1u {
|
||||
rgb_lin = bt2020_to_bt709(rgb_lin);
|
||||
}
|
||||
|
||||
// R'G'B' is gamma-encoded; the HDR target is linear → undo the transfer.
|
||||
let rgb_lin = srgb_to_linear(rgb_gamma);
|
||||
return vec4<f32>(rgb_lin, 1.0);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1703,7 +1703,8 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
let y_view = gpu.y.create_view(&Default::default());
|
||||
let uv_view = gpu.uv.create_view(&Default::default());
|
||||
shared.nv12_blit.blit(
|
||||
device, queue, &y_view, &uv_view, hdr_layer_view, &bt, gpu.full_range, gpu.coeffs,
|
||||
device, queue, &y_view, &uv_view, hdr_layer_view, &bt,
|
||||
gpu.full_range, gpu.coeffs, gpu.transfer, gpu.primaries,
|
||||
);
|
||||
} else {
|
||||
// Reuse the GPU texture for this frame if it's unchanged (a
|
||||
|
|
|
|||
Loading…
Reference in New Issue