Composite grouped/nested video on the GPU path

Imported video is a Group[Video, Audio] that rendered as a Vello-baked
Vector layer, re-uploading the full frame to Vello's image atlas every
frame (~17ms/frame at 1080p, hitting playback and export alike). Extract
video frames out of the Group/clip scene recursion into
VideoRenderInstances so they composite via the GPU Video path; mixed
video+vector containers fall back to Vello (correct, unaccelerated).

Also route video through hardware sRGB decode: upload raw sRGB bytes to an
Rgba8UnormSrgb texture and blit with a non-unpremultiplying shader variant
(blit_straight), removing the per-frame per-pixel CPU sRGB->linear pass.
Add an F3 GPU-timestamp timer and a per-frame video texture cache.

Drops the live composite of a 1080p video from ~17ms to ~2-3ms.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Skyler Lehmkuhl 2026-06-22 17:38:30 -04:00
parent ce151ffd61
commit 5844a0f070
8 changed files with 637 additions and 86 deletions

View File

@ -232,6 +232,21 @@ pub struct VideoRenderInstance {
pub opacity: f32, pub opacity: f32,
} }
/// Sink for pulling video frames out of a container layer's scene recursion, so
/// they composite via the fast GPU Video path instead of baking into Vello.
///
/// Threaded as `Option<&mut VideoExtract>` through the isolated-render scene
/// functions. When present, [`render_video_layer`] pushes a [`VideoRenderInstance`]
/// instead of drawing into the Vello scene. `drew_other` is set whenever any
/// non-video primitive (vector graph, image asset, raster) is emitted; that forces
/// the safe Vello fallback because the container mixes video with other content
/// and we can't preserve z-order by extraction alone.
#[derive(Default)]
struct VideoExtract {
instances: Vec<VideoRenderInstance>,
drew_other: bool,
}
/// Type of rendered layer for compositor handling /// Type of rendered layer for compositor handling
pub enum RenderedLayerType { pub enum RenderedLayerType {
/// Vector / group layer — Vello scene in `RenderedLayer::scene` is used. /// Vector / group layer — Vello scene in `RenderedLayer::scene` is used.
@ -428,6 +443,30 @@ pub fn render_document_for_compositing(
} }
} }
// One-shot diagnostic: dump what the compositor actually receives. Set
// LB_LAYER_DEBUG=1 to print a single snapshot of each layer's resolved type.
if std::env::var("LB_LAYER_DEBUG").is_ok() {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| {
eprintln!("[LB_LAYER_DEBUG] composite layers = {}", rendered_layers.len());
for (i, l) in rendered_layers.iter().enumerate() {
let desc = match &l.layer_type {
RenderedLayerType::Vector =>
format!("Vector (has_content={}, scene_empty={})", l.has_content, l.cpu_pixmap.is_none()),
RenderedLayerType::Raster { width, height, dirty, .. } =>
format!("Raster {width}x{height} dirty={dirty}"),
RenderedLayerType::Video { instances } =>
format!("Video ({} instance(s))", instances.len()),
RenderedLayerType::Float { width, height, .. } =>
format!("Float {width}x{height}"),
RenderedLayerType::Effect { effect_instances } =>
format!("Effect ({} instance(s))", effect_instances.len()),
};
eprintln!("[LB_LAYER_DEBUG] layer[{i}] id={} type={desc}", l.layer_id);
}
});
}
CompositeRenderResult { CompositeRenderResult {
background, background,
background_cpu: None, background_cpu: None,
@ -462,6 +501,9 @@ pub fn render_layer_isolated(
// Render layer content with full opacity (1.0) - opacity applied during compositing // Render layer content with full opacity (1.0) - opacity applied during compositing
match layer { match layer {
AnyLayer::Vector(vector_layer) => { AnyLayer::Vector(vector_layer) => {
// Render into the scene with an extraction sink so a clip that is purely
// video (no vector geometry) composites via the fast GPU Video path.
let mut ex = VideoExtract::default();
render_vector_layer_to_scene( render_vector_layer_to_scene(
document, document,
time, time,
@ -471,10 +513,28 @@ pub fn render_layer_isolated(
1.0, // Full opacity - layer opacity handled in compositing 1.0, // Full opacity - layer opacity handled in compositing
image_cache, image_cache,
video_manager, video_manager,
Some(&mut ex),
); );
rendered.has_content = vector_layer.graph_at_time(time) if !ex.instances.is_empty() && !ex.drew_other {
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted)) // Pure video: discard the (now-empty) scene and emit GPU instances.
|| !vector_layer.clip_instances.is_empty(); rendered.scene = Scene::new();
rendered.has_content = true;
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
} else {
if !ex.instances.is_empty() {
// Mixed video + vector: the first pass diverted the video out of
// the scene, so re-render with no sink to bake it back in (correct
// z-order; Vello path). Rare — fast-splitting is deferred.
rendered.scene = Scene::new();
render_vector_layer_to_scene(
document, time, vector_layer, &mut rendered.scene,
base_transform, 1.0, image_cache, video_manager, None,
);
}
rendered.has_content = vector_layer.graph_at_time(time)
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted))
|| !vector_layer.clip_instances.is_empty();
}
} }
AnyLayer::Audio(_) => { AnyLayer::Audio(_) => {
// Audio layers don't render visually // Audio layers don't render visually
@ -577,15 +637,34 @@ pub fn render_layer_isolated(
return RenderedLayer::effect_layer(layer_id, opacity, active_effects); return RenderedLayer::effect_layer(layer_id, opacity, active_effects);
} }
AnyLayer::Group(group_layer) => { AnyLayer::Group(group_layer) => {
// Render each child layer's content into the group's scene // Render each child into the group's scene with an extraction sink. The
// common imported-video case is a Group[Video, Audio] — audio draws
// nothing, so it's pure video and composites via the GPU Video path.
let mut ex = VideoExtract::default();
for child in &group_layer.children { for child in &group_layer.children {
render_layer( render_layer(
document, time, child, &mut rendered.scene, base_transform, document, time, child, &mut rendered.scene, base_transform,
1.0, // Full opacity - layer opacity handled in compositing 1.0, // Full opacity - layer opacity handled in compositing
image_cache, video_manager, camera_frame, image_cache, video_manager, camera_frame, Some(&mut ex),
); );
} }
rendered.has_content = !group_layer.children.is_empty(); if !ex.instances.is_empty() && !ex.drew_other {
rendered.scene = Scene::new();
rendered.has_content = true;
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
} else {
if !ex.instances.is_empty() {
// Mixed: re-render with no sink to bake the video back into the scene.
rendered.scene = Scene::new();
for child in &group_layer.children {
render_layer(
document, time, child, &mut rendered.scene, base_transform,
1.0, image_cache, video_manager, camera_frame, None,
);
}
}
rendered.has_content = !group_layer.children.is_empty();
}
} }
AnyLayer::Raster(raster_layer) => { AnyLayer::Raster(raster_layer) => {
if let Some(kf) = raster_layer.keyframe_at(time) { if let Some(kf) = raster_layer.keyframe_at(time) {
@ -614,6 +693,7 @@ fn render_vector_layer_to_scene(
parent_opacity: f64, parent_opacity: f64,
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
extract: Option<&mut VideoExtract>,
) { ) {
render_vector_layer( render_vector_layer(
document, document,
@ -624,6 +704,7 @@ fn render_vector_layer_to_scene(
parent_opacity, parent_opacity,
image_cache, image_cache,
video_manager, video_manager,
extract,
); );
} }
@ -691,10 +772,10 @@ pub fn render_document_with_transform(
for layer in document.visible_layers() { for layer in document.visible_layers() {
if any_soloed { if any_soloed {
if layer.soloed() { if layer.soloed() {
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None); render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
} }
} else { } else {
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None); render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
} }
} }
} }
@ -755,10 +836,11 @@ fn render_layer(
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
camera_frame: Option<&crate::webcam::CaptureFrame>, camera_frame: Option<&crate::webcam::CaptureFrame>,
mut extract: Option<&mut VideoExtract>,
) { ) {
match layer { match layer {
AnyLayer::Vector(vector_layer) => { AnyLayer::Vector(vector_layer) => {
render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager) render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager, extract)
} }
AnyLayer::Audio(_) => { AnyLayer::Audio(_) => {
// Audio layers don't render visually // Audio layers don't render visually
@ -766,18 +848,20 @@ fn render_layer(
AnyLayer::Video(video_layer) => { AnyLayer::Video(video_layer) => {
let mut video_mgr = video_manager.lock().unwrap(); let mut video_mgr = video_manager.lock().unwrap();
let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None }; let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None };
render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame); render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame, extract);
} }
AnyLayer::Effect(_) => { AnyLayer::Effect(_) => {
// Effect layers are processed during GPU compositing, not rendered to scene // Effect layers are processed during GPU compositing, not rendered to scene
} }
AnyLayer::Group(group_layer) => { AnyLayer::Group(group_layer) => {
// Render each child layer in the group // Render each child layer in the group, passing the extract sink down.
for child in &group_layer.children { for child in &group_layer.children {
render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame); render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame, extract.as_deref_mut());
} }
} }
AnyLayer::Raster(raster_layer) => { AnyLayer::Raster(raster_layer) => {
// Raster is non-video content — force the Vello fallback if extracting.
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
render_raster_layer_to_scene(raster_layer, time, scene, base_transform); render_raster_layer_to_scene(raster_layer, time, scene, base_transform);
} }
} }
@ -816,6 +900,7 @@ pub fn render_single_clip_instance(
render_clip_instance( render_clip_instance(
document, time, clip_instance, layer_opacity, scene, base_transform, document, time, clip_instance, layer_opacity, scene, base_transform,
&vector_layer.layer.animation_data, image_cache, video_manager, group_end_time, &vector_layer.layer.animation_data, image_cache, video_manager, group_end_time,
None, // edit-inside-clip overlay keeps the Vello path
); );
} }
@ -831,6 +916,7 @@ fn render_clip_instance(
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
group_end_time: Option<f64>, group_end_time: Option<f64>,
mut extract: Option<&mut VideoExtract>,
) { ) {
// Try to find the clip in the document's clip libraries // Try to find the clip in the document's clip libraries
// For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented) // For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented)
@ -972,7 +1058,7 @@ fn render_clip_instance(
if !layer_node.data.visible() { if !layer_node.data.visible() {
continue; continue;
} }
render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None); render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None, extract.as_deref_mut());
} }
} }
@ -986,6 +1072,7 @@ fn render_video_layer(
parent_opacity: f64, parent_opacity: f64,
video_manager: &mut crate::video::VideoManager, video_manager: &mut crate::video::VideoManager,
camera_frame: Option<&crate::webcam::CaptureFrame>, camera_frame: Option<&crate::webcam::CaptureFrame>,
mut extract: Option<&mut VideoExtract>,
) { ) {
use crate::animation::TransformProperty; use crate::animation::TransformProperty;
@ -1151,14 +1238,26 @@ fn render_video_layer(
Affine::IDENTITY Affine::IDENTITY
}; };
// Render video frame as image fill // Extract to the GPU Video path when a sink is present; otherwise bake into
scene.fill( // the Vello scene. The combined frame-pixel → document transform is
Fill::NonZero, // instance_transform * brush_transform (matching the top-level Video path).
instance_transform, if let Some(ex) = extract.as_deref_mut() {
&image_with_alpha, ex.instances.push(VideoRenderInstance {
Some(brush_transform), rgba_data: frame.rgba_data.clone(),
&video_rect, width: frame.width,
); height: frame.height,
transform: instance_transform * brush_transform,
opacity: final_opacity,
});
} else {
scene.fill(
Fill::NonZero,
instance_transform,
&image_with_alpha,
Some(brush_transform),
&video_rect,
);
}
clip_rendered = true; clip_rendered = true;
} }
@ -1194,13 +1293,25 @@ fn render_video_layer(
* Affine::translate((offset_x, offset_y)) * Affine::translate((offset_x, offset_y))
* Affine::scale(uniform_scale); * Affine::scale(uniform_scale);
scene.fill( // preview_transform maps frame-pixel space → document directly, so it
Fill::NonZero, // is exactly the instance transform for the GPU path.
preview_transform, if let Some(ex) = extract.as_deref_mut() {
&image_with_alpha, ex.instances.push(VideoRenderInstance {
None, rgba_data: frame.rgba_data.clone(),
&frame_rect, width: frame.width,
); height: frame.height,
transform: preview_transform,
opacity: final_opacity,
});
} else {
scene.fill(
Fill::NonZero,
preview_transform,
&image_with_alpha,
None,
&frame_rect,
);
}
} }
} }
} }
@ -1352,6 +1463,7 @@ fn render_vector_layer(
parent_opacity: f64, parent_opacity: f64,
image_cache: &mut ImageCache, image_cache: &mut ImageCache,
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>, video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
mut extract: Option<&mut VideoExtract>,
) { ) {
// Cascade opacity: parent_opacity × layer.opacity // Cascade opacity: parent_opacity × layer.opacity
let layer_opacity = parent_opacity * layer.layer.opacity; let layer_opacity = parent_opacity * layer.layer.opacity;
@ -1359,6 +1471,10 @@ fn render_vector_layer(
// Render the layer's own VectorGraph (loose shapes) first, then clip instances // Render the layer's own VectorGraph (loose shapes) first, then clip instances
// (groups / movie clips) on top. Shape tweens are applied here. // (groups / movie clips) on top. Shape tweens are applied here.
if let Some(graph) = layer.tweened_graph_at(time) { if let Some(graph) = layer.tweened_graph_at(time) {
// Loose vector geometry (and any image-asset fills) is non-video content —
// force the Vello fallback. Conservative: a present-but-empty graph still
// trips this, which only costs the fallback, never correctness.
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache); render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache);
} }
@ -1370,7 +1486,7 @@ fn render_vector_layer(
let frame_duration = 1.0 / document.framerate; let frame_duration = 1.0 / document.framerate;
layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration) layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration)
}); });
render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time); render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time, extract.as_deref_mut());
} }
} }

View File

@ -39,6 +39,77 @@ pub fn update_prepare_timing(
t.composite_ms = composite_ms; t.composite_ms = composite_ms;
} }
} }
/// GPU-measured composite cost (from timestamp queries; see `gpu_timer.rs`).
#[derive(Debug, Clone, Default)]
pub struct GpuCompositeTiming {
/// True when the adapter supports timestamp queries (else the ms is meaningless).
pub supported: bool,
/// GPU time of the whole composite section (Vello render + sRGB→linear +
/// compositor + tonemap), in milliseconds. Read back asynchronously, so it
/// lags the displayed frame by a frame or two.
pub composite_gpu_ms: f64,
/// Layers composited this frame.
pub layers: u32,
/// `queue.submit()` calls in the composite section this frame.
pub submits: u32,
}
static GPU_COMPOSITE: OnceLock<Mutex<GpuCompositeTiming>> = OnceLock::new();
/// Called from `VelloCallback::prepare()` with the GPU composite measurement.
pub fn update_gpu_composite(supported: bool, composite_gpu_ms: f64, layers: u32, submits: u32) {
let cell = GPU_COMPOSITE.get_or_init(|| Mutex::new(GpuCompositeTiming::default()));
if let Ok(mut t) = cell.lock() {
t.supported = supported;
t.composite_gpu_ms = composite_gpu_ms;
t.layers = layers;
t.submits = submits;
}
}
fn get_gpu_composite() -> GpuCompositeTiming {
GPU_COMPOSITE
.get_or_init(|| Mutex::new(GpuCompositeTiming::default()))
.lock()
.map(|t| t.clone())
.unwrap_or_default()
}
/// CPU-side breakdown of the composite section (wall-clock `Instant` deltas). Since
/// the GPU idles waiting on these CPU operations, this is where the per-frame cost
/// actually lives. Sums should ≈ the CPU `composite_ms` for the doc's active paths.
#[derive(Debug, Clone, Default)]
pub struct CompositeCpuBreakdown {
/// `renderer.render_to_texture` — Vello scene encode + its internal submit.
pub vello_ms: f64,
/// `srgb_to_linear.convert` — recording the conversion pass.
pub convert_ms: f64,
/// `canvas_blit.blit` — recording + its internal submit.
pub blit_ms: f64,
/// `compositor.composite` — recording + per-call uniforms buffer / bind group alloc.
pub composite_ms: f64,
/// Explicit `queue.submit()` calls.
pub submit_ms: f64,
}
static COMPOSITE_CPU: OnceLock<Mutex<CompositeCpuBreakdown>> = OnceLock::new();
/// Called from `VelloCallback::prepare()` with the composite CPU breakdown.
pub fn update_composite_cpu(b: CompositeCpuBreakdown) {
let cell = COMPOSITE_CPU.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()));
if let Ok(mut t) = cell.lock() {
*t = b;
}
}
fn get_composite_cpu() -> CompositeCpuBreakdown {
COMPOSITE_CPU
.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()))
.lock()
.map(|t| t.clone())
.unwrap_or_default()
}
/// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the /// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the
/// raster-layer texture cache — the only unbounded-by-default VRAM consumer. /// raster-layer texture cache — the only unbounded-by-default VRAM consumer.
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
@ -90,6 +161,12 @@ pub struct DebugStats {
// GPU prepare() timing breakdown (from render thread) // GPU prepare() timing breakdown (from render thread)
pub prepare_timing: PrepareTiming, pub prepare_timing: PrepareTiming,
// GPU-measured composite cost (timestamp queries)
pub gpu_composite: GpuCompositeTiming,
// CPU breakdown of the composite section
pub composite_cpu: CompositeCpuBreakdown,
// Performance metrics for each section // Performance metrics for each section
pub timing_memory_us: u64, pub timing_memory_us: u64,
pub timing_gpu_us: u64, pub timing_gpu_us: u64,
@ -254,6 +331,8 @@ impl DebugStatsCollector {
audio_input_devices, audio_input_devices,
has_pointer, has_pointer,
prepare_timing, prepare_timing,
gpu_composite: get_gpu_composite(),
composite_cpu: get_composite_cpu(),
timing_memory_us, timing_memory_us,
timing_gpu_us, timing_gpu_us,
timing_midi_us, timing_midi_us,
@ -306,8 +385,33 @@ pub fn render_debug_overlay(ctx: &egui::Context, stats: &DebugStats) {
ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms)); ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms));
ui.label(format!(" removals: {:.2} ms", pt.removals_ms)); ui.label(format!(" removals: {:.2} ms", pt.removals_ms));
ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms)); ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms));
ui.label(format!(" scene_build: {:.2} ms", pt.scene_build_ms)); ui.label(format!(" scene_build: {:.2} ms (CPU)", pt.scene_build_ms));
ui.label(format!(" composite: {:.2} ms", pt.composite_ms)); ui.label(format!(" composite: {:.2} ms (CPU)", pt.composite_ms));
// GPU-measured composite cost (timestamp queries).
let gc = &stats.gpu_composite;
if gc.supported {
ui.colored_label(
egui::Color32::LIGHT_GREEN,
format!("GPU composite: {:.2} ms (GPU)", gc.composite_gpu_ms),
);
ui.label(format!(" layers: {} submits: {}", gc.layers, gc.submits));
} else {
ui.label(format!(
"GPU composite: n/a (no timestamp support) layers: {} submits: {}",
gc.layers, gc.submits
));
}
// CPU breakdown of the composite (where the GPU is actually waiting).
let cc = &stats.composite_cpu;
let cc_sum = cc.vello_ms + cc.convert_ms + cc.blit_ms + cc.composite_ms + cc.submit_ms;
ui.colored_label(egui::Color32::LIGHT_BLUE, format!("Composite CPU breakdown: {:.2} ms", cc_sum));
ui.label(format!(" vello(render): {:.2} ms", cc.vello_ms));
ui.label(format!(" srgb→linear: {:.2} ms", cc.convert_ms));
ui.label(format!(" blit: {:.2} ms", cc.blit_ms));
ui.label(format!(" compositor: {:.2} ms", cc.composite_ms));
ui.label(format!(" queue.submit: {:.2} ms", cc.submit_ms));
ui.add_space(8.0); ui.add_space(8.0);

View File

@ -830,22 +830,13 @@ fn composite_document_to_hdr(
if inst.rgba_data.is_empty() { continue; } if inst.rgba_data.is_empty() { continue; }
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
// sRGB straight-alpha → linear premultiplied // Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| { // decodes to linear on sample (no per-pixel CPU conversion). Blit with
let a = p[3] as f32 / 255.0; // blit_straight so the shader doesn't unpremultiply.
let lin = |c: u8| -> f32 { let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex"));
let f = c as f32 / 255.0;
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) }
};
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8;
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8;
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
[r, g, b, p[3]]
}).collect();
let tex = upload_transient_texture(device, queue, &linear, inst.width, inst.height, Some("export_video_frame_tex"));
let tex_view = tex.create_view(&Default::default()); let tex_view = tex.create_view(&Default::default());
let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height); let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height);
gpu_resources.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None); gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal); let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal);
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") }); let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") });
gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None); gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None);
@ -865,7 +856,7 @@ fn composite_document_to_hdr(
}; };
[lin(p[0]), lin(p[1]), lin(p[2]), p[3]] [lin(p[0]), lin(p[1]), lin(p[2]), p[3]]
}).collect(); }).collect();
let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, Some("export_float_tex")); let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, wgpu::TextureFormat::Rgba8Unorm, Some("export_float_tex"));
let tex_view = tex.create_view(&Default::default()); let tex_view = tex.create_view(&Default::default());
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
@ -919,13 +910,16 @@ fn composite_document_to_hdr(
Ok(()) Ok(())
} }
/// Upload `pixels` to a transient `Rgba8Unorm` GPU texture (TEXTURE_BINDING | COPY_DST). /// Upload `pixels` to a transient GPU texture (TEXTURE_BINDING | COPY_DST) in the
/// given format. Use `Rgba8UnormSrgb` to upload raw sRGB bytes and let the GPU
/// decode to linear on sample (no CPU conversion).
fn upload_transient_texture( fn upload_transient_texture(
device: &wgpu::Device, device: &wgpu::Device,
queue: &wgpu::Queue, queue: &wgpu::Queue,
pixels: &[u8], pixels: &[u8],
width: u32, width: u32,
height: u32, height: u32,
format: wgpu::TextureFormat,
label: Option<&'static str>, label: Option<&'static str>,
) -> wgpu::Texture { ) -> wgpu::Texture {
let tex = device.create_texture(&wgpu::TextureDescriptor { let tex = device.create_texture(&wgpu::TextureDescriptor {
@ -933,7 +927,7 @@ fn upload_transient_texture(
size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 },
mip_level_count: 1, sample_count: 1, mip_level_count: 1, sample_count: 1,
dimension: wgpu::TextureDimension::D2, dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm, format,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[], view_formats: &[],
}); });

View File

@ -1951,6 +1951,9 @@ impl GpuBrushEngine {
/// the camera transform. /// the camera transform.
pub struct CanvasBlitPipeline { pub struct CanvasBlitPipeline {
pub pipeline: wgpu::RenderPipeline, pub pipeline: wgpu::RenderPipeline,
/// Variant for straight-alpha sources (hardware-sRGB video frames): the
/// fragment shader skips the unpremultiply. See [`CanvasBlitPipeline::blit_straight`].
pub pipeline_straight: wgpu::RenderPipeline,
pub bg_layout: wgpu::BindGroupLayout, pub bg_layout: wgpu::BindGroupLayout,
pub sampler: wgpu::Sampler, pub sampler: wgpu::Sampler,
/// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res /// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res
@ -2132,6 +2135,39 @@ impl CanvasBlitPipeline {
}, },
); );
// Variant pipeline for straight-alpha sources (hardware-sRGB video frames):
// identical except the fragment shader skips the unpremultiply.
let pipeline_straight = device.create_render_pipeline(
&wgpu::RenderPipelineDescriptor {
label: Some("canvas_blit_pipeline_straight"),
layout: Some(&pipeline_layout),
vertex: wgpu::VertexState {
module: &shader,
entry_point: Some("vs_main"),
buffers: &[],
compilation_options: Default::default(),
},
fragment: Some(wgpu::FragmentState {
module: &shader,
entry_point: Some("fs_main_straight"),
targets: &[Some(wgpu::ColorTargetState {
format: wgpu::TextureFormat::Rgba16Float,
blend: None,
write_mask: wgpu::ColorWrites::ALL,
})],
compilation_options: Default::default(),
}),
primitive: wgpu::PrimitiveState {
topology: wgpu::PrimitiveTopology::TriangleStrip,
..Default::default()
},
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview: None,
cache: None,
},
);
let sampler = device.create_sampler(&wgpu::SamplerDescriptor { let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
label: Some("canvas_blit_sampler"), label: Some("canvas_blit_sampler"),
address_mode_u: wgpu::AddressMode::ClampToEdge, address_mode_u: wgpu::AddressMode::ClampToEdge,
@ -2165,7 +2201,7 @@ impl CanvasBlitPipeline {
..Default::default() ..Default::default()
}); });
Self { pipeline, bg_layout, sampler, linear_sampler, mask_sampler } Self { pipeline, pipeline_straight, bg_layout, sampler, linear_sampler, mask_sampler }
} }
/// Render the canvas texture into `target_view` (Rgba16Float) with the given camera. /// Render the canvas texture into `target_view` (Rgba16Float) with the given camera.
@ -2183,7 +2219,7 @@ impl CanvasBlitPipeline {
transform: &BlitTransform, transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>, mask_view: Option<&wgpu::TextureView>,
) { ) {
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler); self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline);
} }
/// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies). /// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies).
@ -2196,9 +2232,25 @@ impl CanvasBlitPipeline {
transform: &BlitTransform, transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>, mask_view: Option<&wgpu::TextureView>,
) { ) {
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler); self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler, &self.pipeline);
} }
/// Blit a **straight-alpha** source (e.g. a video frame uploaded to an
/// `Rgba8UnormSrgb` texture, hardware-decoded to linear on sample). Uses the
/// `fs_main_straight` pipeline, which skips the unpremultiply that `blit` does.
pub fn blit_straight(
&self,
device: &wgpu::Device,
queue: &wgpu::Queue,
canvas_view: &wgpu::TextureView,
target_view: &wgpu::TextureView,
transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>,
) {
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline_straight);
}
#[allow(clippy::too_many_arguments)]
fn blit_with( fn blit_with(
&self, &self,
device: &wgpu::Device, device: &wgpu::Device,
@ -2208,6 +2260,7 @@ impl CanvasBlitPipeline {
transform: &BlitTransform, transform: &BlitTransform,
mask_view: Option<&wgpu::TextureView>, mask_view: Option<&wgpu::TextureView>,
canvas_sampler: &wgpu::Sampler, canvas_sampler: &wgpu::Sampler,
pipeline: &wgpu::RenderPipeline,
) { ) {
// When no mask is provided, create a temporary 1×1 all-white texture. // When no mask is provided, create a temporary 1×1 all-white texture.
// (queue is already available here, unlike in new()) // (queue is already available here, unlike in new())
@ -2296,7 +2349,7 @@ impl CanvasBlitPipeline {
occlusion_query_set: None, occlusion_query_set: None,
timestamp_writes: None, timestamp_writes: None,
}); });
rp.set_pipeline(&self.pipeline); rp.set_pipeline(pipeline);
rp.set_bind_group(0, &bg, &[]); rp.set_bind_group(0, &bg, &[]);
rp.draw(0..4, 0..1); rp.draw(0..4, 0..1);
} }

View File

@ -0,0 +1,135 @@
//! Minimal GPU timestamp timer for the composite pipeline.
//!
//! Brackets a section of GPU work with two timestamps and reads the elapsed GPU
//! time back asynchronously (no pipeline stall). Used to attribute the per-frame
//! composite cost (Vello render + sRGB→linear + compositor + tonemap) shown in F3.
//!
//! Requires `TIMESTAMP_QUERY` + `TIMESTAMP_QUERY_INSIDE_ENCODERS`; [`FrameGpuTimer::new`]
//! returns `None` when the adapter doesn't support them, and all call sites no-op.
use std::sync::{Arc, Mutex};
/// State of the single readback buffer (shared with the map callback).
#[derive(Clone, Copy, PartialEq)]
enum Readback {
/// Available to resolve into this frame.
Free,
/// Submitted + `map_async` in flight; don't touch until the callback fires.
Mapping,
/// Mapped and ready to read.
Ready,
}
/// Times one GPU section (two timestamps) per frame with intermittent async readback.
pub struct FrameGpuTimer {
query_set: wgpu::QuerySet,
resolve_buf: wgpu::Buffer,
readback_buf: wgpu::Buffer,
state: Arc<Mutex<Readback>>,
/// Nanoseconds per timestamp tick.
period_ns: f32,
/// Most recent measured GPU time for the bracketed section, in milliseconds.
last_ms: f64,
}
impl FrameGpuTimer {
/// Required device features for GPU timestamp timing.
pub fn required_features() -> wgpu::Features {
wgpu::Features::TIMESTAMP_QUERY | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS
}
/// Create a timer, or `None` if the device lacks timestamp support.
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option<Self> {
if !device.features().contains(Self::required_features()) {
return None;
}
let query_set = device.create_query_set(&wgpu::QuerySetDescriptor {
label: Some("composite_gpu_timer"),
ty: wgpu::QueryType::Timestamp,
count: 2,
});
// 2 timestamps × u64.
let size = 2 * std::mem::size_of::<u64>() as u64;
let resolve_buf = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("composite_gpu_timer_resolve"),
size,
usage: wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let readback_buf = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("composite_gpu_timer_readback"),
size,
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
Some(Self {
query_set,
resolve_buf,
readback_buf,
state: Arc::new(Mutex::new(Readback::Free)),
period_ns: queue.get_timestamp_period(),
last_ms: 0.0,
})
}
/// Write the **start** timestamp (call just before the bracketed GPU work).
pub fn start(&self, device: &wgpu::Device, queue: &wgpu::Queue) {
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("composite_gpu_timer_start"),
});
enc.write_timestamp(&self.query_set, 0);
queue.submit(Some(enc.finish()));
}
/// Write the **end** timestamp and, if the readback buffer is free, resolve +
/// kick off an async read. Also consumes a previously-completed read into
/// `last_ms`. Call just after the bracketed GPU work.
pub fn end(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
// 1. Consume a completed readback first (so the buffer is free to reuse).
let cur = *self.state.lock().unwrap();
if cur == Readback::Ready {
{
let view = self.readback_buf.slice(..).get_mapped_range();
let t0 = u64::from_le_bytes(view[0..8].try_into().unwrap());
let t1 = u64::from_le_bytes(view[8..16].try_into().unwrap());
// Timestamps can wrap or arrive out of order across queue resets; guard.
let ticks = t1.saturating_sub(t0);
self.last_ms = ticks as f64 * self.period_ns as f64 / 1.0e6;
}
self.readback_buf.unmap();
*self.state.lock().unwrap() = Readback::Free;
}
// 2. End timestamp + resolve + copy, only when the buffer is free.
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("composite_gpu_timer_end"),
});
enc.write_timestamp(&self.query_set, 1);
let can_read = *self.state.lock().unwrap() == Readback::Free;
if can_read {
enc.resolve_query_set(&self.query_set, 0..2, &self.resolve_buf, 0);
enc.copy_buffer_to_buffer(
&self.resolve_buf,
0,
&self.readback_buf,
0,
2 * std::mem::size_of::<u64>() as u64,
);
}
queue.submit(Some(enc.finish()));
if can_read {
*self.state.lock().unwrap() = Readback::Mapping;
let state = Arc::clone(&self.state);
self.readback_buf.slice(..).map_async(wgpu::MapMode::Read, move |res| {
*state.lock().unwrap() = if res.is_ok() { Readback::Ready } else { Readback::Free };
});
}
}
/// Most recently measured GPU time of the bracketed section, in milliseconds.
pub fn last_ms(&self) -> f64 {
self.last_ms
}
}

View File

@ -50,6 +50,7 @@ use effect_thumbnails::EffectThumbnailGenerator;
mod custom_cursor; mod custom_cursor;
mod tablet; mod tablet;
mod debug_overlay; mod debug_overlay;
mod gpu_timer;
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
mod test_mode; mod test_mode;
@ -174,8 +175,12 @@ fn main() -> eframe::Result {
device_descriptor: std::sync::Arc::new(|adapter| { device_descriptor: std::sync::Arc::new(|adapter| {
let features = adapter.features(); let features = adapter.features();
// Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's // Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's
// unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability) // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability).
let optional_features = wgpu::Features::SHADER_F16; // TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer
// (gpu_timer.rs); both are optional and no-op when unsupported.
let optional_features = wgpu::Features::SHADER_F16
| wgpu::Features::TIMESTAMP_QUERY
| wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS;
let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl {
wgpu::Limits::downlevel_webgl2_defaults() wgpu::Limits::downlevel_webgl2_defaults()

View File

@ -75,3 +75,25 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
let tinted = base + tint - base * tint; let tinted = base + tint - base * tint;
return vec4<f32>(tinted, masked_a); return vec4<f32>(tinted, masked_a);
} }
// Variant for sources that are ALREADY straight-alpha linear notably a video
// frame uploaded to an `Rgba8UnormSrgb` texture, where the hardware decodes
// sRGBlinear on sample and leaves alpha untouched. No unpremultiply (the source
// was never premultiplied), so we skip the divide entirely. The compositor wants
// straight-alpha linear, which is exactly what the sample already is.
@fragment
fn fs_main_straight(in: VertexOutput) -> @location(0) vec4<f32> {
let m = mat3x3<f32>(transform.col0.xyz, transform.col1.xyz, transform.col2.xyz);
let canvas_uv = (m * vec3<f32>(in.uv.x, in.uv.y, 1.0)).xy;
if canvas_uv.x < 0.0 || canvas_uv.x > 1.0
|| canvas_uv.y < 0.0 || canvas_uv.y > 1.0 {
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
}
let c = textureSample(canvas_tex, canvas_sampler, canvas_uv);
let mask = textureSample(mask_tex, mask_sampler, canvas_uv).r;
let tint = vec3<f32>(transform.col0.w, transform.col1.w, transform.col2.w);
let tinted = c.rgb + tint - c.rgb * tint;
return vec4<f32>(tinted, c.a * mask);
}

View File

@ -45,6 +45,89 @@ fn upload_pixmap_to_texture(queue: &wgpu::Queue, texture: &wgpu::Texture, pixmap
/// Set to true to use the new pipeline, false for legacy single-scene rendering /// Set to true to use the new pipeline, false for legacy single-scene rendering
const USE_HDR_COMPOSITING: bool = true; // Enabled for testing const USE_HDR_COMPOSITING: bool = true; // Enabled for testing
/// Caches GPU textures for decoded video frames, keyed by the frame buffer's `Arc`
/// identity. A static/paused video then costs ~nothing per repaint (cache hit → no
/// per-pixel CPU sRGB→linear conversion, no texture allocation, no upload). The
/// cached texture holds premultiplied-linear RGBA8 — exactly what `canvas_blit`
/// expects. During playback each new decoded frame is a fresh `Arc` → one
/// conversion+upload per frame (not per repaint).
struct CachedVideoFrame {
/// Keep the source buffer alive so its pointer (our cache key) can't be reused.
_keep: std::sync::Arc<Vec<u8>>,
texture: wgpu::Texture,
last_seen: u64,
}
struct VideoFrameTexCache {
entries: std::collections::HashMap<usize, CachedVideoFrame>,
frame: u64,
}
impl VideoFrameTexCache {
fn new() -> Self {
Self { entries: std::collections::HashMap::new(), frame: 0 }
}
fn begin_frame(&mut self) {
self.frame = self.frame.wrapping_add(1);
}
/// View of the cached (or freshly converted+uploaded) texture for `rgba`.
fn texture_view(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
rgba: &std::sync::Arc<Vec<u8>>,
w: u32,
h: u32,
) -> wgpu::TextureView {
let key = std::sync::Arc::as_ptr(rgba) as usize;
let frame = self.frame;
if let Some(e) = self.entries.get_mut(&key) {
e.last_seen = frame;
return e.texture.create_view(&wgpu::TextureViewDescriptor::default());
}
// Miss: upload the raw sRGB straight-alpha bytes verbatim into an sRGB
// texture. The GPU decodes sRGB→linear on sample (free), so there is no
// per-pixel CPU conversion — the cost that used to dominate playback/export.
// Blit this with `blit_straight` (it must NOT unpremultiply).
let texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some("video_frame_tex"),
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8UnormSrgb,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
rgba,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(w * 4),
rows_per_image: Some(h),
},
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
self.entries.insert(key, CachedVideoFrame { _keep: rgba.clone(), texture, last_seen: frame });
view
}
/// Drop textures not used in the last couple of frames (bounds VRAM).
fn evict_stale(&mut self) {
let frame = self.frame;
self.entries.retain(|_, e| e.last_seen + 2 >= frame);
}
}
/// Shared Vello resources (created once, reused by all Stage panes) /// Shared Vello resources (created once, reused by all Stage panes)
struct SharedVelloResources { struct SharedVelloResources {
renderer: Arc<Mutex<vello::Renderer>>, renderer: Arc<Mutex<vello::Renderer>>,
@ -72,6 +155,11 @@ struct SharedVelloResources {
/// True when Vello is running its CPU software renderer (either forced or GPU fallback). /// True when Vello is running its CPU software renderer (either forced or GPU fallback).
/// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area. /// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area.
is_cpu_renderer: bool, is_cpu_renderer: bool,
/// GPU timestamp timer for the composite section (F3 debug). Lazily created on
/// the first frame (needs the device/queue); `None` if unsupported.
gpu_timer: Mutex<Option<crate::gpu_timer::FrameGpuTimer>>,
/// Per-frame video texture cache (skips re-converting/uploading a static frame).
video_frame_cache: Mutex<VideoFrameTexCache>,
} }
/// Per-instance Vello resources (created for each Stage pane) /// Per-instance Vello resources (created for each Stage pane)
@ -302,6 +390,8 @@ impl SharedVelloResources {
gpu_brush: Mutex::new(gpu_brush), gpu_brush: Mutex::new(gpu_brush),
canvas_blit, canvas_blit,
is_cpu_renderer: use_cpu || is_cpu_renderer, is_cpu_renderer: use_cpu || is_cpu_renderer,
gpu_timer: Mutex::new(None),
video_frame_cache: Mutex::new(VideoFrameTexCache::new()),
}) })
} }
} }
@ -1079,6 +1169,25 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
// HDR buffer spec for linear buffers // HDR buffer spec for linear buffers
let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float); let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float);
// F3: bracket the composite section with a GPU timestamp (lazily create the
// timer; no-op when the adapter lacks timestamp support).
let ts_supported = device
.features()
.contains(crate::gpu_timer::FrameGpuTimer::required_features());
{
let mut tg = shared.gpu_timer.lock().unwrap();
if tg.is_none() && ts_supported {
*tg = crate::gpu_timer::FrameGpuTimer::new(device, queue);
}
if let Some(t) = tg.as_ref() {
t.start(device, queue);
}
}
shared.video_frame_cache.lock().unwrap().begin_frame();
// F3: CPU breakdown of the composite (the GPU idles waiting on these).
let mut cput = crate::debug_overlay::CompositeCpuBreakdown::default();
// First, render background and composite it // First, render background and composite it
// The background scene contains only a rectangle at document bounds, // The background scene contains only a rectangle at document bounds,
// so we use TRANSPARENT base_color to not fill the whole viewport // so we use TRANSPARENT base_color to not fill the whole viewport
@ -1097,6 +1206,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
antialiasing_method: aa_method, antialiasing_method: aa_method,
}; };
let _t = std::time::Instant::now();
if let Some(pixmap) = &composite_result.background_cpu { if let Some(pixmap) = &composite_result.background_cpu {
if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) { if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) {
upload_pixmap_to_texture(queue, tex, pixmap); upload_pixmap_to_texture(queue, tex, pixmap);
@ -1104,13 +1214,18 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
} else if let Ok(mut renderer) = shared.renderer.lock() { } else if let Ok(mut renderer) = shared.renderer.lock() {
renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok(); renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok();
} }
cput.vello_ms += _t.elapsed().as_secs_f64() * 1000.0;
// Convert sRGB to linear HDR // Convert sRGB to linear HDR
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("bg_srgb_to_linear_encoder"), label: Some("bg_srgb_to_linear_encoder"),
}); });
let _t = std::time::Instant::now();
shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view); shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view);
cput.convert_ms += _t.elapsed().as_secs_f64() * 1000.0;
let _t = std::time::Instant::now();
queue.submit(Some(convert_encoder.finish())); queue.submit(Some(convert_encoder.finish()));
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
// Composite background onto HDR texture (first layer, clears to dark gray for stage area) // Composite background onto HDR texture (first layer, clears to dark gray for stage area)
let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0); let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0);
@ -1120,6 +1235,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
// Clear to dark gray (stage background outside document bounds) // Clear to dark gray (stage background outside document bounds)
// Note: stage_bg values are already in linear space for HDR compositing // Note: stage_bg values are already in linear space for HDR compositing
let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0]; let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0];
let _t = std::time::Instant::now();
shared.compositor.composite( shared.compositor.composite(
device, device,
queue, queue,
@ -1129,7 +1245,10 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
hdr_view, hdr_view,
Some(stage_bg), Some(stage_bg),
); );
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
let _t = std::time::Instant::now();
queue.submit(Some(encoder.finish())); queue.submit(Some(encoder.finish()));
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
} }
buffer_pool.release(bg_srgb_handle); buffer_pool.release(bg_srgb_handle);
buffer_pool.release(bg_hdr_handle); buffer_pool.release(bg_hdr_handle);
@ -1351,6 +1470,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
buffer_pool.get_view(hdr_layer_handle), buffer_pool.get_view(hdr_layer_handle),
&instance_resources.hdr_texture_view, &instance_resources.hdr_texture_view,
) { ) {
let _t_vello = std::time::Instant::now();
if let Some(pixmap) = &rendered_layer.cpu_pixmap { if let Some(pixmap) = &rendered_layer.cpu_pixmap {
if let Some(tex) = buffer_pool.get_texture(srgb_handle) { if let Some(tex) = buffer_pool.get_texture(srgb_handle) {
upload_pixmap_to_texture(queue, tex, pixmap); upload_pixmap_to_texture(queue, tex, pixmap);
@ -1358,6 +1478,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
} else if let Ok(mut renderer) = shared.renderer.lock() { } else if let Ok(mut renderer) = shared.renderer.lock() {
renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok(); renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok();
} }
cput.vello_ms += _t_vello.elapsed().as_secs_f64() * 1000.0;
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("layer_srgb_to_linear_encoder"), label: Some("layer_srgb_to_linear_encoder"),
}); });
@ -1555,7 +1676,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
} }
} }
RenderedLayerType::Video { instances } => { RenderedLayerType::Video { instances } => {
// Video layer — per-instance: upload decoded frame → blit → composite. // Video layer — per-instance: (cached) frame texture → blit → composite.
for inst in instances { for inst in instances {
if inst.rgba_data.is_empty() { continue; } if inst.rgba_data.is_empty() { continue; }
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
@ -1563,40 +1684,20 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
buffer_pool.get_view(hdr_layer_handle), buffer_pool.get_view(hdr_layer_handle),
&instance_resources.hdr_texture_view, &instance_resources.hdr_texture_view,
) { ) {
// Convert sRGB straight-alpha → linear premultiplied. // Reuse the GPU texture for this frame if it's unchanged (a
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| { // static/paused video → no CPU conversion, alloc, or upload).
let a = p[3] as f32 / 255.0; // Timed into `blit_ms` (incl the cache lookup + per-frame view).
let lin = |c: u8| -> f32 { let _t = std::time::Instant::now();
let f = c as f32 / 255.0; let tex_view = shared
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) } .video_frame_cache
}; .lock()
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8; .unwrap()
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8; .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
[r, g, b, p[3]]
}).collect();
let tex = device.create_texture(&wgpu::TextureDescriptor {
label: Some("video_frame_tex"),
size: wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
mip_level_count: 1, sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rgba8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
queue.write_texture(
wgpu::TexelCopyTextureInfo { texture: &tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
&linear,
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(inst.width * 4), rows_per_image: Some(inst.height) },
wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
);
let tex_view = tex.create_view(&wgpu::TextureViewDescriptor::default());
let bt = crate::gpu_brush::BlitTransform::new( let bt = crate::gpu_brush::BlitTransform::new(
inst.transform, inst.width, inst.height, width, height, inst.transform, inst.width, inst.height, width, height,
); );
shared.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None); shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0;
let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new( let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(
hdr_layer_handle, hdr_layer_handle,
@ -1606,10 +1707,14 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("video_composite_encoder"), label: Some("video_composite_encoder"),
}); });
let _t = std::time::Instant::now();
shared.compositor.composite( shared.compositor.composite(
device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None, device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None,
); );
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
let _t = std::time::Instant::now();
queue.submit(Some(encoder.finish())); queue.submit(Some(encoder.finish()));
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
} }
buffer_pool.release(hdr_layer_handle); buffer_pool.release(hdr_layer_handle);
} }
@ -1841,6 +1946,23 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
buffer_pool.next_frame(); buffer_pool.next_frame();
drop(buffer_pool); drop(buffer_pool);
// F3: close the GPU timestamp bracket + publish the composite measurement.
{
let layers = composite_result.layers.len() as u32;
// Submits aren't counted per-site; estimate from the per-layer pattern
// (bg ~3 + ~2 per layer). Drops toward ~1 once the passes are batched.
let submits_est = 3 + 2 * layers;
let mut tg = shared.gpu_timer.lock().unwrap();
if let Some(t) = tg.as_mut() {
t.end(device, queue);
crate::debug_overlay::update_gpu_composite(true, t.last_ms(), layers, submits_est);
} else {
crate::debug_overlay::update_gpu_composite(false, 0.0, layers, submits_est);
}
}
crate::debug_overlay::update_composite_cpu(cput);
shared.video_frame_cache.lock().unwrap().evict_stale();
// --- Frame timing report --- // --- Frame timing report ---
let _t_end = std::time::Instant::now(); let _t_end = std::time::Instant::now();
let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0; let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0;