diff --git a/lightningbeam-ui/lightningbeam-core/src/renderer.rs b/lightningbeam-ui/lightningbeam-core/src/renderer.rs index 7186606..4ee00bd 100644 --- a/lightningbeam-ui/lightningbeam-core/src/renderer.rs +++ b/lightningbeam-ui/lightningbeam-core/src/renderer.rs @@ -232,6 +232,21 @@ pub struct VideoRenderInstance { pub opacity: f32, } +/// Sink for pulling video frames out of a container layer's scene recursion, so +/// they composite via the fast GPU Video path instead of baking into Vello. +/// +/// Threaded as `Option<&mut VideoExtract>` through the isolated-render scene +/// functions. When present, [`render_video_layer`] pushes a [`VideoRenderInstance`] +/// instead of drawing into the Vello scene. `drew_other` is set whenever any +/// non-video primitive (vector graph, image asset, raster) is emitted; that forces +/// the safe Vello fallback because the container mixes video with other content +/// and we can't preserve z-order by extraction alone. +#[derive(Default)] +struct VideoExtract { + instances: Vec, + drew_other: bool, +} + /// Type of rendered layer for compositor handling pub enum RenderedLayerType { /// Vector / group layer — Vello scene in `RenderedLayer::scene` is used. @@ -428,6 +443,30 @@ pub fn render_document_for_compositing( } } + // One-shot diagnostic: dump what the compositor actually receives. Set + // LB_LAYER_DEBUG=1 to print a single snapshot of each layer's resolved type. + if std::env::var("LB_LAYER_DEBUG").is_ok() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + eprintln!("[LB_LAYER_DEBUG] composite layers = {}", rendered_layers.len()); + for (i, l) in rendered_layers.iter().enumerate() { + let desc = match &l.layer_type { + RenderedLayerType::Vector => + format!("Vector (has_content={}, scene_empty={})", l.has_content, l.cpu_pixmap.is_none()), + RenderedLayerType::Raster { width, height, dirty, .. } => + format!("Raster {width}x{height} dirty={dirty}"), + RenderedLayerType::Video { instances } => + format!("Video ({} instance(s))", instances.len()), + RenderedLayerType::Float { width, height, .. } => + format!("Float {width}x{height}"), + RenderedLayerType::Effect { effect_instances } => + format!("Effect ({} instance(s))", effect_instances.len()), + }; + eprintln!("[LB_LAYER_DEBUG] layer[{i}] id={} type={desc}", l.layer_id); + } + }); + } + CompositeRenderResult { background, background_cpu: None, @@ -462,6 +501,9 @@ pub fn render_layer_isolated( // Render layer content with full opacity (1.0) - opacity applied during compositing match layer { AnyLayer::Vector(vector_layer) => { + // Render into the scene with an extraction sink so a clip that is purely + // video (no vector geometry) composites via the fast GPU Video path. + let mut ex = VideoExtract::default(); render_vector_layer_to_scene( document, time, @@ -471,10 +513,28 @@ pub fn render_layer_isolated( 1.0, // Full opacity - layer opacity handled in compositing image_cache, video_manager, + Some(&mut ex), ); - rendered.has_content = vector_layer.graph_at_time(time) - .map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted)) - || !vector_layer.clip_instances.is_empty(); + if !ex.instances.is_empty() && !ex.drew_other { + // Pure video: discard the (now-empty) scene and emit GPU instances. + rendered.scene = Scene::new(); + rendered.has_content = true; + rendered.layer_type = RenderedLayerType::Video { instances: ex.instances }; + } else { + if !ex.instances.is_empty() { + // Mixed video + vector: the first pass diverted the video out of + // the scene, so re-render with no sink to bake it back in (correct + // z-order; Vello path). Rare — fast-splitting is deferred. + rendered.scene = Scene::new(); + render_vector_layer_to_scene( + document, time, vector_layer, &mut rendered.scene, + base_transform, 1.0, image_cache, video_manager, None, + ); + } + rendered.has_content = vector_layer.graph_at_time(time) + .map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted)) + || !vector_layer.clip_instances.is_empty(); + } } AnyLayer::Audio(_) => { // Audio layers don't render visually @@ -577,15 +637,34 @@ pub fn render_layer_isolated( return RenderedLayer::effect_layer(layer_id, opacity, active_effects); } AnyLayer::Group(group_layer) => { - // Render each child layer's content into the group's scene + // Render each child into the group's scene with an extraction sink. The + // common imported-video case is a Group[Video, Audio] — audio draws + // nothing, so it's pure video and composites via the GPU Video path. + let mut ex = VideoExtract::default(); for child in &group_layer.children { render_layer( document, time, child, &mut rendered.scene, base_transform, 1.0, // Full opacity - layer opacity handled in compositing - image_cache, video_manager, camera_frame, + image_cache, video_manager, camera_frame, Some(&mut ex), ); } - rendered.has_content = !group_layer.children.is_empty(); + if !ex.instances.is_empty() && !ex.drew_other { + rendered.scene = Scene::new(); + rendered.has_content = true; + rendered.layer_type = RenderedLayerType::Video { instances: ex.instances }; + } else { + if !ex.instances.is_empty() { + // Mixed: re-render with no sink to bake the video back into the scene. + rendered.scene = Scene::new(); + for child in &group_layer.children { + render_layer( + document, time, child, &mut rendered.scene, base_transform, + 1.0, image_cache, video_manager, camera_frame, None, + ); + } + } + rendered.has_content = !group_layer.children.is_empty(); + } } AnyLayer::Raster(raster_layer) => { if let Some(kf) = raster_layer.keyframe_at(time) { @@ -614,6 +693,7 @@ fn render_vector_layer_to_scene( parent_opacity: f64, image_cache: &mut ImageCache, video_manager: &std::sync::Arc>, + extract: Option<&mut VideoExtract>, ) { render_vector_layer( document, @@ -624,6 +704,7 @@ fn render_vector_layer_to_scene( parent_opacity, image_cache, video_manager, + extract, ); } @@ -691,10 +772,10 @@ pub fn render_document_with_transform( for layer in document.visible_layers() { if any_soloed { if layer.soloed() { - render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None); + render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None); } } else { - render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None); + render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None); } } } @@ -755,10 +836,11 @@ fn render_layer( image_cache: &mut ImageCache, video_manager: &std::sync::Arc>, camera_frame: Option<&crate::webcam::CaptureFrame>, + mut extract: Option<&mut VideoExtract>, ) { match layer { AnyLayer::Vector(vector_layer) => { - render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager) + render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager, extract) } AnyLayer::Audio(_) => { // Audio layers don't render visually @@ -766,18 +848,20 @@ fn render_layer( AnyLayer::Video(video_layer) => { let mut video_mgr = video_manager.lock().unwrap(); let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None }; - render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame); + render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame, extract); } AnyLayer::Effect(_) => { // Effect layers are processed during GPU compositing, not rendered to scene } AnyLayer::Group(group_layer) => { - // Render each child layer in the group + // Render each child layer in the group, passing the extract sink down. for child in &group_layer.children { - render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame); + render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame, extract.as_deref_mut()); } } AnyLayer::Raster(raster_layer) => { + // Raster is non-video content — force the Vello fallback if extracting. + if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; } render_raster_layer_to_scene(raster_layer, time, scene, base_transform); } } @@ -816,6 +900,7 @@ pub fn render_single_clip_instance( render_clip_instance( document, time, clip_instance, layer_opacity, scene, base_transform, &vector_layer.layer.animation_data, image_cache, video_manager, group_end_time, + None, // edit-inside-clip overlay keeps the Vello path ); } @@ -831,6 +916,7 @@ fn render_clip_instance( image_cache: &mut ImageCache, video_manager: &std::sync::Arc>, group_end_time: Option, + mut extract: Option<&mut VideoExtract>, ) { // Try to find the clip in the document's clip libraries // For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented) @@ -972,7 +1058,7 @@ fn render_clip_instance( if !layer_node.data.visible() { continue; } - render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None); + render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None, extract.as_deref_mut()); } } @@ -986,6 +1072,7 @@ fn render_video_layer( parent_opacity: f64, video_manager: &mut crate::video::VideoManager, camera_frame: Option<&crate::webcam::CaptureFrame>, + mut extract: Option<&mut VideoExtract>, ) { use crate::animation::TransformProperty; @@ -1151,14 +1238,26 @@ fn render_video_layer( Affine::IDENTITY }; - // Render video frame as image fill - scene.fill( - Fill::NonZero, - instance_transform, - &image_with_alpha, - Some(brush_transform), - &video_rect, - ); + // Extract to the GPU Video path when a sink is present; otherwise bake into + // the Vello scene. The combined frame-pixel → document transform is + // instance_transform * brush_transform (matching the top-level Video path). + if let Some(ex) = extract.as_deref_mut() { + ex.instances.push(VideoRenderInstance { + rgba_data: frame.rgba_data.clone(), + width: frame.width, + height: frame.height, + transform: instance_transform * brush_transform, + opacity: final_opacity, + }); + } else { + scene.fill( + Fill::NonZero, + instance_transform, + &image_with_alpha, + Some(brush_transform), + &video_rect, + ); + } clip_rendered = true; } @@ -1194,13 +1293,25 @@ fn render_video_layer( * Affine::translate((offset_x, offset_y)) * Affine::scale(uniform_scale); - scene.fill( - Fill::NonZero, - preview_transform, - &image_with_alpha, - None, - &frame_rect, - ); + // preview_transform maps frame-pixel space → document directly, so it + // is exactly the instance transform for the GPU path. + if let Some(ex) = extract.as_deref_mut() { + ex.instances.push(VideoRenderInstance { + rgba_data: frame.rgba_data.clone(), + width: frame.width, + height: frame.height, + transform: preview_transform, + opacity: final_opacity, + }); + } else { + scene.fill( + Fill::NonZero, + preview_transform, + &image_with_alpha, + None, + &frame_rect, + ); + } } } } @@ -1352,6 +1463,7 @@ fn render_vector_layer( parent_opacity: f64, image_cache: &mut ImageCache, video_manager: &std::sync::Arc>, + mut extract: Option<&mut VideoExtract>, ) { // Cascade opacity: parent_opacity × layer.opacity let layer_opacity = parent_opacity * layer.layer.opacity; @@ -1359,6 +1471,10 @@ fn render_vector_layer( // Render the layer's own VectorGraph (loose shapes) first, then clip instances // (groups / movie clips) on top. Shape tweens are applied here. if let Some(graph) = layer.tweened_graph_at(time) { + // Loose vector geometry (and any image-asset fills) is non-video content — + // force the Vello fallback. Conservative: a present-but-empty graph still + // trips this, which only costs the fallback, never correctness. + if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; } render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache); } @@ -1370,7 +1486,7 @@ fn render_vector_layer( let frame_duration = 1.0 / document.framerate; layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration) }); - render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time); + render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time, extract.as_deref_mut()); } } diff --git a/lightningbeam-ui/lightningbeam-editor/src/debug_overlay.rs b/lightningbeam-ui/lightningbeam-editor/src/debug_overlay.rs index 1def09e..c84c7fa 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/debug_overlay.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/debug_overlay.rs @@ -39,6 +39,77 @@ pub fn update_prepare_timing( t.composite_ms = composite_ms; } } +/// GPU-measured composite cost (from timestamp queries; see `gpu_timer.rs`). +#[derive(Debug, Clone, Default)] +pub struct GpuCompositeTiming { + /// True when the adapter supports timestamp queries (else the ms is meaningless). + pub supported: bool, + /// GPU time of the whole composite section (Vello render + sRGB→linear + + /// compositor + tonemap), in milliseconds. Read back asynchronously, so it + /// lags the displayed frame by a frame or two. + pub composite_gpu_ms: f64, + /// Layers composited this frame. + pub layers: u32, + /// `queue.submit()` calls in the composite section this frame. + pub submits: u32, +} + +static GPU_COMPOSITE: OnceLock> = OnceLock::new(); + +/// Called from `VelloCallback::prepare()` with the GPU composite measurement. +pub fn update_gpu_composite(supported: bool, composite_gpu_ms: f64, layers: u32, submits: u32) { + let cell = GPU_COMPOSITE.get_or_init(|| Mutex::new(GpuCompositeTiming::default())); + if let Ok(mut t) = cell.lock() { + t.supported = supported; + t.composite_gpu_ms = composite_gpu_ms; + t.layers = layers; + t.submits = submits; + } +} + +fn get_gpu_composite() -> GpuCompositeTiming { + GPU_COMPOSITE + .get_or_init(|| Mutex::new(GpuCompositeTiming::default())) + .lock() + .map(|t| t.clone()) + .unwrap_or_default() +} + +/// CPU-side breakdown of the composite section (wall-clock `Instant` deltas). Since +/// the GPU idles waiting on these CPU operations, this is where the per-frame cost +/// actually lives. Sums should ≈ the CPU `composite_ms` for the doc's active paths. +#[derive(Debug, Clone, Default)] +pub struct CompositeCpuBreakdown { + /// `renderer.render_to_texture` — Vello scene encode + its internal submit. + pub vello_ms: f64, + /// `srgb_to_linear.convert` — recording the conversion pass. + pub convert_ms: f64, + /// `canvas_blit.blit` — recording + its internal submit. + pub blit_ms: f64, + /// `compositor.composite` — recording + per-call uniforms buffer / bind group alloc. + pub composite_ms: f64, + /// Explicit `queue.submit()` calls. + pub submit_ms: f64, +} + +static COMPOSITE_CPU: OnceLock> = OnceLock::new(); + +/// Called from `VelloCallback::prepare()` with the composite CPU breakdown. +pub fn update_composite_cpu(b: CompositeCpuBreakdown) { + let cell = COMPOSITE_CPU.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default())); + if let Ok(mut t) = cell.lock() { + *t = b; + } +} + +fn get_composite_cpu() -> CompositeCpuBreakdown { + COMPOSITE_CPU + .get_or_init(|| Mutex::new(CompositeCpuBreakdown::default())) + .lock() + .map(|t| t.clone()) + .unwrap_or_default() +} + /// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the /// raster-layer texture cache — the only unbounded-by-default VRAM consumer. #[derive(Debug, Clone, Default)] @@ -90,6 +161,12 @@ pub struct DebugStats { // GPU prepare() timing breakdown (from render thread) pub prepare_timing: PrepareTiming, + // GPU-measured composite cost (timestamp queries) + pub gpu_composite: GpuCompositeTiming, + + // CPU breakdown of the composite section + pub composite_cpu: CompositeCpuBreakdown, + // Performance metrics for each section pub timing_memory_us: u64, pub timing_gpu_us: u64, @@ -254,6 +331,8 @@ impl DebugStatsCollector { audio_input_devices, has_pointer, prepare_timing, + gpu_composite: get_gpu_composite(), + composite_cpu: get_composite_cpu(), timing_memory_us, timing_gpu_us, timing_midi_us, @@ -306,8 +385,33 @@ pub fn render_debug_overlay(ctx: &egui::Context, stats: &DebugStats) { ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms)); ui.label(format!(" removals: {:.2} ms", pt.removals_ms)); ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms)); - ui.label(format!(" scene_build: {:.2} ms", pt.scene_build_ms)); - ui.label(format!(" composite: {:.2} ms", pt.composite_ms)); + ui.label(format!(" scene_build: {:.2} ms (CPU)", pt.scene_build_ms)); + ui.label(format!(" composite: {:.2} ms (CPU)", pt.composite_ms)); + + // GPU-measured composite cost (timestamp queries). + let gc = &stats.gpu_composite; + if gc.supported { + ui.colored_label( + egui::Color32::LIGHT_GREEN, + format!("GPU composite: {:.2} ms (GPU)", gc.composite_gpu_ms), + ); + ui.label(format!(" layers: {} submits: {}", gc.layers, gc.submits)); + } else { + ui.label(format!( + "GPU composite: n/a (no timestamp support) layers: {} submits: {}", + gc.layers, gc.submits + )); + } + + // CPU breakdown of the composite (where the GPU is actually waiting). + let cc = &stats.composite_cpu; + let cc_sum = cc.vello_ms + cc.convert_ms + cc.blit_ms + cc.composite_ms + cc.submit_ms; + ui.colored_label(egui::Color32::LIGHT_BLUE, format!("Composite CPU breakdown: {:.2} ms", cc_sum)); + ui.label(format!(" vello(render): {:.2} ms", cc.vello_ms)); + ui.label(format!(" srgb→linear: {:.2} ms", cc.convert_ms)); + ui.label(format!(" blit: {:.2} ms", cc.blit_ms)); + ui.label(format!(" compositor: {:.2} ms", cc.composite_ms)); + ui.label(format!(" queue.submit: {:.2} ms", cc.submit_ms)); ui.add_space(8.0); diff --git a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs index 1d1fdcc..91ac55d 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs @@ -830,22 +830,13 @@ fn composite_document_to_hdr( if inst.rgba_data.is_empty() { continue; } let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { - // sRGB straight-alpha → linear premultiplied - let linear: Vec = inst.rgba_data.chunks_exact(4).flat_map(|p| { - let a = p[3] as f32 / 255.0; - let lin = |c: u8| -> f32 { - let f = c as f32 / 255.0; - if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) } - }; - let r = (lin(p[0]) * a * 255.0 + 0.5) as u8; - let g = (lin(p[1]) * a * 255.0 + 0.5) as u8; - let b = (lin(p[2]) * a * 255.0 + 0.5) as u8; - [r, g, b, p[3]] - }).collect(); - let tex = upload_transient_texture(device, queue, &linear, inst.width, inst.height, Some("export_video_frame_tex")); + // Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU + // decodes to linear on sample (no per-pixel CPU conversion). Blit with + // blit_straight so the shader doesn't unpremultiply. + let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex")); let tex_view = tex.create_view(&Default::default()); let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height); - gpu_resources.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None); + gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal); let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") }); gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None); @@ -865,7 +856,7 @@ fn composite_document_to_hdr( }; [lin(p[0]), lin(p[1]), lin(p[2]), p[3]] }).collect(); - let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, Some("export_float_tex")); + let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, wgpu::TextureFormat::Rgba8Unorm, Some("export_float_tex")); let tex_view = tex.create_view(&Default::default()); let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec); if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) { @@ -919,13 +910,16 @@ fn composite_document_to_hdr( Ok(()) } -/// Upload `pixels` to a transient `Rgba8Unorm` GPU texture (TEXTURE_BINDING | COPY_DST). +/// Upload `pixels` to a transient GPU texture (TEXTURE_BINDING | COPY_DST) in the +/// given format. Use `Rgba8UnormSrgb` to upload raw sRGB bytes and let the GPU +/// decode to linear on sample (no CPU conversion). fn upload_transient_texture( device: &wgpu::Device, queue: &wgpu::Queue, pixels: &[u8], width: u32, height: u32, + format: wgpu::TextureFormat, label: Option<&'static str>, ) -> wgpu::Texture { let tex = device.create_texture(&wgpu::TextureDescriptor { @@ -933,7 +927,7 @@ fn upload_transient_texture( size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 }, mip_level_count: 1, sample_count: 1, dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rgba8Unorm, + format, usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, view_formats: &[], }); diff --git a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs index d2ae716..5cf8421 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/gpu_brush.rs @@ -1951,6 +1951,9 @@ impl GpuBrushEngine { /// the camera transform. pub struct CanvasBlitPipeline { pub pipeline: wgpu::RenderPipeline, + /// Variant for straight-alpha sources (hardware-sRGB video frames): the + /// fragment shader skips the unpremultiply. See [`CanvasBlitPipeline::blit_straight`]. + pub pipeline_straight: wgpu::RenderPipeline, pub bg_layout: wgpu::BindGroupLayout, pub sampler: wgpu::Sampler, /// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res @@ -2132,6 +2135,39 @@ impl CanvasBlitPipeline { }, ); + // Variant pipeline for straight-alpha sources (hardware-sRGB video frames): + // identical except the fragment shader skips the unpremultiply. + let pipeline_straight = device.create_render_pipeline( + &wgpu::RenderPipelineDescriptor { + label: Some("canvas_blit_pipeline_straight"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main_straight"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba16Float, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }, + ); + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { label: Some("canvas_blit_sampler"), address_mode_u: wgpu::AddressMode::ClampToEdge, @@ -2165,7 +2201,7 @@ impl CanvasBlitPipeline { ..Default::default() }); - Self { pipeline, bg_layout, sampler, linear_sampler, mask_sampler } + Self { pipeline, pipeline_straight, bg_layout, sampler, linear_sampler, mask_sampler } } /// Render the canvas texture into `target_view` (Rgba16Float) with the given camera. @@ -2183,7 +2219,7 @@ impl CanvasBlitPipeline { transform: &BlitTransform, mask_view: Option<&wgpu::TextureView>, ) { - self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler); + self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline); } /// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies). @@ -2196,9 +2232,25 @@ impl CanvasBlitPipeline { transform: &BlitTransform, mask_view: Option<&wgpu::TextureView>, ) { - self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler); + self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler, &self.pipeline); } + /// Blit a **straight-alpha** source (e.g. a video frame uploaded to an + /// `Rgba8UnormSrgb` texture, hardware-decoded to linear on sample). Uses the + /// `fs_main_straight` pipeline, which skips the unpremultiply that `blit` does. + pub fn blit_straight( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + canvas_view: &wgpu::TextureView, + target_view: &wgpu::TextureView, + transform: &BlitTransform, + mask_view: Option<&wgpu::TextureView>, + ) { + self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline_straight); + } + + #[allow(clippy::too_many_arguments)] fn blit_with( &self, device: &wgpu::Device, @@ -2208,6 +2260,7 @@ impl CanvasBlitPipeline { transform: &BlitTransform, mask_view: Option<&wgpu::TextureView>, canvas_sampler: &wgpu::Sampler, + pipeline: &wgpu::RenderPipeline, ) { // When no mask is provided, create a temporary 1×1 all-white texture. // (queue is already available here, unlike in new()) @@ -2296,7 +2349,7 @@ impl CanvasBlitPipeline { occlusion_query_set: None, timestamp_writes: None, }); - rp.set_pipeline(&self.pipeline); + rp.set_pipeline(pipeline); rp.set_bind_group(0, &bg, &[]); rp.draw(0..4, 0..1); } diff --git a/lightningbeam-ui/lightningbeam-editor/src/gpu_timer.rs b/lightningbeam-ui/lightningbeam-editor/src/gpu_timer.rs new file mode 100644 index 0000000..e136a03 --- /dev/null +++ b/lightningbeam-ui/lightningbeam-editor/src/gpu_timer.rs @@ -0,0 +1,135 @@ +//! Minimal GPU timestamp timer for the composite pipeline. +//! +//! Brackets a section of GPU work with two timestamps and reads the elapsed GPU +//! time back asynchronously (no pipeline stall). Used to attribute the per-frame +//! composite cost (Vello render + sRGB→linear + compositor + tonemap) shown in F3. +//! +//! Requires `TIMESTAMP_QUERY` + `TIMESTAMP_QUERY_INSIDE_ENCODERS`; [`FrameGpuTimer::new`] +//! returns `None` when the adapter doesn't support them, and all call sites no-op. + +use std::sync::{Arc, Mutex}; + +/// State of the single readback buffer (shared with the map callback). +#[derive(Clone, Copy, PartialEq)] +enum Readback { + /// Available to resolve into this frame. + Free, + /// Submitted + `map_async` in flight; don't touch until the callback fires. + Mapping, + /// Mapped and ready to read. + Ready, +} + +/// Times one GPU section (two timestamps) per frame with intermittent async readback. +pub struct FrameGpuTimer { + query_set: wgpu::QuerySet, + resolve_buf: wgpu::Buffer, + readback_buf: wgpu::Buffer, + state: Arc>, + /// Nanoseconds per timestamp tick. + period_ns: f32, + /// Most recent measured GPU time for the bracketed section, in milliseconds. + last_ms: f64, +} + +impl FrameGpuTimer { + /// Required device features for GPU timestamp timing. + pub fn required_features() -> wgpu::Features { + wgpu::Features::TIMESTAMP_QUERY | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS + } + + /// Create a timer, or `None` if the device lacks timestamp support. + pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option { + if !device.features().contains(Self::required_features()) { + return None; + } + let query_set = device.create_query_set(&wgpu::QuerySetDescriptor { + label: Some("composite_gpu_timer"), + ty: wgpu::QueryType::Timestamp, + count: 2, + }); + // 2 timestamps × u64. + let size = 2 * std::mem::size_of::() as u64; + let resolve_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("composite_gpu_timer_resolve"), + size, + usage: wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC, + mapped_at_creation: false, + }); + let readback_buf = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("composite_gpu_timer_readback"), + size, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + Some(Self { + query_set, + resolve_buf, + readback_buf, + state: Arc::new(Mutex::new(Readback::Free)), + period_ns: queue.get_timestamp_period(), + last_ms: 0.0, + }) + } + + /// Write the **start** timestamp (call just before the bracketed GPU work). + pub fn start(&self, device: &wgpu::Device, queue: &wgpu::Queue) { + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("composite_gpu_timer_start"), + }); + enc.write_timestamp(&self.query_set, 0); + queue.submit(Some(enc.finish())); + } + + /// Write the **end** timestamp and, if the readback buffer is free, resolve + + /// kick off an async read. Also consumes a previously-completed read into + /// `last_ms`. Call just after the bracketed GPU work. + pub fn end(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) { + // 1. Consume a completed readback first (so the buffer is free to reuse). + let cur = *self.state.lock().unwrap(); + if cur == Readback::Ready { + { + let view = self.readback_buf.slice(..).get_mapped_range(); + let t0 = u64::from_le_bytes(view[0..8].try_into().unwrap()); + let t1 = u64::from_le_bytes(view[8..16].try_into().unwrap()); + // Timestamps can wrap or arrive out of order across queue resets; guard. + let ticks = t1.saturating_sub(t0); + self.last_ms = ticks as f64 * self.period_ns as f64 / 1.0e6; + } + self.readback_buf.unmap(); + *self.state.lock().unwrap() = Readback::Free; + } + + // 2. End timestamp + resolve + copy, only when the buffer is free. + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("composite_gpu_timer_end"), + }); + enc.write_timestamp(&self.query_set, 1); + + let can_read = *self.state.lock().unwrap() == Readback::Free; + if can_read { + enc.resolve_query_set(&self.query_set, 0..2, &self.resolve_buf, 0); + enc.copy_buffer_to_buffer( + &self.resolve_buf, + 0, + &self.readback_buf, + 0, + 2 * std::mem::size_of::() as u64, + ); + } + queue.submit(Some(enc.finish())); + + if can_read { + *self.state.lock().unwrap() = Readback::Mapping; + let state = Arc::clone(&self.state); + self.readback_buf.slice(..).map_async(wgpu::MapMode::Read, move |res| { + *state.lock().unwrap() = if res.is_ok() { Readback::Ready } else { Readback::Free }; + }); + } + } + + /// Most recently measured GPU time of the bracketed section, in milliseconds. + pub fn last_ms(&self) -> f64 { + self.last_ms + } +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/main.rs b/lightningbeam-ui/lightningbeam-editor/src/main.rs index b01ba6c..fade9aa 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/main.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/main.rs @@ -50,6 +50,7 @@ use effect_thumbnails::EffectThumbnailGenerator; mod custom_cursor; mod tablet; mod debug_overlay; +mod gpu_timer; #[cfg(debug_assertions)] mod test_mode; @@ -174,8 +175,12 @@ fn main() -> eframe::Result { device_descriptor: std::sync::Arc::new(|adapter| { let features = adapter.features(); // Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's - // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability) - let optional_features = wgpu::Features::SHADER_F16; + // unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability). + // TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer + // (gpu_timer.rs); both are optional and no-op when unsupported. + let optional_features = wgpu::Features::SHADER_F16 + | wgpu::Features::TIMESTAMP_QUERY + | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS; let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl { wgpu::Limits::downlevel_webgl2_defaults() diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/canvas_blit.wgsl b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/canvas_blit.wgsl index e6e1db9..94c0d41 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/canvas_blit.wgsl +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/shaders/canvas_blit.wgsl @@ -75,3 +75,25 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4 { let tinted = base + tint - base * tint; return vec4(tinted, masked_a); } + +// Variant for sources that are ALREADY straight-alpha linear — notably a video +// frame uploaded to an `Rgba8UnormSrgb` texture, where the hardware decodes +// sRGB→linear on sample and leaves alpha untouched. No unpremultiply (the source +// was never premultiplied), so we skip the divide entirely. The compositor wants +// straight-alpha linear, which is exactly what the sample already is. +@fragment +fn fs_main_straight(in: VertexOutput) -> @location(0) vec4 { + let m = mat3x3(transform.col0.xyz, transform.col1.xyz, transform.col2.xyz); + let canvas_uv = (m * vec3(in.uv.x, in.uv.y, 1.0)).xy; + + if canvas_uv.x < 0.0 || canvas_uv.x > 1.0 + || canvas_uv.y < 0.0 || canvas_uv.y > 1.0 { + return vec4(0.0, 0.0, 0.0, 0.0); + } + + let c = textureSample(canvas_tex, canvas_sampler, canvas_uv); + let mask = textureSample(mask_tex, mask_sampler, canvas_uv).r; + let tint = vec3(transform.col0.w, transform.col1.w, transform.col2.w); + let tinted = c.rgb + tint - c.rgb * tint; + return vec4(tinted, c.a * mask); +} diff --git a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs index c28e116..5bd3181 100644 --- a/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs +++ b/lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs @@ -45,6 +45,89 @@ fn upload_pixmap_to_texture(queue: &wgpu::Queue, texture: &wgpu::Texture, pixmap /// Set to true to use the new pipeline, false for legacy single-scene rendering const USE_HDR_COMPOSITING: bool = true; // Enabled for testing +/// Caches GPU textures for decoded video frames, keyed by the frame buffer's `Arc` +/// identity. A static/paused video then costs ~nothing per repaint (cache hit → no +/// per-pixel CPU sRGB→linear conversion, no texture allocation, no upload). The +/// cached texture holds premultiplied-linear RGBA8 — exactly what `canvas_blit` +/// expects. During playback each new decoded frame is a fresh `Arc` → one +/// conversion+upload per frame (not per repaint). +struct CachedVideoFrame { + /// Keep the source buffer alive so its pointer (our cache key) can't be reused. + _keep: std::sync::Arc>, + texture: wgpu::Texture, + last_seen: u64, +} + +struct VideoFrameTexCache { + entries: std::collections::HashMap, + frame: u64, +} + +impl VideoFrameTexCache { + fn new() -> Self { + Self { entries: std::collections::HashMap::new(), frame: 0 } + } + + fn begin_frame(&mut self) { + self.frame = self.frame.wrapping_add(1); + } + + /// View of the cached (or freshly converted+uploaded) texture for `rgba`. + fn texture_view( + &mut self, + device: &wgpu::Device, + queue: &wgpu::Queue, + rgba: &std::sync::Arc>, + w: u32, + h: u32, + ) -> wgpu::TextureView { + let key = std::sync::Arc::as_ptr(rgba) as usize; + let frame = self.frame; + if let Some(e) = self.entries.get_mut(&key) { + e.last_seen = frame; + return e.texture.create_view(&wgpu::TextureViewDescriptor::default()); + } + // Miss: upload the raw sRGB straight-alpha bytes verbatim into an sRGB + // texture. The GPU decodes sRGB→linear on sample (free), so there is no + // per-pixel CPU conversion — the cost that used to dominate playback/export. + // Blit this with `blit_straight` (it must NOT unpremultiply). + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("video_frame_tex"), + size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8UnormSrgb, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + rgba, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(w * 4), + rows_per_image: Some(h), + }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + self.entries.insert(key, CachedVideoFrame { _keep: rgba.clone(), texture, last_seen: frame }); + view + } + + /// Drop textures not used in the last couple of frames (bounds VRAM). + fn evict_stale(&mut self) { + let frame = self.frame; + self.entries.retain(|_, e| e.last_seen + 2 >= frame); + } +} + /// Shared Vello resources (created once, reused by all Stage panes) struct SharedVelloResources { renderer: Arc>, @@ -72,6 +155,11 @@ struct SharedVelloResources { /// True when Vello is running its CPU software renderer (either forced or GPU fallback). /// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area. is_cpu_renderer: bool, + /// GPU timestamp timer for the composite section (F3 debug). Lazily created on + /// the first frame (needs the device/queue); `None` if unsupported. + gpu_timer: Mutex>, + /// Per-frame video texture cache (skips re-converting/uploading a static frame). + video_frame_cache: Mutex, } /// Per-instance Vello resources (created for each Stage pane) @@ -302,6 +390,8 @@ impl SharedVelloResources { gpu_brush: Mutex::new(gpu_brush), canvas_blit, is_cpu_renderer: use_cpu || is_cpu_renderer, + gpu_timer: Mutex::new(None), + video_frame_cache: Mutex::new(VideoFrameTexCache::new()), }) } } @@ -1079,6 +1169,25 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // HDR buffer spec for linear buffers let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float); + // F3: bracket the composite section with a GPU timestamp (lazily create the + // timer; no-op when the adapter lacks timestamp support). + let ts_supported = device + .features() + .contains(crate::gpu_timer::FrameGpuTimer::required_features()); + { + let mut tg = shared.gpu_timer.lock().unwrap(); + if tg.is_none() && ts_supported { + *tg = crate::gpu_timer::FrameGpuTimer::new(device, queue); + } + if let Some(t) = tg.as_ref() { + t.start(device, queue); + } + } + shared.video_frame_cache.lock().unwrap().begin_frame(); + + // F3: CPU breakdown of the composite (the GPU idles waiting on these). + let mut cput = crate::debug_overlay::CompositeCpuBreakdown::default(); + // First, render background and composite it // The background scene contains only a rectangle at document bounds, // so we use TRANSPARENT base_color to not fill the whole viewport @@ -1097,6 +1206,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback { antialiasing_method: aa_method, }; + let _t = std::time::Instant::now(); if let Some(pixmap) = &composite_result.background_cpu { if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) { upload_pixmap_to_texture(queue, tex, pixmap); @@ -1104,13 +1214,18 @@ impl egui_wgpu::CallbackTrait for VelloCallback { } else if let Ok(mut renderer) = shared.renderer.lock() { renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok(); } + cput.vello_ms += _t.elapsed().as_secs_f64() * 1000.0; // Convert sRGB to linear HDR let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("bg_srgb_to_linear_encoder"), }); + let _t = std::time::Instant::now(); shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view); + cput.convert_ms += _t.elapsed().as_secs_f64() * 1000.0; + let _t = std::time::Instant::now(); queue.submit(Some(convert_encoder.finish())); + cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0; // Composite background onto HDR texture (first layer, clears to dark gray for stage area) let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0); @@ -1120,6 +1235,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback { // Clear to dark gray (stage background outside document bounds) // Note: stage_bg values are already in linear space for HDR compositing let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0]; + let _t = std::time::Instant::now(); shared.compositor.composite( device, queue, @@ -1129,7 +1245,10 @@ impl egui_wgpu::CallbackTrait for VelloCallback { hdr_view, Some(stage_bg), ); + cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0; + let _t = std::time::Instant::now(); queue.submit(Some(encoder.finish())); + cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0; } buffer_pool.release(bg_srgb_handle); buffer_pool.release(bg_hdr_handle); @@ -1351,6 +1470,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback { buffer_pool.get_view(hdr_layer_handle), &instance_resources.hdr_texture_view, ) { + let _t_vello = std::time::Instant::now(); if let Some(pixmap) = &rendered_layer.cpu_pixmap { if let Some(tex) = buffer_pool.get_texture(srgb_handle) { upload_pixmap_to_texture(queue, tex, pixmap); @@ -1358,6 +1478,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback { } else if let Ok(mut renderer) = shared.renderer.lock() { renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok(); } + cput.vello_ms += _t_vello.elapsed().as_secs_f64() * 1000.0; let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("layer_srgb_to_linear_encoder"), }); @@ -1555,7 +1676,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback { } } RenderedLayerType::Video { instances } => { - // Video layer — per-instance: upload decoded frame → blit → composite. + // Video layer — per-instance: (cached) frame texture → blit → composite. for inst in instances { if inst.rgba_data.is_empty() { continue; } let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); @@ -1563,40 +1684,20 @@ impl egui_wgpu::CallbackTrait for VelloCallback { buffer_pool.get_view(hdr_layer_handle), &instance_resources.hdr_texture_view, ) { - // Convert sRGB straight-alpha → linear premultiplied. - let linear: Vec = inst.rgba_data.chunks_exact(4).flat_map(|p| { - let a = p[3] as f32 / 255.0; - let lin = |c: u8| -> f32 { - let f = c as f32 / 255.0; - if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) } - }; - let r = (lin(p[0]) * a * 255.0 + 0.5) as u8; - let g = (lin(p[1]) * a * 255.0 + 0.5) as u8; - let b = (lin(p[2]) * a * 255.0 + 0.5) as u8; - [r, g, b, p[3]] - }).collect(); - - let tex = device.create_texture(&wgpu::TextureDescriptor { - label: Some("video_frame_tex"), - size: wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 }, - mip_level_count: 1, sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: wgpu::TextureFormat::Rgba8Unorm, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); - queue.write_texture( - wgpu::TexelCopyTextureInfo { texture: &tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, - &linear, - wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(inst.width * 4), rows_per_image: Some(inst.height) }, - wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 }, - ); - let tex_view = tex.create_view(&wgpu::TextureViewDescriptor::default()); - + // Reuse the GPU texture for this frame if it's unchanged (a + // static/paused video → no CPU conversion, alloc, or upload). + // Timed into `blit_ms` (incl the cache lookup + per-frame view). + let _t = std::time::Instant::now(); + let tex_view = shared + .video_frame_cache + .lock() + .unwrap() + .texture_view(device, queue, &inst.rgba_data, inst.width, inst.height); let bt = crate::gpu_brush::BlitTransform::new( inst.transform, inst.width, inst.height, width, height, ); - shared.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None); + shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None); + cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0; let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new( hdr_layer_handle, @@ -1606,10 +1707,14 @@ impl egui_wgpu::CallbackTrait for VelloCallback { let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("video_composite_encoder"), }); + let _t = std::time::Instant::now(); shared.compositor.composite( device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None, ); + cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0; + let _t = std::time::Instant::now(); queue.submit(Some(encoder.finish())); + cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0; } buffer_pool.release(hdr_layer_handle); } @@ -1841,6 +1946,23 @@ impl egui_wgpu::CallbackTrait for VelloCallback { buffer_pool.next_frame(); drop(buffer_pool); + // F3: close the GPU timestamp bracket + publish the composite measurement. + { + let layers = composite_result.layers.len() as u32; + // Submits aren't counted per-site; estimate from the per-layer pattern + // (bg ~3 + ~2 per layer). Drops toward ~1 once the passes are batched. + let submits_est = 3 + 2 * layers; + let mut tg = shared.gpu_timer.lock().unwrap(); + if let Some(t) = tg.as_mut() { + t.end(device, queue); + crate::debug_overlay::update_gpu_composite(true, t.last_ms(), layers, submits_est); + } else { + crate::debug_overlay::update_gpu_composite(false, 0.0, layers, submits_est); + } + } + crate::debug_overlay::update_composite_cpu(cput); + shared.video_frame_cache.lock().unwrap().evict_stale(); + // --- Frame timing report --- let _t_end = std::time::Instant::now(); let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0;