Composite grouped/nested video on the GPU path
Imported video is a Group[Video, Audio] that rendered as a Vello-baked Vector layer, re-uploading the full frame to Vello's image atlas every frame (~17ms/frame at 1080p, hitting playback and export alike). Extract video frames out of the Group/clip scene recursion into VideoRenderInstances so they composite via the GPU Video path; mixed video+vector containers fall back to Vello (correct, unaccelerated). Also route video through hardware sRGB decode: upload raw sRGB bytes to an Rgba8UnormSrgb texture and blit with a non-unpremultiplying shader variant (blit_straight), removing the per-frame per-pixel CPU sRGB->linear pass. Add an F3 GPU-timestamp timer and a per-frame video texture cache. Drops the live composite of a 1080p video from ~17ms to ~2-3ms. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Skyler Lehmkuhl
parent
ce151ffd61
commit
5844a0f070
|
|
@ -232,6 +232,21 @@ pub struct VideoRenderInstance {
|
|||
pub opacity: f32,
|
||||
}
|
||||
|
||||
/// Sink for pulling video frames out of a container layer's scene recursion, so
|
||||
/// they composite via the fast GPU Video path instead of baking into Vello.
|
||||
///
|
||||
/// Threaded as `Option<&mut VideoExtract>` through the isolated-render scene
|
||||
/// functions. When present, [`render_video_layer`] pushes a [`VideoRenderInstance`]
|
||||
/// instead of drawing into the Vello scene. `drew_other` is set whenever any
|
||||
/// non-video primitive (vector graph, image asset, raster) is emitted; that forces
|
||||
/// the safe Vello fallback because the container mixes video with other content
|
||||
/// and we can't preserve z-order by extraction alone.
|
||||
#[derive(Default)]
|
||||
struct VideoExtract {
|
||||
instances: Vec<VideoRenderInstance>,
|
||||
drew_other: bool,
|
||||
}
|
||||
|
||||
/// Type of rendered layer for compositor handling
|
||||
pub enum RenderedLayerType {
|
||||
/// Vector / group layer — Vello scene in `RenderedLayer::scene` is used.
|
||||
|
|
@ -428,6 +443,30 @@ pub fn render_document_for_compositing(
|
|||
}
|
||||
}
|
||||
|
||||
// One-shot diagnostic: dump what the compositor actually receives. Set
|
||||
// LB_LAYER_DEBUG=1 to print a single snapshot of each layer's resolved type.
|
||||
if std::env::var("LB_LAYER_DEBUG").is_ok() {
|
||||
static ONCE: std::sync::Once = std::sync::Once::new();
|
||||
ONCE.call_once(|| {
|
||||
eprintln!("[LB_LAYER_DEBUG] composite layers = {}", rendered_layers.len());
|
||||
for (i, l) in rendered_layers.iter().enumerate() {
|
||||
let desc = match &l.layer_type {
|
||||
RenderedLayerType::Vector =>
|
||||
format!("Vector (has_content={}, scene_empty={})", l.has_content, l.cpu_pixmap.is_none()),
|
||||
RenderedLayerType::Raster { width, height, dirty, .. } =>
|
||||
format!("Raster {width}x{height} dirty={dirty}"),
|
||||
RenderedLayerType::Video { instances } =>
|
||||
format!("Video ({} instance(s))", instances.len()),
|
||||
RenderedLayerType::Float { width, height, .. } =>
|
||||
format!("Float {width}x{height}"),
|
||||
RenderedLayerType::Effect { effect_instances } =>
|
||||
format!("Effect ({} instance(s))", effect_instances.len()),
|
||||
};
|
||||
eprintln!("[LB_LAYER_DEBUG] layer[{i}] id={} type={desc}", l.layer_id);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
CompositeRenderResult {
|
||||
background,
|
||||
background_cpu: None,
|
||||
|
|
@ -462,6 +501,9 @@ pub fn render_layer_isolated(
|
|||
// Render layer content with full opacity (1.0) - opacity applied during compositing
|
||||
match layer {
|
||||
AnyLayer::Vector(vector_layer) => {
|
||||
// Render into the scene with an extraction sink so a clip that is purely
|
||||
// video (no vector geometry) composites via the fast GPU Video path.
|
||||
let mut ex = VideoExtract::default();
|
||||
render_vector_layer_to_scene(
|
||||
document,
|
||||
time,
|
||||
|
|
@ -471,10 +513,28 @@ pub fn render_layer_isolated(
|
|||
1.0, // Full opacity - layer opacity handled in compositing
|
||||
image_cache,
|
||||
video_manager,
|
||||
Some(&mut ex),
|
||||
);
|
||||
rendered.has_content = vector_layer.graph_at_time(time)
|
||||
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted))
|
||||
|| !vector_layer.clip_instances.is_empty();
|
||||
if !ex.instances.is_empty() && !ex.drew_other {
|
||||
// Pure video: discard the (now-empty) scene and emit GPU instances.
|
||||
rendered.scene = Scene::new();
|
||||
rendered.has_content = true;
|
||||
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
|
||||
} else {
|
||||
if !ex.instances.is_empty() {
|
||||
// Mixed video + vector: the first pass diverted the video out of
|
||||
// the scene, so re-render with no sink to bake it back in (correct
|
||||
// z-order; Vello path). Rare — fast-splitting is deferred.
|
||||
rendered.scene = Scene::new();
|
||||
render_vector_layer_to_scene(
|
||||
document, time, vector_layer, &mut rendered.scene,
|
||||
base_transform, 1.0, image_cache, video_manager, None,
|
||||
);
|
||||
}
|
||||
rendered.has_content = vector_layer.graph_at_time(time)
|
||||
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted))
|
||||
|| !vector_layer.clip_instances.is_empty();
|
||||
}
|
||||
}
|
||||
AnyLayer::Audio(_) => {
|
||||
// Audio layers don't render visually
|
||||
|
|
@ -577,15 +637,34 @@ pub fn render_layer_isolated(
|
|||
return RenderedLayer::effect_layer(layer_id, opacity, active_effects);
|
||||
}
|
||||
AnyLayer::Group(group_layer) => {
|
||||
// Render each child layer's content into the group's scene
|
||||
// Render each child into the group's scene with an extraction sink. The
|
||||
// common imported-video case is a Group[Video, Audio] — audio draws
|
||||
// nothing, so it's pure video and composites via the GPU Video path.
|
||||
let mut ex = VideoExtract::default();
|
||||
for child in &group_layer.children {
|
||||
render_layer(
|
||||
document, time, child, &mut rendered.scene, base_transform,
|
||||
1.0, // Full opacity - layer opacity handled in compositing
|
||||
image_cache, video_manager, camera_frame,
|
||||
image_cache, video_manager, camera_frame, Some(&mut ex),
|
||||
);
|
||||
}
|
||||
rendered.has_content = !group_layer.children.is_empty();
|
||||
if !ex.instances.is_empty() && !ex.drew_other {
|
||||
rendered.scene = Scene::new();
|
||||
rendered.has_content = true;
|
||||
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
|
||||
} else {
|
||||
if !ex.instances.is_empty() {
|
||||
// Mixed: re-render with no sink to bake the video back into the scene.
|
||||
rendered.scene = Scene::new();
|
||||
for child in &group_layer.children {
|
||||
render_layer(
|
||||
document, time, child, &mut rendered.scene, base_transform,
|
||||
1.0, image_cache, video_manager, camera_frame, None,
|
||||
);
|
||||
}
|
||||
}
|
||||
rendered.has_content = !group_layer.children.is_empty();
|
||||
}
|
||||
}
|
||||
AnyLayer::Raster(raster_layer) => {
|
||||
if let Some(kf) = raster_layer.keyframe_at(time) {
|
||||
|
|
@ -614,6 +693,7 @@ fn render_vector_layer_to_scene(
|
|||
parent_opacity: f64,
|
||||
image_cache: &mut ImageCache,
|
||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||
extract: Option<&mut VideoExtract>,
|
||||
) {
|
||||
render_vector_layer(
|
||||
document,
|
||||
|
|
@ -624,6 +704,7 @@ fn render_vector_layer_to_scene(
|
|||
parent_opacity,
|
||||
image_cache,
|
||||
video_manager,
|
||||
extract,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -691,10 +772,10 @@ pub fn render_document_with_transform(
|
|||
for layer in document.visible_layers() {
|
||||
if any_soloed {
|
||||
if layer.soloed() {
|
||||
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None);
|
||||
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
|
||||
}
|
||||
} else {
|
||||
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None);
|
||||
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -755,10 +836,11 @@ fn render_layer(
|
|||
image_cache: &mut ImageCache,
|
||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||
camera_frame: Option<&crate::webcam::CaptureFrame>,
|
||||
mut extract: Option<&mut VideoExtract>,
|
||||
) {
|
||||
match layer {
|
||||
AnyLayer::Vector(vector_layer) => {
|
||||
render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager)
|
||||
render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager, extract)
|
||||
}
|
||||
AnyLayer::Audio(_) => {
|
||||
// Audio layers don't render visually
|
||||
|
|
@ -766,18 +848,20 @@ fn render_layer(
|
|||
AnyLayer::Video(video_layer) => {
|
||||
let mut video_mgr = video_manager.lock().unwrap();
|
||||
let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None };
|
||||
render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame);
|
||||
render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame, extract);
|
||||
}
|
||||
AnyLayer::Effect(_) => {
|
||||
// Effect layers are processed during GPU compositing, not rendered to scene
|
||||
}
|
||||
AnyLayer::Group(group_layer) => {
|
||||
// Render each child layer in the group
|
||||
// Render each child layer in the group, passing the extract sink down.
|
||||
for child in &group_layer.children {
|
||||
render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame);
|
||||
render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame, extract.as_deref_mut());
|
||||
}
|
||||
}
|
||||
AnyLayer::Raster(raster_layer) => {
|
||||
// Raster is non-video content — force the Vello fallback if extracting.
|
||||
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
|
||||
render_raster_layer_to_scene(raster_layer, time, scene, base_transform);
|
||||
}
|
||||
}
|
||||
|
|
@ -816,6 +900,7 @@ pub fn render_single_clip_instance(
|
|||
render_clip_instance(
|
||||
document, time, clip_instance, layer_opacity, scene, base_transform,
|
||||
&vector_layer.layer.animation_data, image_cache, video_manager, group_end_time,
|
||||
None, // edit-inside-clip overlay keeps the Vello path
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -831,6 +916,7 @@ fn render_clip_instance(
|
|||
image_cache: &mut ImageCache,
|
||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||
group_end_time: Option<f64>,
|
||||
mut extract: Option<&mut VideoExtract>,
|
||||
) {
|
||||
// Try to find the clip in the document's clip libraries
|
||||
// For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented)
|
||||
|
|
@ -972,7 +1058,7 @@ fn render_clip_instance(
|
|||
if !layer_node.data.visible() {
|
||||
continue;
|
||||
}
|
||||
render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None);
|
||||
render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None, extract.as_deref_mut());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -986,6 +1072,7 @@ fn render_video_layer(
|
|||
parent_opacity: f64,
|
||||
video_manager: &mut crate::video::VideoManager,
|
||||
camera_frame: Option<&crate::webcam::CaptureFrame>,
|
||||
mut extract: Option<&mut VideoExtract>,
|
||||
) {
|
||||
use crate::animation::TransformProperty;
|
||||
|
||||
|
|
@ -1151,14 +1238,26 @@ fn render_video_layer(
|
|||
Affine::IDENTITY
|
||||
};
|
||||
|
||||
// Render video frame as image fill
|
||||
scene.fill(
|
||||
Fill::NonZero,
|
||||
instance_transform,
|
||||
&image_with_alpha,
|
||||
Some(brush_transform),
|
||||
&video_rect,
|
||||
);
|
||||
// Extract to the GPU Video path when a sink is present; otherwise bake into
|
||||
// the Vello scene. The combined frame-pixel → document transform is
|
||||
// instance_transform * brush_transform (matching the top-level Video path).
|
||||
if let Some(ex) = extract.as_deref_mut() {
|
||||
ex.instances.push(VideoRenderInstance {
|
||||
rgba_data: frame.rgba_data.clone(),
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
transform: instance_transform * brush_transform,
|
||||
opacity: final_opacity,
|
||||
});
|
||||
} else {
|
||||
scene.fill(
|
||||
Fill::NonZero,
|
||||
instance_transform,
|
||||
&image_with_alpha,
|
||||
Some(brush_transform),
|
||||
&video_rect,
|
||||
);
|
||||
}
|
||||
clip_rendered = true;
|
||||
}
|
||||
|
||||
|
|
@ -1194,13 +1293,25 @@ fn render_video_layer(
|
|||
* Affine::translate((offset_x, offset_y))
|
||||
* Affine::scale(uniform_scale);
|
||||
|
||||
scene.fill(
|
||||
Fill::NonZero,
|
||||
preview_transform,
|
||||
&image_with_alpha,
|
||||
None,
|
||||
&frame_rect,
|
||||
);
|
||||
// preview_transform maps frame-pixel space → document directly, so it
|
||||
// is exactly the instance transform for the GPU path.
|
||||
if let Some(ex) = extract.as_deref_mut() {
|
||||
ex.instances.push(VideoRenderInstance {
|
||||
rgba_data: frame.rgba_data.clone(),
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
transform: preview_transform,
|
||||
opacity: final_opacity,
|
||||
});
|
||||
} else {
|
||||
scene.fill(
|
||||
Fill::NonZero,
|
||||
preview_transform,
|
||||
&image_with_alpha,
|
||||
None,
|
||||
&frame_rect,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1352,6 +1463,7 @@ fn render_vector_layer(
|
|||
parent_opacity: f64,
|
||||
image_cache: &mut ImageCache,
|
||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||
mut extract: Option<&mut VideoExtract>,
|
||||
) {
|
||||
// Cascade opacity: parent_opacity × layer.opacity
|
||||
let layer_opacity = parent_opacity * layer.layer.opacity;
|
||||
|
|
@ -1359,6 +1471,10 @@ fn render_vector_layer(
|
|||
// Render the layer's own VectorGraph (loose shapes) first, then clip instances
|
||||
// (groups / movie clips) on top. Shape tweens are applied here.
|
||||
if let Some(graph) = layer.tweened_graph_at(time) {
|
||||
// Loose vector geometry (and any image-asset fills) is non-video content —
|
||||
// force the Vello fallback. Conservative: a present-but-empty graph still
|
||||
// trips this, which only costs the fallback, never correctness.
|
||||
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
|
||||
render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache);
|
||||
}
|
||||
|
||||
|
|
@ -1370,7 +1486,7 @@ fn render_vector_layer(
|
|||
let frame_duration = 1.0 / document.framerate;
|
||||
layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration)
|
||||
});
|
||||
render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time);
|
||||
render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time, extract.as_deref_mut());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,77 @@ pub fn update_prepare_timing(
|
|||
t.composite_ms = composite_ms;
|
||||
}
|
||||
}
|
||||
/// GPU-measured composite cost (from timestamp queries; see `gpu_timer.rs`).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct GpuCompositeTiming {
|
||||
/// True when the adapter supports timestamp queries (else the ms is meaningless).
|
||||
pub supported: bool,
|
||||
/// GPU time of the whole composite section (Vello render + sRGB→linear +
|
||||
/// compositor + tonemap), in milliseconds. Read back asynchronously, so it
|
||||
/// lags the displayed frame by a frame or two.
|
||||
pub composite_gpu_ms: f64,
|
||||
/// Layers composited this frame.
|
||||
pub layers: u32,
|
||||
/// `queue.submit()` calls in the composite section this frame.
|
||||
pub submits: u32,
|
||||
}
|
||||
|
||||
static GPU_COMPOSITE: OnceLock<Mutex<GpuCompositeTiming>> = OnceLock::new();
|
||||
|
||||
/// Called from `VelloCallback::prepare()` with the GPU composite measurement.
|
||||
pub fn update_gpu_composite(supported: bool, composite_gpu_ms: f64, layers: u32, submits: u32) {
|
||||
let cell = GPU_COMPOSITE.get_or_init(|| Mutex::new(GpuCompositeTiming::default()));
|
||||
if let Ok(mut t) = cell.lock() {
|
||||
t.supported = supported;
|
||||
t.composite_gpu_ms = composite_gpu_ms;
|
||||
t.layers = layers;
|
||||
t.submits = submits;
|
||||
}
|
||||
}
|
||||
|
||||
fn get_gpu_composite() -> GpuCompositeTiming {
|
||||
GPU_COMPOSITE
|
||||
.get_or_init(|| Mutex::new(GpuCompositeTiming::default()))
|
||||
.lock()
|
||||
.map(|t| t.clone())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// CPU-side breakdown of the composite section (wall-clock `Instant` deltas). Since
|
||||
/// the GPU idles waiting on these CPU operations, this is where the per-frame cost
|
||||
/// actually lives. Sums should ≈ the CPU `composite_ms` for the doc's active paths.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CompositeCpuBreakdown {
|
||||
/// `renderer.render_to_texture` — Vello scene encode + its internal submit.
|
||||
pub vello_ms: f64,
|
||||
/// `srgb_to_linear.convert` — recording the conversion pass.
|
||||
pub convert_ms: f64,
|
||||
/// `canvas_blit.blit` — recording + its internal submit.
|
||||
pub blit_ms: f64,
|
||||
/// `compositor.composite` — recording + per-call uniforms buffer / bind group alloc.
|
||||
pub composite_ms: f64,
|
||||
/// Explicit `queue.submit()` calls.
|
||||
pub submit_ms: f64,
|
||||
}
|
||||
|
||||
static COMPOSITE_CPU: OnceLock<Mutex<CompositeCpuBreakdown>> = OnceLock::new();
|
||||
|
||||
/// Called from `VelloCallback::prepare()` with the composite CPU breakdown.
|
||||
pub fn update_composite_cpu(b: CompositeCpuBreakdown) {
|
||||
let cell = COMPOSITE_CPU.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()));
|
||||
if let Ok(mut t) = cell.lock() {
|
||||
*t = b;
|
||||
}
|
||||
}
|
||||
|
||||
fn get_composite_cpu() -> CompositeCpuBreakdown {
|
||||
COMPOSITE_CPU
|
||||
.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()))
|
||||
.lock()
|
||||
.map(|t| t.clone())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the
|
||||
/// raster-layer texture cache — the only unbounded-by-default VRAM consumer.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
|
|
@ -90,6 +161,12 @@ pub struct DebugStats {
|
|||
// GPU prepare() timing breakdown (from render thread)
|
||||
pub prepare_timing: PrepareTiming,
|
||||
|
||||
// GPU-measured composite cost (timestamp queries)
|
||||
pub gpu_composite: GpuCompositeTiming,
|
||||
|
||||
// CPU breakdown of the composite section
|
||||
pub composite_cpu: CompositeCpuBreakdown,
|
||||
|
||||
// Performance metrics for each section
|
||||
pub timing_memory_us: u64,
|
||||
pub timing_gpu_us: u64,
|
||||
|
|
@ -254,6 +331,8 @@ impl DebugStatsCollector {
|
|||
audio_input_devices,
|
||||
has_pointer,
|
||||
prepare_timing,
|
||||
gpu_composite: get_gpu_composite(),
|
||||
composite_cpu: get_composite_cpu(),
|
||||
timing_memory_us,
|
||||
timing_gpu_us,
|
||||
timing_midi_us,
|
||||
|
|
@ -306,8 +385,33 @@ pub fn render_debug_overlay(ctx: &egui::Context, stats: &DebugStats) {
|
|||
ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms));
|
||||
ui.label(format!(" removals: {:.2} ms", pt.removals_ms));
|
||||
ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms));
|
||||
ui.label(format!(" scene_build: {:.2} ms", pt.scene_build_ms));
|
||||
ui.label(format!(" composite: {:.2} ms", pt.composite_ms));
|
||||
ui.label(format!(" scene_build: {:.2} ms (CPU)", pt.scene_build_ms));
|
||||
ui.label(format!(" composite: {:.2} ms (CPU)", pt.composite_ms));
|
||||
|
||||
// GPU-measured composite cost (timestamp queries).
|
||||
let gc = &stats.gpu_composite;
|
||||
if gc.supported {
|
||||
ui.colored_label(
|
||||
egui::Color32::LIGHT_GREEN,
|
||||
format!("GPU composite: {:.2} ms (GPU)", gc.composite_gpu_ms),
|
||||
);
|
||||
ui.label(format!(" layers: {} submits: {}", gc.layers, gc.submits));
|
||||
} else {
|
||||
ui.label(format!(
|
||||
"GPU composite: n/a (no timestamp support) layers: {} submits: {}",
|
||||
gc.layers, gc.submits
|
||||
));
|
||||
}
|
||||
|
||||
// CPU breakdown of the composite (where the GPU is actually waiting).
|
||||
let cc = &stats.composite_cpu;
|
||||
let cc_sum = cc.vello_ms + cc.convert_ms + cc.blit_ms + cc.composite_ms + cc.submit_ms;
|
||||
ui.colored_label(egui::Color32::LIGHT_BLUE, format!("Composite CPU breakdown: {:.2} ms", cc_sum));
|
||||
ui.label(format!(" vello(render): {:.2} ms", cc.vello_ms));
|
||||
ui.label(format!(" srgb→linear: {:.2} ms", cc.convert_ms));
|
||||
ui.label(format!(" blit: {:.2} ms", cc.blit_ms));
|
||||
ui.label(format!(" compositor: {:.2} ms", cc.composite_ms));
|
||||
ui.label(format!(" queue.submit: {:.2} ms", cc.submit_ms));
|
||||
|
||||
ui.add_space(8.0);
|
||||
|
||||
|
|
|
|||
|
|
@ -830,22 +830,13 @@ fn composite_document_to_hdr(
|
|||
if inst.rgba_data.is_empty() { continue; }
|
||||
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
|
||||
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
|
||||
// sRGB straight-alpha → linear premultiplied
|
||||
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| {
|
||||
let a = p[3] as f32 / 255.0;
|
||||
let lin = |c: u8| -> f32 {
|
||||
let f = c as f32 / 255.0;
|
||||
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) }
|
||||
};
|
||||
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8;
|
||||
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8;
|
||||
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
|
||||
[r, g, b, p[3]]
|
||||
}).collect();
|
||||
let tex = upload_transient_texture(device, queue, &linear, inst.width, inst.height, Some("export_video_frame_tex"));
|
||||
// Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU
|
||||
// decodes to linear on sample (no per-pixel CPU conversion). Blit with
|
||||
// blit_straight so the shader doesn't unpremultiply.
|
||||
let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex"));
|
||||
let tex_view = tex.create_view(&Default::default());
|
||||
let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height);
|
||||
gpu_resources.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||
gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||
let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal);
|
||||
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") });
|
||||
gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None);
|
||||
|
|
@ -865,7 +856,7 @@ fn composite_document_to_hdr(
|
|||
};
|
||||
[lin(p[0]), lin(p[1]), lin(p[2]), p[3]]
|
||||
}).collect();
|
||||
let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, Some("export_float_tex"));
|
||||
let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, wgpu::TextureFormat::Rgba8Unorm, Some("export_float_tex"));
|
||||
let tex_view = tex.create_view(&Default::default());
|
||||
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
|
||||
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
|
||||
|
|
@ -919,13 +910,16 @@ fn composite_document_to_hdr(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Upload `pixels` to a transient `Rgba8Unorm` GPU texture (TEXTURE_BINDING | COPY_DST).
|
||||
/// Upload `pixels` to a transient GPU texture (TEXTURE_BINDING | COPY_DST) in the
|
||||
/// given format. Use `Rgba8UnormSrgb` to upload raw sRGB bytes and let the GPU
|
||||
/// decode to linear on sample (no CPU conversion).
|
||||
fn upload_transient_texture(
|
||||
device: &wgpu::Device,
|
||||
queue: &wgpu::Queue,
|
||||
pixels: &[u8],
|
||||
width: u32,
|
||||
height: u32,
|
||||
format: wgpu::TextureFormat,
|
||||
label: Option<&'static str>,
|
||||
) -> wgpu::Texture {
|
||||
let tex = device.create_texture(&wgpu::TextureDescriptor {
|
||||
|
|
@ -933,7 +927,7 @@ fn upload_transient_texture(
|
|||
size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 },
|
||||
mip_level_count: 1, sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: wgpu::TextureFormat::Rgba8Unorm,
|
||||
format,
|
||||
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
||||
view_formats: &[],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1951,6 +1951,9 @@ impl GpuBrushEngine {
|
|||
/// the camera transform.
|
||||
pub struct CanvasBlitPipeline {
|
||||
pub pipeline: wgpu::RenderPipeline,
|
||||
/// Variant for straight-alpha sources (hardware-sRGB video frames): the
|
||||
/// fragment shader skips the unpremultiply. See [`CanvasBlitPipeline::blit_straight`].
|
||||
pub pipeline_straight: wgpu::RenderPipeline,
|
||||
pub bg_layout: wgpu::BindGroupLayout,
|
||||
pub sampler: wgpu::Sampler,
|
||||
/// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res
|
||||
|
|
@ -2132,6 +2135,39 @@ impl CanvasBlitPipeline {
|
|||
},
|
||||
);
|
||||
|
||||
// Variant pipeline for straight-alpha sources (hardware-sRGB video frames):
|
||||
// identical except the fragment shader skips the unpremultiply.
|
||||
let pipeline_straight = device.create_render_pipeline(
|
||||
&wgpu::RenderPipelineDescriptor {
|
||||
label: Some("canvas_blit_pipeline_straight"),
|
||||
layout: Some(&pipeline_layout),
|
||||
vertex: wgpu::VertexState {
|
||||
module: &shader,
|
||||
entry_point: Some("vs_main"),
|
||||
buffers: &[],
|
||||
compilation_options: Default::default(),
|
||||
},
|
||||
fragment: Some(wgpu::FragmentState {
|
||||
module: &shader,
|
||||
entry_point: Some("fs_main_straight"),
|
||||
targets: &[Some(wgpu::ColorTargetState {
|
||||
format: wgpu::TextureFormat::Rgba16Float,
|
||||
blend: None,
|
||||
write_mask: wgpu::ColorWrites::ALL,
|
||||
})],
|
||||
compilation_options: Default::default(),
|
||||
}),
|
||||
primitive: wgpu::PrimitiveState {
|
||||
topology: wgpu::PrimitiveTopology::TriangleStrip,
|
||||
..Default::default()
|
||||
},
|
||||
depth_stencil: None,
|
||||
multisample: wgpu::MultisampleState::default(),
|
||||
multiview: None,
|
||||
cache: None,
|
||||
},
|
||||
);
|
||||
|
||||
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
|
||||
label: Some("canvas_blit_sampler"),
|
||||
address_mode_u: wgpu::AddressMode::ClampToEdge,
|
||||
|
|
@ -2165,7 +2201,7 @@ impl CanvasBlitPipeline {
|
|||
..Default::default()
|
||||
});
|
||||
|
||||
Self { pipeline, bg_layout, sampler, linear_sampler, mask_sampler }
|
||||
Self { pipeline, pipeline_straight, bg_layout, sampler, linear_sampler, mask_sampler }
|
||||
}
|
||||
|
||||
/// Render the canvas texture into `target_view` (Rgba16Float) with the given camera.
|
||||
|
|
@ -2183,7 +2219,7 @@ impl CanvasBlitPipeline {
|
|||
transform: &BlitTransform,
|
||||
mask_view: Option<&wgpu::TextureView>,
|
||||
) {
|
||||
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler);
|
||||
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline);
|
||||
}
|
||||
|
||||
/// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies).
|
||||
|
|
@ -2196,9 +2232,25 @@ impl CanvasBlitPipeline {
|
|||
transform: &BlitTransform,
|
||||
mask_view: Option<&wgpu::TextureView>,
|
||||
) {
|
||||
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler);
|
||||
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler, &self.pipeline);
|
||||
}
|
||||
|
||||
/// Blit a **straight-alpha** source (e.g. a video frame uploaded to an
|
||||
/// `Rgba8UnormSrgb` texture, hardware-decoded to linear on sample). Uses the
|
||||
/// `fs_main_straight` pipeline, which skips the unpremultiply that `blit` does.
|
||||
pub fn blit_straight(
|
||||
&self,
|
||||
device: &wgpu::Device,
|
||||
queue: &wgpu::Queue,
|
||||
canvas_view: &wgpu::TextureView,
|
||||
target_view: &wgpu::TextureView,
|
||||
transform: &BlitTransform,
|
||||
mask_view: Option<&wgpu::TextureView>,
|
||||
) {
|
||||
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline_straight);
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn blit_with(
|
||||
&self,
|
||||
device: &wgpu::Device,
|
||||
|
|
@ -2208,6 +2260,7 @@ impl CanvasBlitPipeline {
|
|||
transform: &BlitTransform,
|
||||
mask_view: Option<&wgpu::TextureView>,
|
||||
canvas_sampler: &wgpu::Sampler,
|
||||
pipeline: &wgpu::RenderPipeline,
|
||||
) {
|
||||
// When no mask is provided, create a temporary 1×1 all-white texture.
|
||||
// (queue is already available here, unlike in new())
|
||||
|
|
@ -2296,7 +2349,7 @@ impl CanvasBlitPipeline {
|
|||
occlusion_query_set: None,
|
||||
timestamp_writes: None,
|
||||
});
|
||||
rp.set_pipeline(&self.pipeline);
|
||||
rp.set_pipeline(pipeline);
|
||||
rp.set_bind_group(0, &bg, &[]);
|
||||
rp.draw(0..4, 0..1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,135 @@
|
|||
//! Minimal GPU timestamp timer for the composite pipeline.
|
||||
//!
|
||||
//! Brackets a section of GPU work with two timestamps and reads the elapsed GPU
|
||||
//! time back asynchronously (no pipeline stall). Used to attribute the per-frame
|
||||
//! composite cost (Vello render + sRGB→linear + compositor + tonemap) shown in F3.
|
||||
//!
|
||||
//! Requires `TIMESTAMP_QUERY` + `TIMESTAMP_QUERY_INSIDE_ENCODERS`; [`FrameGpuTimer::new`]
|
||||
//! returns `None` when the adapter doesn't support them, and all call sites no-op.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// State of the single readback buffer (shared with the map callback).
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Readback {
|
||||
/// Available to resolve into this frame.
|
||||
Free,
|
||||
/// Submitted + `map_async` in flight; don't touch until the callback fires.
|
||||
Mapping,
|
||||
/// Mapped and ready to read.
|
||||
Ready,
|
||||
}
|
||||
|
||||
/// Times one GPU section (two timestamps) per frame with intermittent async readback.
|
||||
pub struct FrameGpuTimer {
|
||||
query_set: wgpu::QuerySet,
|
||||
resolve_buf: wgpu::Buffer,
|
||||
readback_buf: wgpu::Buffer,
|
||||
state: Arc<Mutex<Readback>>,
|
||||
/// Nanoseconds per timestamp tick.
|
||||
period_ns: f32,
|
||||
/// Most recent measured GPU time for the bracketed section, in milliseconds.
|
||||
last_ms: f64,
|
||||
}
|
||||
|
||||
impl FrameGpuTimer {
|
||||
/// Required device features for GPU timestamp timing.
|
||||
pub fn required_features() -> wgpu::Features {
|
||||
wgpu::Features::TIMESTAMP_QUERY | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS
|
||||
}
|
||||
|
||||
/// Create a timer, or `None` if the device lacks timestamp support.
|
||||
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option<Self> {
|
||||
if !device.features().contains(Self::required_features()) {
|
||||
return None;
|
||||
}
|
||||
let query_set = device.create_query_set(&wgpu::QuerySetDescriptor {
|
||||
label: Some("composite_gpu_timer"),
|
||||
ty: wgpu::QueryType::Timestamp,
|
||||
count: 2,
|
||||
});
|
||||
// 2 timestamps × u64.
|
||||
let size = 2 * std::mem::size_of::<u64>() as u64;
|
||||
let resolve_buf = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("composite_gpu_timer_resolve"),
|
||||
size,
|
||||
usage: wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let readback_buf = device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("composite_gpu_timer_readback"),
|
||||
size,
|
||||
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
Some(Self {
|
||||
query_set,
|
||||
resolve_buf,
|
||||
readback_buf,
|
||||
state: Arc::new(Mutex::new(Readback::Free)),
|
||||
period_ns: queue.get_timestamp_period(),
|
||||
last_ms: 0.0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Write the **start** timestamp (call just before the bracketed GPU work).
|
||||
pub fn start(&self, device: &wgpu::Device, queue: &wgpu::Queue) {
|
||||
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||
label: Some("composite_gpu_timer_start"),
|
||||
});
|
||||
enc.write_timestamp(&self.query_set, 0);
|
||||
queue.submit(Some(enc.finish()));
|
||||
}
|
||||
|
||||
/// Write the **end** timestamp and, if the readback buffer is free, resolve +
|
||||
/// kick off an async read. Also consumes a previously-completed read into
|
||||
/// `last_ms`. Call just after the bracketed GPU work.
|
||||
pub fn end(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
|
||||
// 1. Consume a completed readback first (so the buffer is free to reuse).
|
||||
let cur = *self.state.lock().unwrap();
|
||||
if cur == Readback::Ready {
|
||||
{
|
||||
let view = self.readback_buf.slice(..).get_mapped_range();
|
||||
let t0 = u64::from_le_bytes(view[0..8].try_into().unwrap());
|
||||
let t1 = u64::from_le_bytes(view[8..16].try_into().unwrap());
|
||||
// Timestamps can wrap or arrive out of order across queue resets; guard.
|
||||
let ticks = t1.saturating_sub(t0);
|
||||
self.last_ms = ticks as f64 * self.period_ns as f64 / 1.0e6;
|
||||
}
|
||||
self.readback_buf.unmap();
|
||||
*self.state.lock().unwrap() = Readback::Free;
|
||||
}
|
||||
|
||||
// 2. End timestamp + resolve + copy, only when the buffer is free.
|
||||
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||
label: Some("composite_gpu_timer_end"),
|
||||
});
|
||||
enc.write_timestamp(&self.query_set, 1);
|
||||
|
||||
let can_read = *self.state.lock().unwrap() == Readback::Free;
|
||||
if can_read {
|
||||
enc.resolve_query_set(&self.query_set, 0..2, &self.resolve_buf, 0);
|
||||
enc.copy_buffer_to_buffer(
|
||||
&self.resolve_buf,
|
||||
0,
|
||||
&self.readback_buf,
|
||||
0,
|
||||
2 * std::mem::size_of::<u64>() as u64,
|
||||
);
|
||||
}
|
||||
queue.submit(Some(enc.finish()));
|
||||
|
||||
if can_read {
|
||||
*self.state.lock().unwrap() = Readback::Mapping;
|
||||
let state = Arc::clone(&self.state);
|
||||
self.readback_buf.slice(..).map_async(wgpu::MapMode::Read, move |res| {
|
||||
*state.lock().unwrap() = if res.is_ok() { Readback::Ready } else { Readback::Free };
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Most recently measured GPU time of the bracketed section, in milliseconds.
|
||||
pub fn last_ms(&self) -> f64 {
|
||||
self.last_ms
|
||||
}
|
||||
}
|
||||
|
|
@ -50,6 +50,7 @@ use effect_thumbnails::EffectThumbnailGenerator;
|
|||
mod custom_cursor;
|
||||
mod tablet;
|
||||
mod debug_overlay;
|
||||
mod gpu_timer;
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
mod test_mode;
|
||||
|
|
@ -174,8 +175,12 @@ fn main() -> eframe::Result {
|
|||
device_descriptor: std::sync::Arc::new(|adapter| {
|
||||
let features = adapter.features();
|
||||
// Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's
|
||||
// unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability)
|
||||
let optional_features = wgpu::Features::SHADER_F16;
|
||||
// unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability).
|
||||
// TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer
|
||||
// (gpu_timer.rs); both are optional and no-op when unsupported.
|
||||
let optional_features = wgpu::Features::SHADER_F16
|
||||
| wgpu::Features::TIMESTAMP_QUERY
|
||||
| wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS;
|
||||
|
||||
let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl {
|
||||
wgpu::Limits::downlevel_webgl2_defaults()
|
||||
|
|
|
|||
|
|
@ -75,3 +75,25 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
|
|||
let tinted = base + tint - base * tint;
|
||||
return vec4<f32>(tinted, masked_a);
|
||||
}
|
||||
|
||||
// Variant for sources that are ALREADY straight-alpha linear — notably a video
|
||||
// frame uploaded to an `Rgba8UnormSrgb` texture, where the hardware decodes
|
||||
// sRGB→linear on sample and leaves alpha untouched. No unpremultiply (the source
|
||||
// was never premultiplied), so we skip the divide entirely. The compositor wants
|
||||
// straight-alpha linear, which is exactly what the sample already is.
|
||||
@fragment
|
||||
fn fs_main_straight(in: VertexOutput) -> @location(0) vec4<f32> {
|
||||
let m = mat3x3<f32>(transform.col0.xyz, transform.col1.xyz, transform.col2.xyz);
|
||||
let canvas_uv = (m * vec3<f32>(in.uv.x, in.uv.y, 1.0)).xy;
|
||||
|
||||
if canvas_uv.x < 0.0 || canvas_uv.x > 1.0
|
||||
|| canvas_uv.y < 0.0 || canvas_uv.y > 1.0 {
|
||||
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
|
||||
}
|
||||
|
||||
let c = textureSample(canvas_tex, canvas_sampler, canvas_uv);
|
||||
let mask = textureSample(mask_tex, mask_sampler, canvas_uv).r;
|
||||
let tint = vec3<f32>(transform.col0.w, transform.col1.w, transform.col2.w);
|
||||
let tinted = c.rgb + tint - c.rgb * tint;
|
||||
return vec4<f32>(tinted, c.a * mask);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,6 +45,89 @@ fn upload_pixmap_to_texture(queue: &wgpu::Queue, texture: &wgpu::Texture, pixmap
|
|||
/// Set to true to use the new pipeline, false for legacy single-scene rendering
|
||||
const USE_HDR_COMPOSITING: bool = true; // Enabled for testing
|
||||
|
||||
/// Caches GPU textures for decoded video frames, keyed by the frame buffer's `Arc`
|
||||
/// identity. A static/paused video then costs ~nothing per repaint (cache hit → no
|
||||
/// per-pixel CPU sRGB→linear conversion, no texture allocation, no upload). The
|
||||
/// cached texture holds premultiplied-linear RGBA8 — exactly what `canvas_blit`
|
||||
/// expects. During playback each new decoded frame is a fresh `Arc` → one
|
||||
/// conversion+upload per frame (not per repaint).
|
||||
struct CachedVideoFrame {
|
||||
/// Keep the source buffer alive so its pointer (our cache key) can't be reused.
|
||||
_keep: std::sync::Arc<Vec<u8>>,
|
||||
texture: wgpu::Texture,
|
||||
last_seen: u64,
|
||||
}
|
||||
|
||||
struct VideoFrameTexCache {
|
||||
entries: std::collections::HashMap<usize, CachedVideoFrame>,
|
||||
frame: u64,
|
||||
}
|
||||
|
||||
impl VideoFrameTexCache {
|
||||
fn new() -> Self {
|
||||
Self { entries: std::collections::HashMap::new(), frame: 0 }
|
||||
}
|
||||
|
||||
fn begin_frame(&mut self) {
|
||||
self.frame = self.frame.wrapping_add(1);
|
||||
}
|
||||
|
||||
/// View of the cached (or freshly converted+uploaded) texture for `rgba`.
|
||||
fn texture_view(
|
||||
&mut self,
|
||||
device: &wgpu::Device,
|
||||
queue: &wgpu::Queue,
|
||||
rgba: &std::sync::Arc<Vec<u8>>,
|
||||
w: u32,
|
||||
h: u32,
|
||||
) -> wgpu::TextureView {
|
||||
let key = std::sync::Arc::as_ptr(rgba) as usize;
|
||||
let frame = self.frame;
|
||||
if let Some(e) = self.entries.get_mut(&key) {
|
||||
e.last_seen = frame;
|
||||
return e.texture.create_view(&wgpu::TextureViewDescriptor::default());
|
||||
}
|
||||
// Miss: upload the raw sRGB straight-alpha bytes verbatim into an sRGB
|
||||
// texture. The GPU decodes sRGB→linear on sample (free), so there is no
|
||||
// per-pixel CPU conversion — the cost that used to dominate playback/export.
|
||||
// Blit this with `blit_straight` (it must NOT unpremultiply).
|
||||
let texture = device.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some("video_frame_tex"),
|
||||
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
|
||||
mip_level_count: 1,
|
||||
sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: wgpu::TextureFormat::Rgba8UnormSrgb,
|
||||
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
||||
view_formats: &[],
|
||||
});
|
||||
queue.write_texture(
|
||||
wgpu::TexelCopyTextureInfo {
|
||||
texture: &texture,
|
||||
mip_level: 0,
|
||||
origin: wgpu::Origin3d::ZERO,
|
||||
aspect: wgpu::TextureAspect::All,
|
||||
},
|
||||
rgba,
|
||||
wgpu::TexelCopyBufferLayout {
|
||||
offset: 0,
|
||||
bytes_per_row: Some(w * 4),
|
||||
rows_per_image: Some(h),
|
||||
},
|
||||
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
|
||||
);
|
||||
let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
|
||||
self.entries.insert(key, CachedVideoFrame { _keep: rgba.clone(), texture, last_seen: frame });
|
||||
view
|
||||
}
|
||||
|
||||
/// Drop textures not used in the last couple of frames (bounds VRAM).
|
||||
fn evict_stale(&mut self) {
|
||||
let frame = self.frame;
|
||||
self.entries.retain(|_, e| e.last_seen + 2 >= frame);
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared Vello resources (created once, reused by all Stage panes)
|
||||
struct SharedVelloResources {
|
||||
renderer: Arc<Mutex<vello::Renderer>>,
|
||||
|
|
@ -72,6 +155,11 @@ struct SharedVelloResources {
|
|||
/// True when Vello is running its CPU software renderer (either forced or GPU fallback).
|
||||
/// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area.
|
||||
is_cpu_renderer: bool,
|
||||
/// GPU timestamp timer for the composite section (F3 debug). Lazily created on
|
||||
/// the first frame (needs the device/queue); `None` if unsupported.
|
||||
gpu_timer: Mutex<Option<crate::gpu_timer::FrameGpuTimer>>,
|
||||
/// Per-frame video texture cache (skips re-converting/uploading a static frame).
|
||||
video_frame_cache: Mutex<VideoFrameTexCache>,
|
||||
}
|
||||
|
||||
/// Per-instance Vello resources (created for each Stage pane)
|
||||
|
|
@ -302,6 +390,8 @@ impl SharedVelloResources {
|
|||
gpu_brush: Mutex::new(gpu_brush),
|
||||
canvas_blit,
|
||||
is_cpu_renderer: use_cpu || is_cpu_renderer,
|
||||
gpu_timer: Mutex::new(None),
|
||||
video_frame_cache: Mutex::new(VideoFrameTexCache::new()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1079,6 +1169,25 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
// HDR buffer spec for linear buffers
|
||||
let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float);
|
||||
|
||||
// F3: bracket the composite section with a GPU timestamp (lazily create the
|
||||
// timer; no-op when the adapter lacks timestamp support).
|
||||
let ts_supported = device
|
||||
.features()
|
||||
.contains(crate::gpu_timer::FrameGpuTimer::required_features());
|
||||
{
|
||||
let mut tg = shared.gpu_timer.lock().unwrap();
|
||||
if tg.is_none() && ts_supported {
|
||||
*tg = crate::gpu_timer::FrameGpuTimer::new(device, queue);
|
||||
}
|
||||
if let Some(t) = tg.as_ref() {
|
||||
t.start(device, queue);
|
||||
}
|
||||
}
|
||||
shared.video_frame_cache.lock().unwrap().begin_frame();
|
||||
|
||||
// F3: CPU breakdown of the composite (the GPU idles waiting on these).
|
||||
let mut cput = crate::debug_overlay::CompositeCpuBreakdown::default();
|
||||
|
||||
// First, render background and composite it
|
||||
// The background scene contains only a rectangle at document bounds,
|
||||
// so we use TRANSPARENT base_color to not fill the whole viewport
|
||||
|
|
@ -1097,6 +1206,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
antialiasing_method: aa_method,
|
||||
};
|
||||
|
||||
let _t = std::time::Instant::now();
|
||||
if let Some(pixmap) = &composite_result.background_cpu {
|
||||
if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) {
|
||||
upload_pixmap_to_texture(queue, tex, pixmap);
|
||||
|
|
@ -1104,13 +1214,18 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
} else if let Ok(mut renderer) = shared.renderer.lock() {
|
||||
renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok();
|
||||
}
|
||||
cput.vello_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
// Convert sRGB to linear HDR
|
||||
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||
label: Some("bg_srgb_to_linear_encoder"),
|
||||
});
|
||||
let _t = std::time::Instant::now();
|
||||
shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view);
|
||||
cput.convert_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
let _t = std::time::Instant::now();
|
||||
queue.submit(Some(convert_encoder.finish()));
|
||||
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
// Composite background onto HDR texture (first layer, clears to dark gray for stage area)
|
||||
let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0);
|
||||
|
|
@ -1120,6 +1235,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
// Clear to dark gray (stage background outside document bounds)
|
||||
// Note: stage_bg values are already in linear space for HDR compositing
|
||||
let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0];
|
||||
let _t = std::time::Instant::now();
|
||||
shared.compositor.composite(
|
||||
device,
|
||||
queue,
|
||||
|
|
@ -1129,7 +1245,10 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
hdr_view,
|
||||
Some(stage_bg),
|
||||
);
|
||||
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
let _t = std::time::Instant::now();
|
||||
queue.submit(Some(encoder.finish()));
|
||||
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
}
|
||||
buffer_pool.release(bg_srgb_handle);
|
||||
buffer_pool.release(bg_hdr_handle);
|
||||
|
|
@ -1351,6 +1470,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
buffer_pool.get_view(hdr_layer_handle),
|
||||
&instance_resources.hdr_texture_view,
|
||||
) {
|
||||
let _t_vello = std::time::Instant::now();
|
||||
if let Some(pixmap) = &rendered_layer.cpu_pixmap {
|
||||
if let Some(tex) = buffer_pool.get_texture(srgb_handle) {
|
||||
upload_pixmap_to_texture(queue, tex, pixmap);
|
||||
|
|
@ -1358,6 +1478,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
} else if let Ok(mut renderer) = shared.renderer.lock() {
|
||||
renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok();
|
||||
}
|
||||
cput.vello_ms += _t_vello.elapsed().as_secs_f64() * 1000.0;
|
||||
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||
label: Some("layer_srgb_to_linear_encoder"),
|
||||
});
|
||||
|
|
@ -1555,7 +1676,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
}
|
||||
}
|
||||
RenderedLayerType::Video { instances } => {
|
||||
// Video layer — per-instance: upload decoded frame → blit → composite.
|
||||
// Video layer — per-instance: (cached) frame texture → blit → composite.
|
||||
for inst in instances {
|
||||
if inst.rgba_data.is_empty() { continue; }
|
||||
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
|
||||
|
|
@ -1563,40 +1684,20 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
buffer_pool.get_view(hdr_layer_handle),
|
||||
&instance_resources.hdr_texture_view,
|
||||
) {
|
||||
// Convert sRGB straight-alpha → linear premultiplied.
|
||||
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| {
|
||||
let a = p[3] as f32 / 255.0;
|
||||
let lin = |c: u8| -> f32 {
|
||||
let f = c as f32 / 255.0;
|
||||
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) }
|
||||
};
|
||||
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8;
|
||||
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8;
|
||||
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
|
||||
[r, g, b, p[3]]
|
||||
}).collect();
|
||||
|
||||
let tex = device.create_texture(&wgpu::TextureDescriptor {
|
||||
label: Some("video_frame_tex"),
|
||||
size: wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
|
||||
mip_level_count: 1, sample_count: 1,
|
||||
dimension: wgpu::TextureDimension::D2,
|
||||
format: wgpu::TextureFormat::Rgba8Unorm,
|
||||
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
||||
view_formats: &[],
|
||||
});
|
||||
queue.write_texture(
|
||||
wgpu::TexelCopyTextureInfo { texture: &tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
|
||||
&linear,
|
||||
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(inst.width * 4), rows_per_image: Some(inst.height) },
|
||||
wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
|
||||
);
|
||||
let tex_view = tex.create_view(&wgpu::TextureViewDescriptor::default());
|
||||
|
||||
// Reuse the GPU texture for this frame if it's unchanged (a
|
||||
// static/paused video → no CPU conversion, alloc, or upload).
|
||||
// Timed into `blit_ms` (incl the cache lookup + per-frame view).
|
||||
let _t = std::time::Instant::now();
|
||||
let tex_view = shared
|
||||
.video_frame_cache
|
||||
.lock()
|
||||
.unwrap()
|
||||
.texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
|
||||
let bt = crate::gpu_brush::BlitTransform::new(
|
||||
inst.transform, inst.width, inst.height, width, height,
|
||||
);
|
||||
shared.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||
shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||
cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(
|
||||
hdr_layer_handle,
|
||||
|
|
@ -1606,10 +1707,14 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||
label: Some("video_composite_encoder"),
|
||||
});
|
||||
let _t = std::time::Instant::now();
|
||||
shared.compositor.composite(
|
||||
device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None,
|
||||
);
|
||||
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
let _t = std::time::Instant::now();
|
||||
queue.submit(Some(encoder.finish()));
|
||||
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||
}
|
||||
buffer_pool.release(hdr_layer_handle);
|
||||
}
|
||||
|
|
@ -1841,6 +1946,23 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
|||
buffer_pool.next_frame();
|
||||
drop(buffer_pool);
|
||||
|
||||
// F3: close the GPU timestamp bracket + publish the composite measurement.
|
||||
{
|
||||
let layers = composite_result.layers.len() as u32;
|
||||
// Submits aren't counted per-site; estimate from the per-layer pattern
|
||||
// (bg ~3 + ~2 per layer). Drops toward ~1 once the passes are batched.
|
||||
let submits_est = 3 + 2 * layers;
|
||||
let mut tg = shared.gpu_timer.lock().unwrap();
|
||||
if let Some(t) = tg.as_mut() {
|
||||
t.end(device, queue);
|
||||
crate::debug_overlay::update_gpu_composite(true, t.last_ms(), layers, submits_est);
|
||||
} else {
|
||||
crate::debug_overlay::update_gpu_composite(false, 0.0, layers, submits_est);
|
||||
}
|
||||
}
|
||||
crate::debug_overlay::update_composite_cpu(cput);
|
||||
shared.video_frame_cache.lock().unwrap().evict_stale();
|
||||
|
||||
// --- Frame timing report ---
|
||||
let _t_end = std::time::Instant::now();
|
||||
let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0;
|
||||
|
|
|
|||
Loading…
Reference in New Issue