Composite grouped/nested video on the GPU path
Imported video is a Group[Video, Audio] that rendered as a Vello-baked Vector layer, re-uploading the full frame to Vello's image atlas every frame (~17ms/frame at 1080p, hitting playback and export alike). Extract video frames out of the Group/clip scene recursion into VideoRenderInstances so they composite via the GPU Video path; mixed video+vector containers fall back to Vello (correct, unaccelerated). Also route video through hardware sRGB decode: upload raw sRGB bytes to an Rgba8UnormSrgb texture and blit with a non-unpremultiplying shader variant (blit_straight), removing the per-frame per-pixel CPU sRGB->linear pass. Add an F3 GPU-timestamp timer and a per-frame video texture cache. Drops the live composite of a 1080p video from ~17ms to ~2-3ms. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ce151ffd61
commit
5844a0f070
|
|
@ -232,6 +232,21 @@ pub struct VideoRenderInstance {
|
||||||
pub opacity: f32,
|
pub opacity: f32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sink for pulling video frames out of a container layer's scene recursion, so
|
||||||
|
/// they composite via the fast GPU Video path instead of baking into Vello.
|
||||||
|
///
|
||||||
|
/// Threaded as `Option<&mut VideoExtract>` through the isolated-render scene
|
||||||
|
/// functions. When present, [`render_video_layer`] pushes a [`VideoRenderInstance`]
|
||||||
|
/// instead of drawing into the Vello scene. `drew_other` is set whenever any
|
||||||
|
/// non-video primitive (vector graph, image asset, raster) is emitted; that forces
|
||||||
|
/// the safe Vello fallback because the container mixes video with other content
|
||||||
|
/// and we can't preserve z-order by extraction alone.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct VideoExtract {
|
||||||
|
instances: Vec<VideoRenderInstance>,
|
||||||
|
drew_other: bool,
|
||||||
|
}
|
||||||
|
|
||||||
/// Type of rendered layer for compositor handling
|
/// Type of rendered layer for compositor handling
|
||||||
pub enum RenderedLayerType {
|
pub enum RenderedLayerType {
|
||||||
/// Vector / group layer — Vello scene in `RenderedLayer::scene` is used.
|
/// Vector / group layer — Vello scene in `RenderedLayer::scene` is used.
|
||||||
|
|
@ -428,6 +443,30 @@ pub fn render_document_for_compositing(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One-shot diagnostic: dump what the compositor actually receives. Set
|
||||||
|
// LB_LAYER_DEBUG=1 to print a single snapshot of each layer's resolved type.
|
||||||
|
if std::env::var("LB_LAYER_DEBUG").is_ok() {
|
||||||
|
static ONCE: std::sync::Once = std::sync::Once::new();
|
||||||
|
ONCE.call_once(|| {
|
||||||
|
eprintln!("[LB_LAYER_DEBUG] composite layers = {}", rendered_layers.len());
|
||||||
|
for (i, l) in rendered_layers.iter().enumerate() {
|
||||||
|
let desc = match &l.layer_type {
|
||||||
|
RenderedLayerType::Vector =>
|
||||||
|
format!("Vector (has_content={}, scene_empty={})", l.has_content, l.cpu_pixmap.is_none()),
|
||||||
|
RenderedLayerType::Raster { width, height, dirty, .. } =>
|
||||||
|
format!("Raster {width}x{height} dirty={dirty}"),
|
||||||
|
RenderedLayerType::Video { instances } =>
|
||||||
|
format!("Video ({} instance(s))", instances.len()),
|
||||||
|
RenderedLayerType::Float { width, height, .. } =>
|
||||||
|
format!("Float {width}x{height}"),
|
||||||
|
RenderedLayerType::Effect { effect_instances } =>
|
||||||
|
format!("Effect ({} instance(s))", effect_instances.len()),
|
||||||
|
};
|
||||||
|
eprintln!("[LB_LAYER_DEBUG] layer[{i}] id={} type={desc}", l.layer_id);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
CompositeRenderResult {
|
CompositeRenderResult {
|
||||||
background,
|
background,
|
||||||
background_cpu: None,
|
background_cpu: None,
|
||||||
|
|
@ -462,6 +501,9 @@ pub fn render_layer_isolated(
|
||||||
// Render layer content with full opacity (1.0) - opacity applied during compositing
|
// Render layer content with full opacity (1.0) - opacity applied during compositing
|
||||||
match layer {
|
match layer {
|
||||||
AnyLayer::Vector(vector_layer) => {
|
AnyLayer::Vector(vector_layer) => {
|
||||||
|
// Render into the scene with an extraction sink so a clip that is purely
|
||||||
|
// video (no vector geometry) composites via the fast GPU Video path.
|
||||||
|
let mut ex = VideoExtract::default();
|
||||||
render_vector_layer_to_scene(
|
render_vector_layer_to_scene(
|
||||||
document,
|
document,
|
||||||
time,
|
time,
|
||||||
|
|
@ -471,11 +513,29 @@ pub fn render_layer_isolated(
|
||||||
1.0, // Full opacity - layer opacity handled in compositing
|
1.0, // Full opacity - layer opacity handled in compositing
|
||||||
image_cache,
|
image_cache,
|
||||||
video_manager,
|
video_manager,
|
||||||
|
Some(&mut ex),
|
||||||
);
|
);
|
||||||
|
if !ex.instances.is_empty() && !ex.drew_other {
|
||||||
|
// Pure video: discard the (now-empty) scene and emit GPU instances.
|
||||||
|
rendered.scene = Scene::new();
|
||||||
|
rendered.has_content = true;
|
||||||
|
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
|
||||||
|
} else {
|
||||||
|
if !ex.instances.is_empty() {
|
||||||
|
// Mixed video + vector: the first pass diverted the video out of
|
||||||
|
// the scene, so re-render with no sink to bake it back in (correct
|
||||||
|
// z-order; Vello path). Rare — fast-splitting is deferred.
|
||||||
|
rendered.scene = Scene::new();
|
||||||
|
render_vector_layer_to_scene(
|
||||||
|
document, time, vector_layer, &mut rendered.scene,
|
||||||
|
base_transform, 1.0, image_cache, video_manager, None,
|
||||||
|
);
|
||||||
|
}
|
||||||
rendered.has_content = vector_layer.graph_at_time(time)
|
rendered.has_content = vector_layer.graph_at_time(time)
|
||||||
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted))
|
.map_or(false, |graph| !graph.edges.iter().all(|e| e.deleted) || !graph.fills.iter().all(|f| f.deleted))
|
||||||
|| !vector_layer.clip_instances.is_empty();
|
|| !vector_layer.clip_instances.is_empty();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
AnyLayer::Audio(_) => {
|
AnyLayer::Audio(_) => {
|
||||||
// Audio layers don't render visually
|
// Audio layers don't render visually
|
||||||
rendered.has_content = false;
|
rendered.has_content = false;
|
||||||
|
|
@ -577,16 +637,35 @@ pub fn render_layer_isolated(
|
||||||
return RenderedLayer::effect_layer(layer_id, opacity, active_effects);
|
return RenderedLayer::effect_layer(layer_id, opacity, active_effects);
|
||||||
}
|
}
|
||||||
AnyLayer::Group(group_layer) => {
|
AnyLayer::Group(group_layer) => {
|
||||||
// Render each child layer's content into the group's scene
|
// Render each child into the group's scene with an extraction sink. The
|
||||||
|
// common imported-video case is a Group[Video, Audio] — audio draws
|
||||||
|
// nothing, so it's pure video and composites via the GPU Video path.
|
||||||
|
let mut ex = VideoExtract::default();
|
||||||
for child in &group_layer.children {
|
for child in &group_layer.children {
|
||||||
render_layer(
|
render_layer(
|
||||||
document, time, child, &mut rendered.scene, base_transform,
|
document, time, child, &mut rendered.scene, base_transform,
|
||||||
1.0, // Full opacity - layer opacity handled in compositing
|
1.0, // Full opacity - layer opacity handled in compositing
|
||||||
image_cache, video_manager, camera_frame,
|
image_cache, video_manager, camera_frame, Some(&mut ex),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if !ex.instances.is_empty() && !ex.drew_other {
|
||||||
|
rendered.scene = Scene::new();
|
||||||
|
rendered.has_content = true;
|
||||||
|
rendered.layer_type = RenderedLayerType::Video { instances: ex.instances };
|
||||||
|
} else {
|
||||||
|
if !ex.instances.is_empty() {
|
||||||
|
// Mixed: re-render with no sink to bake the video back into the scene.
|
||||||
|
rendered.scene = Scene::new();
|
||||||
|
for child in &group_layer.children {
|
||||||
|
render_layer(
|
||||||
|
document, time, child, &mut rendered.scene, base_transform,
|
||||||
|
1.0, image_cache, video_manager, camera_frame, None,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
rendered.has_content = !group_layer.children.is_empty();
|
rendered.has_content = !group_layer.children.is_empty();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
AnyLayer::Raster(raster_layer) => {
|
AnyLayer::Raster(raster_layer) => {
|
||||||
if let Some(kf) = raster_layer.keyframe_at(time) {
|
if let Some(kf) = raster_layer.keyframe_at(time) {
|
||||||
rendered.has_content = kf.has_pixels();
|
rendered.has_content = kf.has_pixels();
|
||||||
|
|
@ -614,6 +693,7 @@ fn render_vector_layer_to_scene(
|
||||||
parent_opacity: f64,
|
parent_opacity: f64,
|
||||||
image_cache: &mut ImageCache,
|
image_cache: &mut ImageCache,
|
||||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||||
|
extract: Option<&mut VideoExtract>,
|
||||||
) {
|
) {
|
||||||
render_vector_layer(
|
render_vector_layer(
|
||||||
document,
|
document,
|
||||||
|
|
@ -624,6 +704,7 @@ fn render_vector_layer_to_scene(
|
||||||
parent_opacity,
|
parent_opacity,
|
||||||
image_cache,
|
image_cache,
|
||||||
video_manager,
|
video_manager,
|
||||||
|
extract,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -691,10 +772,10 @@ pub fn render_document_with_transform(
|
||||||
for layer in document.visible_layers() {
|
for layer in document.visible_layers() {
|
||||||
if any_soloed {
|
if any_soloed {
|
||||||
if layer.soloed() {
|
if layer.soloed() {
|
||||||
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None);
|
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None);
|
render_layer(document, time, layer, scene, base_transform, 1.0, image_cache, video_manager, None, None);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -755,10 +836,11 @@ fn render_layer(
|
||||||
image_cache: &mut ImageCache,
|
image_cache: &mut ImageCache,
|
||||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||||
camera_frame: Option<&crate::webcam::CaptureFrame>,
|
camera_frame: Option<&crate::webcam::CaptureFrame>,
|
||||||
|
mut extract: Option<&mut VideoExtract>,
|
||||||
) {
|
) {
|
||||||
match layer {
|
match layer {
|
||||||
AnyLayer::Vector(vector_layer) => {
|
AnyLayer::Vector(vector_layer) => {
|
||||||
render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager)
|
render_vector_layer(document, time, vector_layer, scene, base_transform, parent_opacity, image_cache, video_manager, extract)
|
||||||
}
|
}
|
||||||
AnyLayer::Audio(_) => {
|
AnyLayer::Audio(_) => {
|
||||||
// Audio layers don't render visually
|
// Audio layers don't render visually
|
||||||
|
|
@ -766,18 +848,20 @@ fn render_layer(
|
||||||
AnyLayer::Video(video_layer) => {
|
AnyLayer::Video(video_layer) => {
|
||||||
let mut video_mgr = video_manager.lock().unwrap();
|
let mut video_mgr = video_manager.lock().unwrap();
|
||||||
let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None };
|
let layer_camera_frame = if video_layer.camera_enabled { camera_frame } else { None };
|
||||||
render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame);
|
render_video_layer(document, time, video_layer, scene, base_transform, parent_opacity, &mut video_mgr, layer_camera_frame, extract);
|
||||||
}
|
}
|
||||||
AnyLayer::Effect(_) => {
|
AnyLayer::Effect(_) => {
|
||||||
// Effect layers are processed during GPU compositing, not rendered to scene
|
// Effect layers are processed during GPU compositing, not rendered to scene
|
||||||
}
|
}
|
||||||
AnyLayer::Group(group_layer) => {
|
AnyLayer::Group(group_layer) => {
|
||||||
// Render each child layer in the group
|
// Render each child layer in the group, passing the extract sink down.
|
||||||
for child in &group_layer.children {
|
for child in &group_layer.children {
|
||||||
render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame);
|
render_layer(document, time, child, scene, base_transform, parent_opacity, image_cache, video_manager, camera_frame, extract.as_deref_mut());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
AnyLayer::Raster(raster_layer) => {
|
AnyLayer::Raster(raster_layer) => {
|
||||||
|
// Raster is non-video content — force the Vello fallback if extracting.
|
||||||
|
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
|
||||||
render_raster_layer_to_scene(raster_layer, time, scene, base_transform);
|
render_raster_layer_to_scene(raster_layer, time, scene, base_transform);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -816,6 +900,7 @@ pub fn render_single_clip_instance(
|
||||||
render_clip_instance(
|
render_clip_instance(
|
||||||
document, time, clip_instance, layer_opacity, scene, base_transform,
|
document, time, clip_instance, layer_opacity, scene, base_transform,
|
||||||
&vector_layer.layer.animation_data, image_cache, video_manager, group_end_time,
|
&vector_layer.layer.animation_data, image_cache, video_manager, group_end_time,
|
||||||
|
None, // edit-inside-clip overlay keeps the Vello path
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -831,6 +916,7 @@ fn render_clip_instance(
|
||||||
image_cache: &mut ImageCache,
|
image_cache: &mut ImageCache,
|
||||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||||
group_end_time: Option<f64>,
|
group_end_time: Option<f64>,
|
||||||
|
mut extract: Option<&mut VideoExtract>,
|
||||||
) {
|
) {
|
||||||
// Try to find the clip in the document's clip libraries
|
// Try to find the clip in the document's clip libraries
|
||||||
// For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented)
|
// For now, only handle VectorClips (VideoClip and AudioClip rendering not yet implemented)
|
||||||
|
|
@ -972,7 +1058,7 @@ fn render_clip_instance(
|
||||||
if !layer_node.data.visible() {
|
if !layer_node.data.visible() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None);
|
render_layer(document, clip_time, &layer_node.data, scene, instance_transform, clip_opacity, image_cache, video_manager, None, extract.as_deref_mut());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -986,6 +1072,7 @@ fn render_video_layer(
|
||||||
parent_opacity: f64,
|
parent_opacity: f64,
|
||||||
video_manager: &mut crate::video::VideoManager,
|
video_manager: &mut crate::video::VideoManager,
|
||||||
camera_frame: Option<&crate::webcam::CaptureFrame>,
|
camera_frame: Option<&crate::webcam::CaptureFrame>,
|
||||||
|
mut extract: Option<&mut VideoExtract>,
|
||||||
) {
|
) {
|
||||||
use crate::animation::TransformProperty;
|
use crate::animation::TransformProperty;
|
||||||
|
|
||||||
|
|
@ -1151,7 +1238,18 @@ fn render_video_layer(
|
||||||
Affine::IDENTITY
|
Affine::IDENTITY
|
||||||
};
|
};
|
||||||
|
|
||||||
// Render video frame as image fill
|
// Extract to the GPU Video path when a sink is present; otherwise bake into
|
||||||
|
// the Vello scene. The combined frame-pixel → document transform is
|
||||||
|
// instance_transform * brush_transform (matching the top-level Video path).
|
||||||
|
if let Some(ex) = extract.as_deref_mut() {
|
||||||
|
ex.instances.push(VideoRenderInstance {
|
||||||
|
rgba_data: frame.rgba_data.clone(),
|
||||||
|
width: frame.width,
|
||||||
|
height: frame.height,
|
||||||
|
transform: instance_transform * brush_transform,
|
||||||
|
opacity: final_opacity,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
scene.fill(
|
scene.fill(
|
||||||
Fill::NonZero,
|
Fill::NonZero,
|
||||||
instance_transform,
|
instance_transform,
|
||||||
|
|
@ -1159,6 +1257,7 @@ fn render_video_layer(
|
||||||
Some(brush_transform),
|
Some(brush_transform),
|
||||||
&video_rect,
|
&video_rect,
|
||||||
);
|
);
|
||||||
|
}
|
||||||
clip_rendered = true;
|
clip_rendered = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1194,6 +1293,17 @@ fn render_video_layer(
|
||||||
* Affine::translate((offset_x, offset_y))
|
* Affine::translate((offset_x, offset_y))
|
||||||
* Affine::scale(uniform_scale);
|
* Affine::scale(uniform_scale);
|
||||||
|
|
||||||
|
// preview_transform maps frame-pixel space → document directly, so it
|
||||||
|
// is exactly the instance transform for the GPU path.
|
||||||
|
if let Some(ex) = extract.as_deref_mut() {
|
||||||
|
ex.instances.push(VideoRenderInstance {
|
||||||
|
rgba_data: frame.rgba_data.clone(),
|
||||||
|
width: frame.width,
|
||||||
|
height: frame.height,
|
||||||
|
transform: preview_transform,
|
||||||
|
opacity: final_opacity,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
scene.fill(
|
scene.fill(
|
||||||
Fill::NonZero,
|
Fill::NonZero,
|
||||||
preview_transform,
|
preview_transform,
|
||||||
|
|
@ -1204,6 +1314,7 @@ fn render_video_layer(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Compute start/end canvas points for a linear gradient across a bounding box.
|
/// Compute start/end canvas points for a linear gradient across a bounding box.
|
||||||
///
|
///
|
||||||
|
|
@ -1352,6 +1463,7 @@ fn render_vector_layer(
|
||||||
parent_opacity: f64,
|
parent_opacity: f64,
|
||||||
image_cache: &mut ImageCache,
|
image_cache: &mut ImageCache,
|
||||||
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
video_manager: &std::sync::Arc<std::sync::Mutex<crate::video::VideoManager>>,
|
||||||
|
mut extract: Option<&mut VideoExtract>,
|
||||||
) {
|
) {
|
||||||
// Cascade opacity: parent_opacity × layer.opacity
|
// Cascade opacity: parent_opacity × layer.opacity
|
||||||
let layer_opacity = parent_opacity * layer.layer.opacity;
|
let layer_opacity = parent_opacity * layer.layer.opacity;
|
||||||
|
|
@ -1359,6 +1471,10 @@ fn render_vector_layer(
|
||||||
// Render the layer's own VectorGraph (loose shapes) first, then clip instances
|
// Render the layer's own VectorGraph (loose shapes) first, then clip instances
|
||||||
// (groups / movie clips) on top. Shape tweens are applied here.
|
// (groups / movie clips) on top. Shape tweens are applied here.
|
||||||
if let Some(graph) = layer.tweened_graph_at(time) {
|
if let Some(graph) = layer.tweened_graph_at(time) {
|
||||||
|
// Loose vector geometry (and any image-asset fills) is non-video content —
|
||||||
|
// force the Vello fallback. Conservative: a present-but-empty graph still
|
||||||
|
// trips this, which only costs the fallback, never correctness.
|
||||||
|
if let Some(ex) = extract.as_deref_mut() { ex.drew_other = true; }
|
||||||
render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache);
|
render_vector_graph(&graph, scene, base_transform, layer_opacity, document, image_cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1370,7 +1486,7 @@ fn render_vector_layer(
|
||||||
let frame_duration = 1.0 / document.framerate;
|
let frame_duration = 1.0 / document.framerate;
|
||||||
layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration)
|
layer.group_visibility_end(&clip_instance.id, clip_instance.timeline_start, frame_duration)
|
||||||
});
|
});
|
||||||
render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time);
|
render_clip_instance(document, time, clip_instance, layer_opacity, scene, base_transform, &layer.layer.animation_data, image_cache, video_manager, group_end_time, extract.as_deref_mut());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,77 @@ pub fn update_prepare_timing(
|
||||||
t.composite_ms = composite_ms;
|
t.composite_ms = composite_ms;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/// GPU-measured composite cost (from timestamp queries; see `gpu_timer.rs`).
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct GpuCompositeTiming {
|
||||||
|
/// True when the adapter supports timestamp queries (else the ms is meaningless).
|
||||||
|
pub supported: bool,
|
||||||
|
/// GPU time of the whole composite section (Vello render + sRGB→linear +
|
||||||
|
/// compositor + tonemap), in milliseconds. Read back asynchronously, so it
|
||||||
|
/// lags the displayed frame by a frame or two.
|
||||||
|
pub composite_gpu_ms: f64,
|
||||||
|
/// Layers composited this frame.
|
||||||
|
pub layers: u32,
|
||||||
|
/// `queue.submit()` calls in the composite section this frame.
|
||||||
|
pub submits: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
static GPU_COMPOSITE: OnceLock<Mutex<GpuCompositeTiming>> = OnceLock::new();
|
||||||
|
|
||||||
|
/// Called from `VelloCallback::prepare()` with the GPU composite measurement.
|
||||||
|
pub fn update_gpu_composite(supported: bool, composite_gpu_ms: f64, layers: u32, submits: u32) {
|
||||||
|
let cell = GPU_COMPOSITE.get_or_init(|| Mutex::new(GpuCompositeTiming::default()));
|
||||||
|
if let Ok(mut t) = cell.lock() {
|
||||||
|
t.supported = supported;
|
||||||
|
t.composite_gpu_ms = composite_gpu_ms;
|
||||||
|
t.layers = layers;
|
||||||
|
t.submits = submits;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_gpu_composite() -> GpuCompositeTiming {
|
||||||
|
GPU_COMPOSITE
|
||||||
|
.get_or_init(|| Mutex::new(GpuCompositeTiming::default()))
|
||||||
|
.lock()
|
||||||
|
.map(|t| t.clone())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// CPU-side breakdown of the composite section (wall-clock `Instant` deltas). Since
|
||||||
|
/// the GPU idles waiting on these CPU operations, this is where the per-frame cost
|
||||||
|
/// actually lives. Sums should ≈ the CPU `composite_ms` for the doc's active paths.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct CompositeCpuBreakdown {
|
||||||
|
/// `renderer.render_to_texture` — Vello scene encode + its internal submit.
|
||||||
|
pub vello_ms: f64,
|
||||||
|
/// `srgb_to_linear.convert` — recording the conversion pass.
|
||||||
|
pub convert_ms: f64,
|
||||||
|
/// `canvas_blit.blit` — recording + its internal submit.
|
||||||
|
pub blit_ms: f64,
|
||||||
|
/// `compositor.composite` — recording + per-call uniforms buffer / bind group alloc.
|
||||||
|
pub composite_ms: f64,
|
||||||
|
/// Explicit `queue.submit()` calls.
|
||||||
|
pub submit_ms: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
static COMPOSITE_CPU: OnceLock<Mutex<CompositeCpuBreakdown>> = OnceLock::new();
|
||||||
|
|
||||||
|
/// Called from `VelloCallback::prepare()` with the composite CPU breakdown.
|
||||||
|
pub fn update_composite_cpu(b: CompositeCpuBreakdown) {
|
||||||
|
let cell = COMPOSITE_CPU.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()));
|
||||||
|
if let Ok(mut t) = cell.lock() {
|
||||||
|
*t = b;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_composite_cpu() -> CompositeCpuBreakdown {
|
||||||
|
COMPOSITE_CPU
|
||||||
|
.get_or_init(|| Mutex::new(CompositeCpuBreakdown::default()))
|
||||||
|
.lock()
|
||||||
|
.map(|t| t.clone())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
/// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the
|
/// GPU memory the editor tracks itself (wgpu has no allocator query). Currently the
|
||||||
/// raster-layer texture cache — the only unbounded-by-default VRAM consumer.
|
/// raster-layer texture cache — the only unbounded-by-default VRAM consumer.
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
|
|
@ -90,6 +161,12 @@ pub struct DebugStats {
|
||||||
// GPU prepare() timing breakdown (from render thread)
|
// GPU prepare() timing breakdown (from render thread)
|
||||||
pub prepare_timing: PrepareTiming,
|
pub prepare_timing: PrepareTiming,
|
||||||
|
|
||||||
|
// GPU-measured composite cost (timestamp queries)
|
||||||
|
pub gpu_composite: GpuCompositeTiming,
|
||||||
|
|
||||||
|
// CPU breakdown of the composite section
|
||||||
|
pub composite_cpu: CompositeCpuBreakdown,
|
||||||
|
|
||||||
// Performance metrics for each section
|
// Performance metrics for each section
|
||||||
pub timing_memory_us: u64,
|
pub timing_memory_us: u64,
|
||||||
pub timing_gpu_us: u64,
|
pub timing_gpu_us: u64,
|
||||||
|
|
@ -254,6 +331,8 @@ impl DebugStatsCollector {
|
||||||
audio_input_devices,
|
audio_input_devices,
|
||||||
has_pointer,
|
has_pointer,
|
||||||
prepare_timing,
|
prepare_timing,
|
||||||
|
gpu_composite: get_gpu_composite(),
|
||||||
|
composite_cpu: get_composite_cpu(),
|
||||||
timing_memory_us,
|
timing_memory_us,
|
||||||
timing_gpu_us,
|
timing_gpu_us,
|
||||||
timing_midi_us,
|
timing_midi_us,
|
||||||
|
|
@ -306,8 +385,33 @@ pub fn render_debug_overlay(ctx: &egui::Context, stats: &DebugStats) {
|
||||||
ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms));
|
ui.colored_label(egui::Color32::YELLOW, format!("GPU prepare: {:.2} ms", pt.total_ms));
|
||||||
ui.label(format!(" removals: {:.2} ms", pt.removals_ms));
|
ui.label(format!(" removals: {:.2} ms", pt.removals_ms));
|
||||||
ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms));
|
ui.label(format!(" gpu_dispatch: {:.2} ms", pt.gpu_dispatches_ms));
|
||||||
ui.label(format!(" scene_build: {:.2} ms", pt.scene_build_ms));
|
ui.label(format!(" scene_build: {:.2} ms (CPU)", pt.scene_build_ms));
|
||||||
ui.label(format!(" composite: {:.2} ms", pt.composite_ms));
|
ui.label(format!(" composite: {:.2} ms (CPU)", pt.composite_ms));
|
||||||
|
|
||||||
|
// GPU-measured composite cost (timestamp queries).
|
||||||
|
let gc = &stats.gpu_composite;
|
||||||
|
if gc.supported {
|
||||||
|
ui.colored_label(
|
||||||
|
egui::Color32::LIGHT_GREEN,
|
||||||
|
format!("GPU composite: {:.2} ms (GPU)", gc.composite_gpu_ms),
|
||||||
|
);
|
||||||
|
ui.label(format!(" layers: {} submits: {}", gc.layers, gc.submits));
|
||||||
|
} else {
|
||||||
|
ui.label(format!(
|
||||||
|
"GPU composite: n/a (no timestamp support) layers: {} submits: {}",
|
||||||
|
gc.layers, gc.submits
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
// CPU breakdown of the composite (where the GPU is actually waiting).
|
||||||
|
let cc = &stats.composite_cpu;
|
||||||
|
let cc_sum = cc.vello_ms + cc.convert_ms + cc.blit_ms + cc.composite_ms + cc.submit_ms;
|
||||||
|
ui.colored_label(egui::Color32::LIGHT_BLUE, format!("Composite CPU breakdown: {:.2} ms", cc_sum));
|
||||||
|
ui.label(format!(" vello(render): {:.2} ms", cc.vello_ms));
|
||||||
|
ui.label(format!(" srgb→linear: {:.2} ms", cc.convert_ms));
|
||||||
|
ui.label(format!(" blit: {:.2} ms", cc.blit_ms));
|
||||||
|
ui.label(format!(" compositor: {:.2} ms", cc.composite_ms));
|
||||||
|
ui.label(format!(" queue.submit: {:.2} ms", cc.submit_ms));
|
||||||
|
|
||||||
ui.add_space(8.0);
|
ui.add_space(8.0);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -830,22 +830,13 @@ fn composite_document_to_hdr(
|
||||||
if inst.rgba_data.is_empty() { continue; }
|
if inst.rgba_data.is_empty() { continue; }
|
||||||
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
|
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
|
||||||
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
|
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
|
||||||
// sRGB straight-alpha → linear premultiplied
|
// Upload raw sRGB straight-alpha bytes into an sRGB texture; the GPU
|
||||||
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| {
|
// decodes to linear on sample (no per-pixel CPU conversion). Blit with
|
||||||
let a = p[3] as f32 / 255.0;
|
// blit_straight so the shader doesn't unpremultiply.
|
||||||
let lin = |c: u8| -> f32 {
|
let tex = upload_transient_texture(device, queue, &inst.rgba_data, inst.width, inst.height, wgpu::TextureFormat::Rgba8UnormSrgb, Some("export_video_frame_tex"));
|
||||||
let f = c as f32 / 255.0;
|
|
||||||
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) }
|
|
||||||
};
|
|
||||||
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8;
|
|
||||||
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8;
|
|
||||||
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
|
|
||||||
[r, g, b, p[3]]
|
|
||||||
}).collect();
|
|
||||||
let tex = upload_transient_texture(device, queue, &linear, inst.width, inst.height, Some("export_video_frame_tex"));
|
|
||||||
let tex_view = tex.create_view(&Default::default());
|
let tex_view = tex.create_view(&Default::default());
|
||||||
let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height);
|
let bt = crate::gpu_brush::BlitTransform::new(inst.transform, inst.width, inst.height, width, height);
|
||||||
gpu_resources.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
gpu_resources.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||||
let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal);
|
let compositor_layer = CompositorLayer::new(hdr_layer_handle, inst.opacity, lightningbeam_core::gpu::BlendMode::Normal);
|
||||||
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") });
|
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("export_video_composite") });
|
||||||
gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None);
|
gpu_resources.compositor.composite(device, queue, &mut enc, &[compositor_layer], &gpu_resources.buffer_pool, &gpu_resources.hdr_texture_view, None);
|
||||||
|
|
@ -865,7 +856,7 @@ fn composite_document_to_hdr(
|
||||||
};
|
};
|
||||||
[lin(p[0]), lin(p[1]), lin(p[2]), p[3]]
|
[lin(p[0]), lin(p[1]), lin(p[2]), p[3]]
|
||||||
}).collect();
|
}).collect();
|
||||||
let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, Some("export_float_tex"));
|
let tex = upload_transient_texture(device, queue, &linear, *fw, *fh, wgpu::TextureFormat::Rgba8Unorm, Some("export_float_tex"));
|
||||||
let tex_view = tex.create_view(&Default::default());
|
let tex_view = tex.create_view(&Default::default());
|
||||||
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
|
let hdr_layer_handle = gpu_resources.buffer_pool.acquire(device, hdr_spec);
|
||||||
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
|
if let Some(hdr_layer_view) = gpu_resources.buffer_pool.get_view(hdr_layer_handle) {
|
||||||
|
|
@ -919,13 +910,16 @@ fn composite_document_to_hdr(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Upload `pixels` to a transient `Rgba8Unorm` GPU texture (TEXTURE_BINDING | COPY_DST).
|
/// Upload `pixels` to a transient GPU texture (TEXTURE_BINDING | COPY_DST) in the
|
||||||
|
/// given format. Use `Rgba8UnormSrgb` to upload raw sRGB bytes and let the GPU
|
||||||
|
/// decode to linear on sample (no CPU conversion).
|
||||||
fn upload_transient_texture(
|
fn upload_transient_texture(
|
||||||
device: &wgpu::Device,
|
device: &wgpu::Device,
|
||||||
queue: &wgpu::Queue,
|
queue: &wgpu::Queue,
|
||||||
pixels: &[u8],
|
pixels: &[u8],
|
||||||
width: u32,
|
width: u32,
|
||||||
height: u32,
|
height: u32,
|
||||||
|
format: wgpu::TextureFormat,
|
||||||
label: Option<&'static str>,
|
label: Option<&'static str>,
|
||||||
) -> wgpu::Texture {
|
) -> wgpu::Texture {
|
||||||
let tex = device.create_texture(&wgpu::TextureDescriptor {
|
let tex = device.create_texture(&wgpu::TextureDescriptor {
|
||||||
|
|
@ -933,7 +927,7 @@ fn upload_transient_texture(
|
||||||
size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 },
|
size: wgpu::Extent3d { width, height, depth_or_array_layers: 1 },
|
||||||
mip_level_count: 1, sample_count: 1,
|
mip_level_count: 1, sample_count: 1,
|
||||||
dimension: wgpu::TextureDimension::D2,
|
dimension: wgpu::TextureDimension::D2,
|
||||||
format: wgpu::TextureFormat::Rgba8Unorm,
|
format,
|
||||||
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
||||||
view_formats: &[],
|
view_formats: &[],
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1951,6 +1951,9 @@ impl GpuBrushEngine {
|
||||||
/// the camera transform.
|
/// the camera transform.
|
||||||
pub struct CanvasBlitPipeline {
|
pub struct CanvasBlitPipeline {
|
||||||
pub pipeline: wgpu::RenderPipeline,
|
pub pipeline: wgpu::RenderPipeline,
|
||||||
|
/// Variant for straight-alpha sources (hardware-sRGB video frames): the
|
||||||
|
/// fragment shader skips the unpremultiply. See [`CanvasBlitPipeline::blit_straight`].
|
||||||
|
pub pipeline_straight: wgpu::RenderPipeline,
|
||||||
pub bg_layout: wgpu::BindGroupLayout,
|
pub bg_layout: wgpu::BindGroupLayout,
|
||||||
pub sampler: wgpu::Sampler,
|
pub sampler: wgpu::Sampler,
|
||||||
/// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res
|
/// Bilinear sampler for smooth upscaling (used by `blit_smooth`, e.g. low-res
|
||||||
|
|
@ -2132,6 +2135,39 @@ impl CanvasBlitPipeline {
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Variant pipeline for straight-alpha sources (hardware-sRGB video frames):
|
||||||
|
// identical except the fragment shader skips the unpremultiply.
|
||||||
|
let pipeline_straight = device.create_render_pipeline(
|
||||||
|
&wgpu::RenderPipelineDescriptor {
|
||||||
|
label: Some("canvas_blit_pipeline_straight"),
|
||||||
|
layout: Some(&pipeline_layout),
|
||||||
|
vertex: wgpu::VertexState {
|
||||||
|
module: &shader,
|
||||||
|
entry_point: Some("vs_main"),
|
||||||
|
buffers: &[],
|
||||||
|
compilation_options: Default::default(),
|
||||||
|
},
|
||||||
|
fragment: Some(wgpu::FragmentState {
|
||||||
|
module: &shader,
|
||||||
|
entry_point: Some("fs_main_straight"),
|
||||||
|
targets: &[Some(wgpu::ColorTargetState {
|
||||||
|
format: wgpu::TextureFormat::Rgba16Float,
|
||||||
|
blend: None,
|
||||||
|
write_mask: wgpu::ColorWrites::ALL,
|
||||||
|
})],
|
||||||
|
compilation_options: Default::default(),
|
||||||
|
}),
|
||||||
|
primitive: wgpu::PrimitiveState {
|
||||||
|
topology: wgpu::PrimitiveTopology::TriangleStrip,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
depth_stencil: None,
|
||||||
|
multisample: wgpu::MultisampleState::default(),
|
||||||
|
multiview: None,
|
||||||
|
cache: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
|
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
|
||||||
label: Some("canvas_blit_sampler"),
|
label: Some("canvas_blit_sampler"),
|
||||||
address_mode_u: wgpu::AddressMode::ClampToEdge,
|
address_mode_u: wgpu::AddressMode::ClampToEdge,
|
||||||
|
|
@ -2165,7 +2201,7 @@ impl CanvasBlitPipeline {
|
||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
|
|
||||||
Self { pipeline, bg_layout, sampler, linear_sampler, mask_sampler }
|
Self { pipeline, pipeline_straight, bg_layout, sampler, linear_sampler, mask_sampler }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Render the canvas texture into `target_view` (Rgba16Float) with the given camera.
|
/// Render the canvas texture into `target_view` (Rgba16Float) with the given camera.
|
||||||
|
|
@ -2183,7 +2219,7 @@ impl CanvasBlitPipeline {
|
||||||
transform: &BlitTransform,
|
transform: &BlitTransform,
|
||||||
mask_view: Option<&wgpu::TextureView>,
|
mask_view: Option<&wgpu::TextureView>,
|
||||||
) {
|
) {
|
||||||
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler);
|
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies).
|
/// Blit with a bilinear sampler — smooth upscaling for low-res sources (proxies).
|
||||||
|
|
@ -2196,9 +2232,25 @@ impl CanvasBlitPipeline {
|
||||||
transform: &BlitTransform,
|
transform: &BlitTransform,
|
||||||
mask_view: Option<&wgpu::TextureView>,
|
mask_view: Option<&wgpu::TextureView>,
|
||||||
) {
|
) {
|
||||||
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler);
|
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.linear_sampler, &self.pipeline);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Blit a **straight-alpha** source (e.g. a video frame uploaded to an
|
||||||
|
/// `Rgba8UnormSrgb` texture, hardware-decoded to linear on sample). Uses the
|
||||||
|
/// `fs_main_straight` pipeline, which skips the unpremultiply that `blit` does.
|
||||||
|
pub fn blit_straight(
|
||||||
|
&self,
|
||||||
|
device: &wgpu::Device,
|
||||||
|
queue: &wgpu::Queue,
|
||||||
|
canvas_view: &wgpu::TextureView,
|
||||||
|
target_view: &wgpu::TextureView,
|
||||||
|
transform: &BlitTransform,
|
||||||
|
mask_view: Option<&wgpu::TextureView>,
|
||||||
|
) {
|
||||||
|
self.blit_with(device, queue, canvas_view, target_view, transform, mask_view, &self.sampler, &self.pipeline_straight);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn blit_with(
|
fn blit_with(
|
||||||
&self,
|
&self,
|
||||||
device: &wgpu::Device,
|
device: &wgpu::Device,
|
||||||
|
|
@ -2208,6 +2260,7 @@ impl CanvasBlitPipeline {
|
||||||
transform: &BlitTransform,
|
transform: &BlitTransform,
|
||||||
mask_view: Option<&wgpu::TextureView>,
|
mask_view: Option<&wgpu::TextureView>,
|
||||||
canvas_sampler: &wgpu::Sampler,
|
canvas_sampler: &wgpu::Sampler,
|
||||||
|
pipeline: &wgpu::RenderPipeline,
|
||||||
) {
|
) {
|
||||||
// When no mask is provided, create a temporary 1×1 all-white texture.
|
// When no mask is provided, create a temporary 1×1 all-white texture.
|
||||||
// (queue is already available here, unlike in new())
|
// (queue is already available here, unlike in new())
|
||||||
|
|
@ -2296,7 +2349,7 @@ impl CanvasBlitPipeline {
|
||||||
occlusion_query_set: None,
|
occlusion_query_set: None,
|
||||||
timestamp_writes: None,
|
timestamp_writes: None,
|
||||||
});
|
});
|
||||||
rp.set_pipeline(&self.pipeline);
|
rp.set_pipeline(pipeline);
|
||||||
rp.set_bind_group(0, &bg, &[]);
|
rp.set_bind_group(0, &bg, &[]);
|
||||||
rp.draw(0..4, 0..1);
|
rp.draw(0..4, 0..1);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,135 @@
|
||||||
|
//! Minimal GPU timestamp timer for the composite pipeline.
|
||||||
|
//!
|
||||||
|
//! Brackets a section of GPU work with two timestamps and reads the elapsed GPU
|
||||||
|
//! time back asynchronously (no pipeline stall). Used to attribute the per-frame
|
||||||
|
//! composite cost (Vello render + sRGB→linear + compositor + tonemap) shown in F3.
|
||||||
|
//!
|
||||||
|
//! Requires `TIMESTAMP_QUERY` + `TIMESTAMP_QUERY_INSIDE_ENCODERS`; [`FrameGpuTimer::new`]
|
||||||
|
//! returns `None` when the adapter doesn't support them, and all call sites no-op.
|
||||||
|
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
/// State of the single readback buffer (shared with the map callback).
|
||||||
|
#[derive(Clone, Copy, PartialEq)]
|
||||||
|
enum Readback {
|
||||||
|
/// Available to resolve into this frame.
|
||||||
|
Free,
|
||||||
|
/// Submitted + `map_async` in flight; don't touch until the callback fires.
|
||||||
|
Mapping,
|
||||||
|
/// Mapped and ready to read.
|
||||||
|
Ready,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Times one GPU section (two timestamps) per frame with intermittent async readback.
|
||||||
|
pub struct FrameGpuTimer {
|
||||||
|
query_set: wgpu::QuerySet,
|
||||||
|
resolve_buf: wgpu::Buffer,
|
||||||
|
readback_buf: wgpu::Buffer,
|
||||||
|
state: Arc<Mutex<Readback>>,
|
||||||
|
/// Nanoseconds per timestamp tick.
|
||||||
|
period_ns: f32,
|
||||||
|
/// Most recent measured GPU time for the bracketed section, in milliseconds.
|
||||||
|
last_ms: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FrameGpuTimer {
|
||||||
|
/// Required device features for GPU timestamp timing.
|
||||||
|
pub fn required_features() -> wgpu::Features {
|
||||||
|
wgpu::Features::TIMESTAMP_QUERY | wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a timer, or `None` if the device lacks timestamp support.
|
||||||
|
pub fn new(device: &wgpu::Device, queue: &wgpu::Queue) -> Option<Self> {
|
||||||
|
if !device.features().contains(Self::required_features()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let query_set = device.create_query_set(&wgpu::QuerySetDescriptor {
|
||||||
|
label: Some("composite_gpu_timer"),
|
||||||
|
ty: wgpu::QueryType::Timestamp,
|
||||||
|
count: 2,
|
||||||
|
});
|
||||||
|
// 2 timestamps × u64.
|
||||||
|
let size = 2 * std::mem::size_of::<u64>() as u64;
|
||||||
|
let resolve_buf = device.create_buffer(&wgpu::BufferDescriptor {
|
||||||
|
label: Some("composite_gpu_timer_resolve"),
|
||||||
|
size,
|
||||||
|
usage: wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC,
|
||||||
|
mapped_at_creation: false,
|
||||||
|
});
|
||||||
|
let readback_buf = device.create_buffer(&wgpu::BufferDescriptor {
|
||||||
|
label: Some("composite_gpu_timer_readback"),
|
||||||
|
size,
|
||||||
|
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
|
||||||
|
mapped_at_creation: false,
|
||||||
|
});
|
||||||
|
Some(Self {
|
||||||
|
query_set,
|
||||||
|
resolve_buf,
|
||||||
|
readback_buf,
|
||||||
|
state: Arc::new(Mutex::new(Readback::Free)),
|
||||||
|
period_ns: queue.get_timestamp_period(),
|
||||||
|
last_ms: 0.0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write the **start** timestamp (call just before the bracketed GPU work).
|
||||||
|
pub fn start(&self, device: &wgpu::Device, queue: &wgpu::Queue) {
|
||||||
|
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||||
|
label: Some("composite_gpu_timer_start"),
|
||||||
|
});
|
||||||
|
enc.write_timestamp(&self.query_set, 0);
|
||||||
|
queue.submit(Some(enc.finish()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write the **end** timestamp and, if the readback buffer is free, resolve +
|
||||||
|
/// kick off an async read. Also consumes a previously-completed read into
|
||||||
|
/// `last_ms`. Call just after the bracketed GPU work.
|
||||||
|
pub fn end(&mut self, device: &wgpu::Device, queue: &wgpu::Queue) {
|
||||||
|
// 1. Consume a completed readback first (so the buffer is free to reuse).
|
||||||
|
let cur = *self.state.lock().unwrap();
|
||||||
|
if cur == Readback::Ready {
|
||||||
|
{
|
||||||
|
let view = self.readback_buf.slice(..).get_mapped_range();
|
||||||
|
let t0 = u64::from_le_bytes(view[0..8].try_into().unwrap());
|
||||||
|
let t1 = u64::from_le_bytes(view[8..16].try_into().unwrap());
|
||||||
|
// Timestamps can wrap or arrive out of order across queue resets; guard.
|
||||||
|
let ticks = t1.saturating_sub(t0);
|
||||||
|
self.last_ms = ticks as f64 * self.period_ns as f64 / 1.0e6;
|
||||||
|
}
|
||||||
|
self.readback_buf.unmap();
|
||||||
|
*self.state.lock().unwrap() = Readback::Free;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. End timestamp + resolve + copy, only when the buffer is free.
|
||||||
|
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||||
|
label: Some("composite_gpu_timer_end"),
|
||||||
|
});
|
||||||
|
enc.write_timestamp(&self.query_set, 1);
|
||||||
|
|
||||||
|
let can_read = *self.state.lock().unwrap() == Readback::Free;
|
||||||
|
if can_read {
|
||||||
|
enc.resolve_query_set(&self.query_set, 0..2, &self.resolve_buf, 0);
|
||||||
|
enc.copy_buffer_to_buffer(
|
||||||
|
&self.resolve_buf,
|
||||||
|
0,
|
||||||
|
&self.readback_buf,
|
||||||
|
0,
|
||||||
|
2 * std::mem::size_of::<u64>() as u64,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
queue.submit(Some(enc.finish()));
|
||||||
|
|
||||||
|
if can_read {
|
||||||
|
*self.state.lock().unwrap() = Readback::Mapping;
|
||||||
|
let state = Arc::clone(&self.state);
|
||||||
|
self.readback_buf.slice(..).map_async(wgpu::MapMode::Read, move |res| {
|
||||||
|
*state.lock().unwrap() = if res.is_ok() { Readback::Ready } else { Readback::Free };
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Most recently measured GPU time of the bracketed section, in milliseconds.
|
||||||
|
pub fn last_ms(&self) -> f64 {
|
||||||
|
self.last_ms
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -50,6 +50,7 @@ use effect_thumbnails::EffectThumbnailGenerator;
|
||||||
mod custom_cursor;
|
mod custom_cursor;
|
||||||
mod tablet;
|
mod tablet;
|
||||||
mod debug_overlay;
|
mod debug_overlay;
|
||||||
|
mod gpu_timer;
|
||||||
|
|
||||||
#[cfg(debug_assertions)]
|
#[cfg(debug_assertions)]
|
||||||
mod test_mode;
|
mod test_mode;
|
||||||
|
|
@ -174,8 +175,12 @@ fn main() -> eframe::Result {
|
||||||
device_descriptor: std::sync::Arc::new(|adapter| {
|
device_descriptor: std::sync::Arc::new(|adapter| {
|
||||||
let features = adapter.features();
|
let features = adapter.features();
|
||||||
// Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's
|
// Request SHADER_F16 if available — needed on Mesa/llvmpipe for vello's
|
||||||
// unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability)
|
// unpack2x16float (enables the SHADER_F16_IN_F32 downlevel capability).
|
||||||
let optional_features = wgpu::Features::SHADER_F16;
|
// TIMESTAMP_QUERY(+INSIDE_ENCODERS) drives the F3 GPU composite timer
|
||||||
|
// (gpu_timer.rs); both are optional and no-op when unsupported.
|
||||||
|
let optional_features = wgpu::Features::SHADER_F16
|
||||||
|
| wgpu::Features::TIMESTAMP_QUERY
|
||||||
|
| wgpu::Features::TIMESTAMP_QUERY_INSIDE_ENCODERS;
|
||||||
|
|
||||||
let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl {
|
let base_limits = if adapter.get_info().backend == wgpu::Backend::Gl {
|
||||||
wgpu::Limits::downlevel_webgl2_defaults()
|
wgpu::Limits::downlevel_webgl2_defaults()
|
||||||
|
|
|
||||||
|
|
@ -75,3 +75,25 @@ fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
|
||||||
let tinted = base + tint - base * tint;
|
let tinted = base + tint - base * tint;
|
||||||
return vec4<f32>(tinted, masked_a);
|
return vec4<f32>(tinted, masked_a);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Variant for sources that are ALREADY straight-alpha linear — notably a video
|
||||||
|
// frame uploaded to an `Rgba8UnormSrgb` texture, where the hardware decodes
|
||||||
|
// sRGB→linear on sample and leaves alpha untouched. No unpremultiply (the source
|
||||||
|
// was never premultiplied), so we skip the divide entirely. The compositor wants
|
||||||
|
// straight-alpha linear, which is exactly what the sample already is.
|
||||||
|
@fragment
|
||||||
|
fn fs_main_straight(in: VertexOutput) -> @location(0) vec4<f32> {
|
||||||
|
let m = mat3x3<f32>(transform.col0.xyz, transform.col1.xyz, transform.col2.xyz);
|
||||||
|
let canvas_uv = (m * vec3<f32>(in.uv.x, in.uv.y, 1.0)).xy;
|
||||||
|
|
||||||
|
if canvas_uv.x < 0.0 || canvas_uv.x > 1.0
|
||||||
|
|| canvas_uv.y < 0.0 || canvas_uv.y > 1.0 {
|
||||||
|
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
let c = textureSample(canvas_tex, canvas_sampler, canvas_uv);
|
||||||
|
let mask = textureSample(mask_tex, mask_sampler, canvas_uv).r;
|
||||||
|
let tint = vec3<f32>(transform.col0.w, transform.col1.w, transform.col2.w);
|
||||||
|
let tinted = c.rgb + tint - c.rgb * tint;
|
||||||
|
return vec4<f32>(tinted, c.a * mask);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,89 @@ fn upload_pixmap_to_texture(queue: &wgpu::Queue, texture: &wgpu::Texture, pixmap
|
||||||
/// Set to true to use the new pipeline, false for legacy single-scene rendering
|
/// Set to true to use the new pipeline, false for legacy single-scene rendering
|
||||||
const USE_HDR_COMPOSITING: bool = true; // Enabled for testing
|
const USE_HDR_COMPOSITING: bool = true; // Enabled for testing
|
||||||
|
|
||||||
|
/// Caches GPU textures for decoded video frames, keyed by the frame buffer's `Arc`
|
||||||
|
/// identity. A static/paused video then costs ~nothing per repaint (cache hit → no
|
||||||
|
/// per-pixel CPU sRGB→linear conversion, no texture allocation, no upload). The
|
||||||
|
/// cached texture holds premultiplied-linear RGBA8 — exactly what `canvas_blit`
|
||||||
|
/// expects. During playback each new decoded frame is a fresh `Arc` → one
|
||||||
|
/// conversion+upload per frame (not per repaint).
|
||||||
|
struct CachedVideoFrame {
|
||||||
|
/// Keep the source buffer alive so its pointer (our cache key) can't be reused.
|
||||||
|
_keep: std::sync::Arc<Vec<u8>>,
|
||||||
|
texture: wgpu::Texture,
|
||||||
|
last_seen: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct VideoFrameTexCache {
|
||||||
|
entries: std::collections::HashMap<usize, CachedVideoFrame>,
|
||||||
|
frame: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VideoFrameTexCache {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self { entries: std::collections::HashMap::new(), frame: 0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn begin_frame(&mut self) {
|
||||||
|
self.frame = self.frame.wrapping_add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// View of the cached (or freshly converted+uploaded) texture for `rgba`.
|
||||||
|
fn texture_view(
|
||||||
|
&mut self,
|
||||||
|
device: &wgpu::Device,
|
||||||
|
queue: &wgpu::Queue,
|
||||||
|
rgba: &std::sync::Arc<Vec<u8>>,
|
||||||
|
w: u32,
|
||||||
|
h: u32,
|
||||||
|
) -> wgpu::TextureView {
|
||||||
|
let key = std::sync::Arc::as_ptr(rgba) as usize;
|
||||||
|
let frame = self.frame;
|
||||||
|
if let Some(e) = self.entries.get_mut(&key) {
|
||||||
|
e.last_seen = frame;
|
||||||
|
return e.texture.create_view(&wgpu::TextureViewDescriptor::default());
|
||||||
|
}
|
||||||
|
// Miss: upload the raw sRGB straight-alpha bytes verbatim into an sRGB
|
||||||
|
// texture. The GPU decodes sRGB→linear on sample (free), so there is no
|
||||||
|
// per-pixel CPU conversion — the cost that used to dominate playback/export.
|
||||||
|
// Blit this with `blit_straight` (it must NOT unpremultiply).
|
||||||
|
let texture = device.create_texture(&wgpu::TextureDescriptor {
|
||||||
|
label: Some("video_frame_tex"),
|
||||||
|
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
|
||||||
|
mip_level_count: 1,
|
||||||
|
sample_count: 1,
|
||||||
|
dimension: wgpu::TextureDimension::D2,
|
||||||
|
format: wgpu::TextureFormat::Rgba8UnormSrgb,
|
||||||
|
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
||||||
|
view_formats: &[],
|
||||||
|
});
|
||||||
|
queue.write_texture(
|
||||||
|
wgpu::TexelCopyTextureInfo {
|
||||||
|
texture: &texture,
|
||||||
|
mip_level: 0,
|
||||||
|
origin: wgpu::Origin3d::ZERO,
|
||||||
|
aspect: wgpu::TextureAspect::All,
|
||||||
|
},
|
||||||
|
rgba,
|
||||||
|
wgpu::TexelCopyBufferLayout {
|
||||||
|
offset: 0,
|
||||||
|
bytes_per_row: Some(w * 4),
|
||||||
|
rows_per_image: Some(h),
|
||||||
|
},
|
||||||
|
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
|
||||||
|
);
|
||||||
|
let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
|
||||||
|
self.entries.insert(key, CachedVideoFrame { _keep: rgba.clone(), texture, last_seen: frame });
|
||||||
|
view
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop textures not used in the last couple of frames (bounds VRAM).
|
||||||
|
fn evict_stale(&mut self) {
|
||||||
|
let frame = self.frame;
|
||||||
|
self.entries.retain(|_, e| e.last_seen + 2 >= frame);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Shared Vello resources (created once, reused by all Stage panes)
|
/// Shared Vello resources (created once, reused by all Stage panes)
|
||||||
struct SharedVelloResources {
|
struct SharedVelloResources {
|
||||||
renderer: Arc<Mutex<vello::Renderer>>,
|
renderer: Arc<Mutex<vello::Renderer>>,
|
||||||
|
|
@ -72,6 +155,11 @@ struct SharedVelloResources {
|
||||||
/// True when Vello is running its CPU software renderer (either forced or GPU fallback).
|
/// True when Vello is running its CPU software renderer (either forced or GPU fallback).
|
||||||
/// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area.
|
/// Used to select cheaper antialiasing — Msaa16 on CPU costs 16× as much as Area.
|
||||||
is_cpu_renderer: bool,
|
is_cpu_renderer: bool,
|
||||||
|
/// GPU timestamp timer for the composite section (F3 debug). Lazily created on
|
||||||
|
/// the first frame (needs the device/queue); `None` if unsupported.
|
||||||
|
gpu_timer: Mutex<Option<crate::gpu_timer::FrameGpuTimer>>,
|
||||||
|
/// Per-frame video texture cache (skips re-converting/uploading a static frame).
|
||||||
|
video_frame_cache: Mutex<VideoFrameTexCache>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Per-instance Vello resources (created for each Stage pane)
|
/// Per-instance Vello resources (created for each Stage pane)
|
||||||
|
|
@ -302,6 +390,8 @@ impl SharedVelloResources {
|
||||||
gpu_brush: Mutex::new(gpu_brush),
|
gpu_brush: Mutex::new(gpu_brush),
|
||||||
canvas_blit,
|
canvas_blit,
|
||||||
is_cpu_renderer: use_cpu || is_cpu_renderer,
|
is_cpu_renderer: use_cpu || is_cpu_renderer,
|
||||||
|
gpu_timer: Mutex::new(None),
|
||||||
|
video_frame_cache: Mutex::new(VideoFrameTexCache::new()),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1079,6 +1169,25 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
// HDR buffer spec for linear buffers
|
// HDR buffer spec for linear buffers
|
||||||
let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float);
|
let hdr_spec = BufferSpec::new(width, height, BufferFormat::Rgba16Float);
|
||||||
|
|
||||||
|
// F3: bracket the composite section with a GPU timestamp (lazily create the
|
||||||
|
// timer; no-op when the adapter lacks timestamp support).
|
||||||
|
let ts_supported = device
|
||||||
|
.features()
|
||||||
|
.contains(crate::gpu_timer::FrameGpuTimer::required_features());
|
||||||
|
{
|
||||||
|
let mut tg = shared.gpu_timer.lock().unwrap();
|
||||||
|
if tg.is_none() && ts_supported {
|
||||||
|
*tg = crate::gpu_timer::FrameGpuTimer::new(device, queue);
|
||||||
|
}
|
||||||
|
if let Some(t) = tg.as_ref() {
|
||||||
|
t.start(device, queue);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
shared.video_frame_cache.lock().unwrap().begin_frame();
|
||||||
|
|
||||||
|
// F3: CPU breakdown of the composite (the GPU idles waiting on these).
|
||||||
|
let mut cput = crate::debug_overlay::CompositeCpuBreakdown::default();
|
||||||
|
|
||||||
// First, render background and composite it
|
// First, render background and composite it
|
||||||
// The background scene contains only a rectangle at document bounds,
|
// The background scene contains only a rectangle at document bounds,
|
||||||
// so we use TRANSPARENT base_color to not fill the whole viewport
|
// so we use TRANSPARENT base_color to not fill the whole viewport
|
||||||
|
|
@ -1097,6 +1206,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
antialiasing_method: aa_method,
|
antialiasing_method: aa_method,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let _t = std::time::Instant::now();
|
||||||
if let Some(pixmap) = &composite_result.background_cpu {
|
if let Some(pixmap) = &composite_result.background_cpu {
|
||||||
if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) {
|
if let Some(tex) = buffer_pool.get_texture(bg_srgb_handle) {
|
||||||
upload_pixmap_to_texture(queue, tex, pixmap);
|
upload_pixmap_to_texture(queue, tex, pixmap);
|
||||||
|
|
@ -1104,13 +1214,18 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
} else if let Ok(mut renderer) = shared.renderer.lock() {
|
} else if let Ok(mut renderer) = shared.renderer.lock() {
|
||||||
renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok();
|
renderer.render_to_texture(device, queue, &composite_result.background, bg_srgb_view, &bg_render_params).ok();
|
||||||
}
|
}
|
||||||
|
cput.vello_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
|
||||||
// Convert sRGB to linear HDR
|
// Convert sRGB to linear HDR
|
||||||
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||||
label: Some("bg_srgb_to_linear_encoder"),
|
label: Some("bg_srgb_to_linear_encoder"),
|
||||||
});
|
});
|
||||||
|
let _t = std::time::Instant::now();
|
||||||
shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view);
|
shared.srgb_to_linear.convert(device, &mut convert_encoder, bg_srgb_view, bg_hdr_view);
|
||||||
|
cput.convert_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
let _t = std::time::Instant::now();
|
||||||
queue.submit(Some(convert_encoder.finish()));
|
queue.submit(Some(convert_encoder.finish()));
|
||||||
|
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
|
||||||
// Composite background onto HDR texture (first layer, clears to dark gray for stage area)
|
// Composite background onto HDR texture (first layer, clears to dark gray for stage area)
|
||||||
let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0);
|
let bg_compositor_layer = lightningbeam_core::gpu::CompositorLayer::normal(bg_hdr_handle, 1.0);
|
||||||
|
|
@ -1120,6 +1235,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
// Clear to dark gray (stage background outside document bounds)
|
// Clear to dark gray (stage background outside document bounds)
|
||||||
// Note: stage_bg values are already in linear space for HDR compositing
|
// Note: stage_bg values are already in linear space for HDR compositing
|
||||||
let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0];
|
let stage_bg = [45.0 / 255.0, 45.0 / 255.0, 48.0 / 255.0, 1.0];
|
||||||
|
let _t = std::time::Instant::now();
|
||||||
shared.compositor.composite(
|
shared.compositor.composite(
|
||||||
device,
|
device,
|
||||||
queue,
|
queue,
|
||||||
|
|
@ -1129,7 +1245,10 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
hdr_view,
|
hdr_view,
|
||||||
Some(stage_bg),
|
Some(stage_bg),
|
||||||
);
|
);
|
||||||
|
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
let _t = std::time::Instant::now();
|
||||||
queue.submit(Some(encoder.finish()));
|
queue.submit(Some(encoder.finish()));
|
||||||
|
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
}
|
}
|
||||||
buffer_pool.release(bg_srgb_handle);
|
buffer_pool.release(bg_srgb_handle);
|
||||||
buffer_pool.release(bg_hdr_handle);
|
buffer_pool.release(bg_hdr_handle);
|
||||||
|
|
@ -1351,6 +1470,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
buffer_pool.get_view(hdr_layer_handle),
|
buffer_pool.get_view(hdr_layer_handle),
|
||||||
&instance_resources.hdr_texture_view,
|
&instance_resources.hdr_texture_view,
|
||||||
) {
|
) {
|
||||||
|
let _t_vello = std::time::Instant::now();
|
||||||
if let Some(pixmap) = &rendered_layer.cpu_pixmap {
|
if let Some(pixmap) = &rendered_layer.cpu_pixmap {
|
||||||
if let Some(tex) = buffer_pool.get_texture(srgb_handle) {
|
if let Some(tex) = buffer_pool.get_texture(srgb_handle) {
|
||||||
upload_pixmap_to_texture(queue, tex, pixmap);
|
upload_pixmap_to_texture(queue, tex, pixmap);
|
||||||
|
|
@ -1358,6 +1478,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
} else if let Ok(mut renderer) = shared.renderer.lock() {
|
} else if let Ok(mut renderer) = shared.renderer.lock() {
|
||||||
renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok();
|
renderer.render_to_texture(device, queue, &rendered_layer.scene, srgb_view, &layer_render_params).ok();
|
||||||
}
|
}
|
||||||
|
cput.vello_ms += _t_vello.elapsed().as_secs_f64() * 1000.0;
|
||||||
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
let mut convert_encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||||
label: Some("layer_srgb_to_linear_encoder"),
|
label: Some("layer_srgb_to_linear_encoder"),
|
||||||
});
|
});
|
||||||
|
|
@ -1555,7 +1676,7 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
RenderedLayerType::Video { instances } => {
|
RenderedLayerType::Video { instances } => {
|
||||||
// Video layer — per-instance: upload decoded frame → blit → composite.
|
// Video layer — per-instance: (cached) frame texture → blit → composite.
|
||||||
for inst in instances {
|
for inst in instances {
|
||||||
if inst.rgba_data.is_empty() { continue; }
|
if inst.rgba_data.is_empty() { continue; }
|
||||||
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
|
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
|
||||||
|
|
@ -1563,40 +1684,20 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
buffer_pool.get_view(hdr_layer_handle),
|
buffer_pool.get_view(hdr_layer_handle),
|
||||||
&instance_resources.hdr_texture_view,
|
&instance_resources.hdr_texture_view,
|
||||||
) {
|
) {
|
||||||
// Convert sRGB straight-alpha → linear premultiplied.
|
// Reuse the GPU texture for this frame if it's unchanged (a
|
||||||
let linear: Vec<u8> = inst.rgba_data.chunks_exact(4).flat_map(|p| {
|
// static/paused video → no CPU conversion, alloc, or upload).
|
||||||
let a = p[3] as f32 / 255.0;
|
// Timed into `blit_ms` (incl the cache lookup + per-frame view).
|
||||||
let lin = |c: u8| -> f32 {
|
let _t = std::time::Instant::now();
|
||||||
let f = c as f32 / 255.0;
|
let tex_view = shared
|
||||||
if f <= 0.04045 { f / 12.92 } else { ((f + 0.055) / 1.055).powf(2.4) }
|
.video_frame_cache
|
||||||
};
|
.lock()
|
||||||
let r = (lin(p[0]) * a * 255.0 + 0.5) as u8;
|
.unwrap()
|
||||||
let g = (lin(p[1]) * a * 255.0 + 0.5) as u8;
|
.texture_view(device, queue, &inst.rgba_data, inst.width, inst.height);
|
||||||
let b = (lin(p[2]) * a * 255.0 + 0.5) as u8;
|
|
||||||
[r, g, b, p[3]]
|
|
||||||
}).collect();
|
|
||||||
|
|
||||||
let tex = device.create_texture(&wgpu::TextureDescriptor {
|
|
||||||
label: Some("video_frame_tex"),
|
|
||||||
size: wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
|
|
||||||
mip_level_count: 1, sample_count: 1,
|
|
||||||
dimension: wgpu::TextureDimension::D2,
|
|
||||||
format: wgpu::TextureFormat::Rgba8Unorm,
|
|
||||||
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
|
|
||||||
view_formats: &[],
|
|
||||||
});
|
|
||||||
queue.write_texture(
|
|
||||||
wgpu::TexelCopyTextureInfo { texture: &tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
|
|
||||||
&linear,
|
|
||||||
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(inst.width * 4), rows_per_image: Some(inst.height) },
|
|
||||||
wgpu::Extent3d { width: inst.width, height: inst.height, depth_or_array_layers: 1 },
|
|
||||||
);
|
|
||||||
let tex_view = tex.create_view(&wgpu::TextureViewDescriptor::default());
|
|
||||||
|
|
||||||
let bt = crate::gpu_brush::BlitTransform::new(
|
let bt = crate::gpu_brush::BlitTransform::new(
|
||||||
inst.transform, inst.width, inst.height, width, height,
|
inst.transform, inst.width, inst.height, width, height,
|
||||||
);
|
);
|
||||||
shared.canvas_blit.blit(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
shared.canvas_blit.blit_straight(device, queue, &tex_view, hdr_layer_view, &bt, None);
|
||||||
|
cput.blit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
|
||||||
let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(
|
let compositor_layer = lightningbeam_core::gpu::CompositorLayer::new(
|
||||||
hdr_layer_handle,
|
hdr_layer_handle,
|
||||||
|
|
@ -1606,10 +1707,14 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||||
label: Some("video_composite_encoder"),
|
label: Some("video_composite_encoder"),
|
||||||
});
|
});
|
||||||
|
let _t = std::time::Instant::now();
|
||||||
shared.compositor.composite(
|
shared.compositor.composite(
|
||||||
device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None,
|
device, queue, &mut encoder, &[compositor_layer], &buffer_pool, hdr_view, None,
|
||||||
);
|
);
|
||||||
|
cput.composite_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
|
let _t = std::time::Instant::now();
|
||||||
queue.submit(Some(encoder.finish()));
|
queue.submit(Some(encoder.finish()));
|
||||||
|
cput.submit_ms += _t.elapsed().as_secs_f64() * 1000.0;
|
||||||
}
|
}
|
||||||
buffer_pool.release(hdr_layer_handle);
|
buffer_pool.release(hdr_layer_handle);
|
||||||
}
|
}
|
||||||
|
|
@ -1841,6 +1946,23 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
|
||||||
buffer_pool.next_frame();
|
buffer_pool.next_frame();
|
||||||
drop(buffer_pool);
|
drop(buffer_pool);
|
||||||
|
|
||||||
|
// F3: close the GPU timestamp bracket + publish the composite measurement.
|
||||||
|
{
|
||||||
|
let layers = composite_result.layers.len() as u32;
|
||||||
|
// Submits aren't counted per-site; estimate from the per-layer pattern
|
||||||
|
// (bg ~3 + ~2 per layer). Drops toward ~1 once the passes are batched.
|
||||||
|
let submits_est = 3 + 2 * layers;
|
||||||
|
let mut tg = shared.gpu_timer.lock().unwrap();
|
||||||
|
if let Some(t) = tg.as_mut() {
|
||||||
|
t.end(device, queue);
|
||||||
|
crate::debug_overlay::update_gpu_composite(true, t.last_ms(), layers, submits_est);
|
||||||
|
} else {
|
||||||
|
crate::debug_overlay::update_gpu_composite(false, 0.0, layers, submits_est);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
crate::debug_overlay::update_composite_cpu(cput);
|
||||||
|
shared.video_frame_cache.lock().unwrap().evict_stale();
|
||||||
|
|
||||||
// --- Frame timing report ---
|
// --- Frame timing report ---
|
||||||
let _t_end = std::time::Instant::now();
|
let _t_end = std::time::Instant::now();
|
||||||
let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0;
|
let total_ms = (_t_end - _t_prepare_start).as_secs_f64() * 1000.0;
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue