Phase 3a-3: low-res image proxy for cold-scrub raster frames

Scrubbing onto a paged-out raster keyframe flashed blank for the 1-2 frames its
full pixels took to page in. Now a low-res proxy is shown in that gap.

- core: `MediaKind::RasterProxy` (id derived from the keyframe id via
  `raster_proxy_media_id`); `brush_engine::encode_raster_proxy_png` downscales a full
  RGBA buffer to a ≤192px-long-edge PNG. Save writes a proxy beside each resident
  frame's full PNG (paged-out frames keep their existing proxy row, like the full).
  Load eagerly decodes proxies (small) into `RasterKeyframe::proxy`.
- editor: a separate `proxy_layer_cache` in the GPU brush (own recency LRU, budget 64
  since each is ~1/100th a full frame) + `ensure_proxy_texture`/`get_proxy_texture`.
  The raster render, when the full texture isn't resident, blits the proxy mapped to
  the keyframe's FULL logical dims so it upscales via the sampler. F3 VRAM figure now
  includes proxy textures.

When the full pixels land (async fault-in), the full path takes over automatically.
Proxies only exist after a save+reload; freshly-painted unsaved frames stay resident
so they need none.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Skyler Lehmkuhl 2026-06-20 20:05:46 -04:00
parent a1e3cc841f
commit 1bfd09f151
6 changed files with 176 additions and 12 deletions

View File

@ -58,6 +58,10 @@ pub enum MediaKind {
/// A pack of precomputed video thumbnails for a video clip (keyed by a /// A pack of precomputed video thumbnails for a video clip (keyed by a
/// sentinel-derived id from the clip id). Opaque blob; format owned by the editor. /// sentinel-derived id from the clip id). Opaque blob; format owned by the editor.
Thumbnail = 5, Thumbnail = 5,
/// A low-res PNG proxy of a raster keyframe (keyed by a sentinel-derived id from
/// the keyframe id). Decoded eagerly on load and shown while the full-res pixels
/// page in, so cold scrubs don't flash blank. See `raster_proxy_media_id`.
RasterProxy = 6,
} }
impl MediaKind { impl MediaKind {
@ -69,6 +73,7 @@ impl MediaKind {
3 => Some(Self::ImageAsset), 3 => Some(Self::ImageAsset),
4 => Some(Self::Waveform), 4 => Some(Self::Waveform),
5 => Some(Self::Thumbnail), 5 => Some(Self::Thumbnail),
6 => Some(Self::RasterProxy),
_ => None, _ => None,
} }
} }

View File

@ -575,6 +575,27 @@ pub fn encode_png(img: &RgbaImage) -> Result<Vec<u8>, String> {
Ok(buf.into_inner()) Ok(buf.into_inner())
} }
/// Long-edge cap (pixels) for raster keyframe proxies — low-res page-in placeholders.
pub const RASTER_PROXY_MAX_EDGE: u32 = 192;
/// Downscale a full RGBA buffer to a low-res proxy PNG (long edge ≤
/// `RASTER_PROXY_MAX_EDGE`). Returns `None` on invalid dims/length or encode failure.
pub fn encode_raster_proxy_png(raw: &[u8], width: u32, height: u32) -> Option<Vec<u8>> {
if width == 0 || height == 0 || raw.len() != (width as usize * height as usize * 4) {
return None;
}
let long = width.max(height);
let (pw, ph) = if long <= RASTER_PROXY_MAX_EDGE {
(width, height)
} else {
let s = RASTER_PROXY_MAX_EDGE as f32 / long as f32;
(((width as f32 * s).round() as u32).max(1), ((height as f32 * s).round() as u32).max(1))
};
let img = RgbaImage::from_raw(width, height, raw.to_vec())?;
let resized = image::imageops::resize(&img, pw, ph, image::imageops::FilterType::Triangle);
encode_png(&resized).ok()
}
/// Decode PNG bytes into an `RgbaImage` /// Decode PNG bytes into an `RgbaImage`
pub fn decode_png(data: &[u8]) -> Result<RgbaImage, String> { pub fn decode_png(data: &[u8]) -> Result<RgbaImage, String> {
image::load_from_memory(data) image::load_from_memory(data)

View File

@ -254,6 +254,14 @@ fn thumbnail_media_id(clip_id: Uuid) -> Uuid {
Uuid::from_u128(clip_id.as_u128() ^ SENTINEL) Uuid::from_u128(clip_id.as_u128() ^ SENTINEL)
} }
/// Derived id for a raster keyframe's low-res proxy row (distinct from the keyframe's
/// own full-res `Raster` row, which is keyed by the raw keyframe id).
fn raster_proxy_media_id(kf_id: Uuid) -> Uuid {
// "LBPX" repeated — distinct id region from the full raster + thumbnail rows.
const SENTINEL: u128 = 0x4C42_5058_4C42_5058_4C42_5058_4C42_5058;
Uuid::from_u128(kf_id.as_u128() ^ SENTINEL)
}
pub fn save_beam( pub fn save_beam(
path: &Path, path: &Path,
document: &Document, document: &Document,
@ -418,9 +426,24 @@ pub fn save_beam(
} }
Err(e) => eprintln!("⚠️ [SAVE_BEAM] Failed to encode raster {}: {}", kf.id, e), Err(e) => eprintln!("⚠️ [SAVE_BEAM] Failed to encode raster {}: {}", kf.id, e),
} }
// Low-res proxy alongside the full PNG (shown while the full pages
// in on a later load). Regenerated from the resident pixels.
let proxy_id = raster_proxy_media_id(kf.id);
if let Some(proxy_png) =
crate::brush_engine::encode_raster_proxy_png(&kf.raw_pixels, kf.width, kf.height)
{
txn.put_media_packed(
proxy_id, MediaKind::RasterProxy, "png", &proxy_png, MediaMeta::default(),
)?;
live_media.insert(proxy_id);
}
} else if txn.media_exists(kf.id)? { } else if txn.media_exists(kf.id)? {
// Pixels not resident but already stored — keep the row. // Pixels not resident but already stored — keep both rows.
live_media.insert(kf.id); live_media.insert(kf.id);
let proxy_id = raster_proxy_media_id(kf.id);
if txn.media_exists(proxy_id)? {
live_media.insert(proxy_id);
}
} }
} }
} }
@ -574,15 +597,33 @@ fn load_beam_sqlite(path: &Path) -> Result<LoadedProject, String> {
// instant and only the resident window lives in RAM. Mark every keyframe // instant and only the resident window lives in RAM. Mark every keyframe
// `needs_fault_in` (recursively, incl. nested layers) so the renderer requests a // `needs_fault_in` (recursively, incl. nested layers) so the renderer requests a
// page-in; a freshly-created keyframe stays `false` (blank-resident, nothing to load). // page-in; a freshly-created keyframe stays `false` (blank-resident, nothing to load).
// Proxies (low-res, ~tens of KB each) ARE decoded eagerly so a cold scrub onto a
// not-yet-paged frame shows the proxy instantly instead of flashing blank.
let mut raster_load_count = 0usize; let mut raster_load_count = 0usize;
let mut proxy_load_count = 0usize;
for layer in document.all_layers_mut() { for layer in document.all_layers_mut() {
if let crate::layer::AnyLayer::Raster(rl) = layer { if let crate::layer::AnyLayer::Raster(rl) = layer {
for kf in &mut rl.keyframes { for kf in &mut rl.keyframes {
kf.needs_fault_in = true; kf.needs_fault_in = true;
raster_load_count += 1; raster_load_count += 1;
let proxy_id = raster_proxy_media_id(kf.id);
if let Ok(Some(_)) = archive.media_info(proxy_id) {
if let Ok(bytes) = archive.read_media_full(proxy_id) {
if let Ok(img) = crate::brush_engine::decode_png(&bytes) {
let (w, h) = (img.width(), img.height());
kf.proxy = Some(crate::raster_layer::RasterProxy {
width: w,
height: h,
pixels: img.into_raw(),
});
proxy_load_count += 1;
} }
} }
} }
}
}
}
let _ = proxy_load_count;
// Missing external files (referenced entries whose file no longer exists). // Missing external files (referenced entries whose file no longer exists).
let project_dir = path.parent().unwrap_or_else(|| Path::new(".")); let project_dir = path.parent().unwrap_or_else(|| Path::new("."));

View File

@ -92,6 +92,16 @@ impl Default for TweenType {
} }
} }
/// A low-res decoded RGBA proxy of a keyframe's pixels, shown while the full-res
/// buffer pages in from the container so cold scrubs don't flash blank.
#[derive(Clone, Debug)]
pub struct RasterProxy {
pub width: u32,
pub height: u32,
/// RGBA, `width * height * 4` bytes.
pub pixels: Vec<u8>,
}
/// A single keyframe of a raster layer /// A single keyframe of a raster layer
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
pub struct RasterKeyframe { pub struct RasterKeyframe {
@ -130,6 +140,11 @@ pub struct RasterKeyframe {
/// unsaved edit). Cleared on a successful save. Never serialized. /// unsaved edit). Cleared on a successful save. Never serialized.
#[serde(skip)] #[serde(skip)]
pub dirty: bool, pub dirty: bool,
/// Phase 3a-3: low-res proxy decoded from the container on load, rendered while
/// the full pixels page in (removes the cold-scrub blank flash). `None` if the
/// keyframe has no persisted proxy yet (new/unsaved, or pre-proxy project).
#[serde(skip)]
pub proxy: Option<RasterProxy>,
} }
fn default_true() -> bool { true } fn default_true() -> bool { true }
@ -155,6 +170,7 @@ impl RasterKeyframe {
texture_dirty: true, texture_dirty: true,
needs_fault_in: false, needs_fault_in: false,
dirty: false, dirty: false,
proxy: None,
} }
} }
} }

View File

@ -1054,6 +1054,12 @@ pub struct GpuBrushEngine {
/// timeline would otherwise grow this map without bound (~w·h·16 bytes of VRAM per /// timeline would otherwise grow this map without bound (~w·h·16 bytes of VRAM per
/// entry); we evict the oldest once it exceeds `RASTER_LAYER_CACHE_MAX`. /// entry); we evict the oldest once it exceeds `RASTER_LAYER_CACHE_MAX`.
raster_layer_lru: Vec<Uuid>, raster_layer_lru: Vec<Uuid>,
/// Low-res proxy textures (one small `CanvasPair` per keyframe), shown while the
/// full-res pixels page in. Keyed by keyframe id; separate from
/// `raster_layer_cache` so a proxy and its full frame never collide. Bounded by
/// its own (generous, since each is tiny) recency LRU.
proxy_layer_cache: HashMap<Uuid, CanvasPair>,
proxy_layer_lru: Vec<Uuid>,
} }
/// CPU-side parameters uniform for the compute shader. /// CPU-side parameters uniform for the compute shader.
@ -1077,6 +1083,10 @@ impl GpuBrushEngine {
/// revisit from the faulted-in pixels). /// revisit from the faulted-in pixels).
const RASTER_LAYER_CACHE_MAX: usize = 12; const RASTER_LAYER_CACHE_MAX: usize = 12;
/// Proxies are ~1/100th the VRAM of a full frame, so we can keep many resident
/// for instant cold-scrub display before evicting the least-recently-used.
const RASTER_PROXY_CACHE_MAX: usize = 64;
/// Create the pipeline. Returns `Err` if the device lacks the required /// Create the pipeline. Returns `Err` if the device lacks the required
/// storage-texture capability for `Rgba16Float`. /// storage-texture capability for `Rgba16Float`.
pub fn new(device: &wgpu::Device) -> Self { pub fn new(device: &wgpu::Device) -> Self {
@ -1172,6 +1182,8 @@ impl GpuBrushEngine {
displacement_bufs: HashMap::new(), displacement_bufs: HashMap::new(),
raster_layer_cache: HashMap::new(), raster_layer_cache: HashMap::new(),
raster_layer_lru: Vec::new(), raster_layer_lru: Vec::new(),
proxy_layer_cache: HashMap::new(),
proxy_layer_lru: Vec::new(),
} }
} }
@ -1566,10 +1578,12 @@ impl GpuBrushEngine {
/// Estimated VRAM footprint of `raster_layer_cache` (two `Rgba16Float` textures = /// Estimated VRAM footprint of `raster_layer_cache` (two `Rgba16Float` textures =
/// `w·h·16` bytes per entry), published to the F3 debug overlay. /// `w·h·16` bytes per entry), published to the F3 debug overlay.
fn report_raster_cache_vram(&self) { fn report_raster_cache_vram(&self) {
let bytes: usize = self.raster_layer_cache.values() let bytes = |cache: &HashMap<Uuid, CanvasPair>| -> usize {
.map(|c| (c.width as usize) * (c.height as usize) * 16) cache.values().map(|c| (c.width as usize) * (c.height as usize) * 16).sum()
.sum(); };
crate::debug_overlay::update_gpu_memory(self.raster_layer_cache.len(), bytes); let total = bytes(&self.raster_layer_cache) + bytes(&self.proxy_layer_cache);
let count = self.raster_layer_cache.len() + self.proxy_layer_cache.len();
crate::debug_overlay::update_gpu_memory(count, total);
} }
/// Get the cached display texture for a raster layer keyframe. /// Get the cached display texture for a raster layer keyframe.
@ -1577,6 +1591,47 @@ impl GpuBrushEngine {
self.raster_layer_cache.get(kf_id) self.raster_layer_cache.get(kf_id)
} }
/// Ensure a low-res proxy texture exists for `kf_id` (uploaded once; proxies are
/// immutable). Bumps recency and evicts the least-recently-used past the budget.
/// `pixels` is sRGB-premultiplied RGBA of length `w * h * 4`.
pub fn ensure_proxy_texture(
&mut self,
device: &wgpu::Device,
queue: &wgpu::Queue,
kf_id: Uuid,
pixels: &[u8],
w: u32,
h: u32,
) {
if pixels.len() != (w * h * 4) as usize {
return; // malformed proxy — skip rather than panic in the render loop
}
let mut changed = false;
if !self.proxy_layer_cache.contains_key(&kf_id) {
let canvas = CanvasPair::new(device, w, h);
canvas.upload(queue, pixels);
self.proxy_layer_cache.insert(kf_id, canvas);
changed = true;
}
if let Some(pos) = self.proxy_layer_lru.iter().position(|id| *id == kf_id) {
self.proxy_layer_lru.remove(pos);
}
self.proxy_layer_lru.push(kf_id);
while self.proxy_layer_lru.len() > Self::RASTER_PROXY_CACHE_MAX {
let old = self.proxy_layer_lru.remove(0);
self.proxy_layer_cache.remove(&old);
changed = true;
}
if changed {
self.report_raster_cache_vram();
}
}
/// Get the cached low-res proxy texture for a raster keyframe.
pub fn get_proxy_texture(&self, kf_id: &Uuid) -> Option<&CanvasPair> {
self.proxy_layer_cache.get(kf_id)
}
/// Remove the cached texture for a raster layer keyframe (e.g. when deleted). /// Remove the cached texture for a raster layer keyframe (e.g. when deleted).
pub fn remove_layer_texture(&mut self, kf_id: &Uuid) { pub fn remove_layer_texture(&mut self, kf_id: &Uuid) {
if self.raster_layer_cache.remove(kf_id).is_some() { if self.raster_layer_cache.remove(kf_id).is_some() {

View File

@ -1175,6 +1175,9 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
// 4. Raster layer texture cache: for idle raster layers (no active tool canvas). // 4. Raster layer texture cache: for idle raster layers (no active tool canvas).
// Upload raw_pixels to the cache if texture_dirty; then use the cache entry. // Upload raw_pixels to the cache if texture_dirty; then use the cache entry.
// If the full pixels aren't resident, fall back to the low-res proxy:
// (kf_id, logical_w, logical_h) so the blit upscales it to full size.
let mut raster_proxy_blit: Option<(uuid::Uuid, u32, u32)> = None;
let raster_cache_kf: Option<uuid::Uuid> = if gpu_canvas_kf.is_none() { let raster_cache_kf: Option<uuid::Uuid> = if gpu_canvas_kf.is_none() {
// Find the active keyframe for this raster layer. // Find the active keyframe for this raster layer.
let doc = &self.ctx.document; let doc = &self.ctx.document;
@ -1221,6 +1224,15 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
reqs.insert(kf_id); reqs.insert(kf_id);
} }
} }
// Show the low-res proxy (if decoded) while the full
// pages in, so the cold scrub doesn't flash blank.
if let Some(proxy) = &kf.proxy {
gpu_brush.ensure_proxy_texture(
device, queue, kf_id,
&proxy.pixels, proxy.width, proxy.height,
);
raster_proxy_blit = Some((kf_id, kf.width, kf.height));
}
None None
} }
} else { } else {
@ -1236,7 +1248,9 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
None None
}; };
if !rendered_layer.has_content && gpu_canvas_kf.is_none() && raster_cache_kf.is_none() { if !rendered_layer.has_content && gpu_canvas_kf.is_none() && raster_cache_kf.is_none()
&& raster_proxy_blit.is_none()
{
continue; continue;
} }
@ -1283,20 +1297,32 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
} }
RenderedLayerType::Raster { transform: layer_transform, .. } => { RenderedLayerType::Raster { transform: layer_transform, .. } => {
// Raster layer — GPU canvas blit directly to HDR (bypasses Vello). // Raster layer — GPU canvas blit directly to HDR (bypasses Vello).
// Tool override canvas (gpu_canvas_kf) takes priority over cached texture. // Tool override canvas (gpu_canvas_kf) takes priority over cached
if let Some(use_kf_id) = gpu_canvas_kf.or(raster_cache_kf) { // texture; if neither full-res source is present, the low-res proxy.
let full_kf = gpu_canvas_kf.or(raster_cache_kf);
if full_kf.is_some() || raster_proxy_blit.is_some() {
let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec); let hdr_layer_handle = buffer_pool.acquire(device, hdr_spec);
if let (Some(hdr_layer_view), Some(hdr_view)) = ( if let (Some(hdr_layer_view), Some(hdr_view)) = (
buffer_pool.get_view(hdr_layer_handle), buffer_pool.get_view(hdr_layer_handle),
&instance_resources.hdr_texture_view, &instance_resources.hdr_texture_view,
) { ) {
if let Ok(gpu_brush) = shared.gpu_brush.lock() { if let Ok(gpu_brush) = shared.gpu_brush.lock() {
let canvas = gpu_brush.canvases.get(&use_kf_id) // Pick the source texture and the LOGICAL dims the blit
.or_else(|| gpu_brush.raster_layer_cache.get(&use_kf_id)); // should map it to. A full texture uses its own dims; a
if let Some(canvas) = canvas { // proxy uses the keyframe's full dims so it upscales.
let blit = if let Some(use_kf_id) = full_kf {
gpu_brush.canvases.get(&use_kf_id)
.or_else(|| gpu_brush.raster_layer_cache.get(&use_kf_id))
.map(|c| (c, c.width, c.height))
} else if let Some((pkf, lw, lh)) = raster_proxy_blit {
gpu_brush.get_proxy_texture(&pkf).map(|c| (c, lw, lh))
} else {
None
};
if let Some((canvas, logical_w, logical_h)) = blit {
let bt = crate::gpu_brush::BlitTransform::new( let bt = crate::gpu_brush::BlitTransform::new(
*layer_transform, *layer_transform,
canvas.width, canvas.height, logical_w, logical_h,
width, height, width, height,
); );
shared.canvas_blit.blit( shared.canvas_blit.blit(